From 3118f271975721addfde486a43c2cbed72fcfd4a Mon Sep 17 00:00:00 2001 From: Jeremy Friesen Date: Fri, 3 Jun 2022 13:45:10 -0400 Subject: [PATCH] Adding feed experiment for 2022-06-03 (#17824) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit contains two separate but related changes: 1. A check-list for creating a new experiment. 2. A new experiment along with declaring a winner for the previous experiment. Below are the diffs of the two new variants versus the incumbent for the experiment: ``` ❯ diff config/feed-variants/20220603-variant-b.json config/feed-variants/20220518-variant.json 3,4c3 < "description": "As 202205518-variant but with modificiation to `order_by` lever.", < "order_by": "final_order_by_random_weighted_to_score", --- > "order_by": "relevancy_score_and_publication_date", ``` ``` ❯ diff config/feed-variants/20220603-variant-a.json config/feed-variants/20220518-variant.json 3d2 < "description": "As 202205518-variant but with modificiation to `matching_positive_tags_intersection_count`.", 69c68 < [0, 0.5], --- > [0, 0.7], ``` Closes forem/forem#17822 --- config/feed-variants/20220603-variant-a.json | 96 ++++++++++++++++++++ config/feed-variants/20220603-variant-b.json | 96 ++++++++++++++++++++ config/field_test.yml | 53 +++++++++++ 3 files changed, 245 insertions(+) create mode 100644 config/feed-variants/20220603-variant-a.json create mode 100644 config/feed-variants/20220603-variant-b.json diff --git a/config/feed-variants/20220603-variant-a.json b/config/feed-variants/20220603-variant-a.json new file mode 100644 index 000000000..cd5da2ae1 --- /dev/null +++ b/config/feed-variants/20220603-variant-a.json @@ -0,0 +1,96 @@ +{ + "max_days_since_published": 15, + "description": "As 202205518-variant but with modificiation to `matching_positive_tags_intersection_count`.", + "order_by": "relevancy_score_and_publication_date", + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.985], + [3, 0.98], + [4, 0.975], + [5, 0.97], + [6, 0.965], + [7, 0.96], + [8, 0.955], + [9, 0.95], + [10, 0.945], + [11, 0.94], + [12, 0.935], + [13, 0.93], + [14, 0.925] + ], + "fallback": 0.9 + }, + "comments_count_by_those_followed": { + "cases": [ + [0, 0.95], + [1, 0.98], + [2, 0.99] + ], + "fallback": 0.93 + }, + "comments_count": { + "cases": [ + [0, 0.8], + [1, 0.82], + [2, 0.84], + [3, 0.86], + [4, 0.88], + [5, 0.9], + [6, 0.92], + [7, 0.94], + [8, 0.96], + [9, 0.98] + ], + "fallback": 1 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.85 }, + "latest_comment": { + "cases": [ + [0, 1], + [1, 0.9988] + ], + "fallback": 0.988 + }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.5], + [2, 0.4], + [3, 0.3], + [4, 0.2] + ], + "fallback": 0 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.5], + [1, 0.85], + [2, 0.9], + [3, 0.95], + [4, 1] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -10, + "positive_reaction_threshold": 10 + }, + "public_reactions": { + "cases": [ + [0, 0.9988], + [1, 0.9988], + [2, 0.9988], + [3, 0.9988] + ], + "fallback": 1 + } + } +} diff --git a/config/feed-variants/20220603-variant-b.json b/config/feed-variants/20220603-variant-b.json new file mode 100644 index 000000000..437178b63 --- /dev/null +++ b/config/feed-variants/20220603-variant-b.json @@ -0,0 +1,96 @@ +{ + "max_days_since_published": 15, + "description": "As 202205518-variant but with modificiation to `order_by` lever.", + "order_by": "final_order_by_random_weighted_to_score", + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.985], + [3, 0.98], + [4, 0.975], + [5, 0.97], + [6, 0.965], + [7, 0.96], + [8, 0.955], + [9, 0.95], + [10, 0.945], + [11, 0.94], + [12, 0.935], + [13, 0.93], + [14, 0.925] + ], + "fallback": 0.9 + }, + "comments_count_by_those_followed": { + "cases": [ + [0, 0.95], + [1, 0.98], + [2, 0.99] + ], + "fallback": 0.93 + }, + "comments_count": { + "cases": [ + [0, 0.8], + [1, 0.82], + [2, 0.84], + [3, 0.86], + [4, 0.88], + [5, 0.9], + [6, 0.92], + [7, 0.94], + [8, 0.96], + [9, 0.98] + ], + "fallback": 1 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.85 }, + "latest_comment": { + "cases": [ + [0, 1], + [1, 0.9988] + ], + "fallback": 0.988 + }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.5], + [2, 0.4], + [3, 0.3], + [4, 0.2] + ], + "fallback": 0 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.7], + [1, 0.85], + [2, 0.9], + [3, 0.95], + [4, 1] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -10, + "positive_reaction_threshold": 10 + }, + "public_reactions": { + "cases": [ + [0, 0.9988], + [1, 0.9988], + [2, 0.9988], + [3, 0.9988] + ], + "fallback": 1 + } + } +} diff --git a/config/field_test.yml b/config/field_test.yml index 0195d76e8..03f4f122d 100644 --- a/config/field_test.yml +++ b/config/field_test.yml @@ -1,7 +1,60 @@ # See AbExperiment namespace for details of goals and variants +# +################################################################################ +################################################################################ +# +# Checklist for creating a new feed experiment: +# +# - [ ] Create and/or reference the forem/forem issue with desired configuration +# - [ ] Identifyer the winner for the past experiment by adding the `winner: +# ` attribute to the past experiment. +# - [ ] Copy the previous experiment and prepend and rename to +# `feed_strategy_starting_CCYYMMDD` +# - [ ] Update the `start_date` to `CCYY-MM-DD` +# - [ ] Assign the variants; the winner of the previous experiment SHOULD be in +# this array. +# - [ ] Remove the winner from the newly copied `feed_strategy_starting_CCYYMMDD` +# - [ ] Assign weights and make the corresponding +# `./config/feed-variants/*.json` files where * is the new named variants. +# - [ ] Each newly named variant should have a description; typically briefly +# explaining the difference between it and the winner of the last experiment. +# - [ ] Run `bin/rspec spec/services/articles/feeds/variant_query_spec.rb` to +# verify each newly added variant's SQL +# - [ ] Submit a pull request and ping the Forem Core (in Slack) for a review. +# +################################################################################ +################################################################################ experiments: # NOTE: Our feed strategy testing experiment must begin with "feed_strategy" + feed_strategy_starting_20220603: + # NOTE: Required as we want only want to consider for conversion events that + # occurred on or after the given start_date. + start_date: 2022-06-03 + variants: + - 20220518-variant + - 20220603-variant-a + - 20220603-variant-b + weights: + - 50 + - 25 + - 25 + goals: + - user_creates_pageview + - user_creates_article_reaction + - user_creates_comment + - user_publishes_post + - user_views_pages_on_at_least_four_different_days_within_a_week + - user_creates_article_reaction_on_four_different_days_within_a_week + - user_creates_comment_on_at_least_four_different_days_within_a_week + - user_publishes_post_on_four_different_days_within_a_week + # These are historical and may be userful + - user_views_pages_on_at_least_four_different_hours_within_a_day + - user_views_pages_on_at_least_twelve_different_hours_within_five_days + - user_views_pages_on_at_least_nine_different_days_within_two_weeks + - user_publishes_post_at_least_two_times_within_week + - user_publishes_post_at_least_two_times_within_two_weeks feed_strategy_starting_20220526: + winner: 20220518-variant # NOTE: Required as we want only want to consider for conversion events that # occurred on or after the given start_date. start_date: 2022-05-09