From 510d85dc7465e24abd438ba025b6001c39335e28 Mon Sep 17 00:00:00 2001 From: Jeremy Friesen Date: Tue, 19 Jul 2022 14:59:53 -0400 Subject: [PATCH] Adding 2022-07-19 variant (#18156) Diff between `20220617-variant-a` and `20220719-variant-a` (e.g. the new incumbent with minor tweaks): ``` 3c3 < "description": "Blend of 20220603-variant-b and 20220603-variant-a", --- > "description": "Copy of 20220617-variant-a with tweak to negative privileged user", 84,85c84,85 < "negative_reaction_threshold": -10, < "positive_reaction_threshold": 10 --- > "negative_reaction_threshold": -9, > "positive_reaction_threshold": 4 ``` **Note**: In the above I'm adjusting the `negative_reaction_threshold` and `positive_reaction_threshold` to reflect the value of the privileged user reaction points: ```ruby class Reaction < ApplicationRecord BASE_POINTS = { "vomit" => -50.0, "thumbsup" => 5.0, "thumbsdown" => -10.0 }.freeze ``` See the [current state of app/models/reaction.rb](https://github.com/forem/forem/blob/817db3e6f851b88e4d694fb906c0620733879d73/app/models/reaction.rb#L1-L6) Further, the thresholds are exclusive (e.g. `<` or `>` not `<=` or `>=`). Diff between `20220719-variant-a` (e.g. the new incumbent) and `20220719-variant-b` (e.g. the challenger) ``` 3c3 < "description": "Copy of 20220617-variant-a with tweak to negative privileged user", --- > "description": "Copy of 20220619-variant-a with tweak to comments range", 37,46c37,54 < [0, 0.8], < [1, 0.82], < [2, 0.84], < [3, 0.86], < [4, 0.88], < [5, 0.9], < [6, 0.92], < [7, 0.94], < [8, 0.96], < [9, 0.98] --- > [0, 0.5], > [1, 0.6], > [2, 0.66], > [3, 0.7], > [4, 0.75], > [5, 0.8], > [6, 0.85], > [7, 0.88], > [8, 0.9], > [9, 0.92], > [12, 1.0], > [18, 1.0], > [22, 1.0], > [25, 1.0], > [30, 1.0], > [35, 1.0], > [40, 1.0], > [45, 1.0] 48c56 < "fallback": 1 --- > "fallback": 0.98 ``` Closes forem/forem#18155 Related to forem/forem-internal-eng#453 --- config/feed-variants/20220719-variant-a.json | 97 +++++++++++++++++ config/feed-variants/20220719-variant-b.json | 105 +++++++++++++++++++ config/field_test.yml | 26 +++++ 3 files changed, 228 insertions(+) create mode 100644 config/feed-variants/20220719-variant-a.json create mode 100644 config/feed-variants/20220719-variant-b.json diff --git a/config/feed-variants/20220719-variant-a.json b/config/feed-variants/20220719-variant-a.json new file mode 100644 index 000000000..7ba2e733f --- /dev/null +++ b/config/feed-variants/20220719-variant-a.json @@ -0,0 +1,97 @@ +{ + "max_days_since_published": 15, + "description": "Copy of 20220617-variant-a with tweak to negative privileged user", + "order_by": "final_order_by_random_weighted_to_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.985], + [3, 0.98], + [4, 0.975], + [5, 0.97], + [6, 0.965], + [7, 0.96], + [8, 0.955], + [9, 0.95], + [10, 0.945], + [11, 0.94], + [12, 0.935], + [13, 0.93], + [14, 0.925] + ], + "fallback": 0.9 + }, + "comments_count_by_those_followed": { + "cases": [ + [0, 0.95], + [1, 0.98], + [2, 0.99] + ], + "fallback": 0.93 + }, + "comments_count": { + "cases": [ + [0, 0.8], + [1, 0.82], + [2, 0.84], + [3, 0.86], + [4, 0.88], + [5, 0.9], + [6, 0.92], + [7, 0.94], + [8, 0.96], + [9, 0.98] + ], + "fallback": 1 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.85 }, + "latest_comment": { + "cases": [ + [0, 1], + [1, 0.9988] + ], + "fallback": 0.988 + }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.5], + [2, 0.4], + [3, 0.3], + [4, 0.2] + ], + "fallback": 0 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.5], + [1, 0.85], + [2, 0.9], + [3, 0.95], + [4, 1] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions": { + "cases": [ + [0, 0.9988], + [1, 0.9988], + [2, 0.9988], + [3, 0.9988] + ], + "fallback": 1 + } + } +} diff --git a/config/feed-variants/20220719-variant-b.json b/config/feed-variants/20220719-variant-b.json new file mode 100644 index 000000000..cc76431fd --- /dev/null +++ b/config/feed-variants/20220719-variant-b.json @@ -0,0 +1,105 @@ +{ + "max_days_since_published": 15, + "description": "Copy of 20220619-variant-a with tweak to comments range", + "order_by": "final_order_by_random_weighted_to_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.985], + [3, 0.98], + [4, 0.975], + [5, 0.97], + [6, 0.965], + [7, 0.96], + [8, 0.955], + [9, 0.95], + [10, 0.945], + [11, 0.94], + [12, 0.935], + [13, 0.93], + [14, 0.925] + ], + "fallback": 0.9 + }, + "comments_count_by_those_followed": { + "cases": [ + [0, 0.95], + [1, 0.98], + [2, 0.99] + ], + "fallback": 0.93 + }, + "comments_count": { + "cases": [ + [0, 0.5], + [1, 0.6], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.98 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.85 }, + "latest_comment": { + "cases": [ + [0, 1], + [1, 0.9988] + ], + "fallback": 0.988 + }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.5], + [2, 0.4], + [3, 0.3], + [4, 0.2] + ], + "fallback": 0 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.5], + [1, 0.85], + [2, 0.9], + [3, 0.95], + [4, 1] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions": { + "cases": [ + [0, 0.9988], + [1, 0.9988], + [2, 0.9988], + [3, 0.9988] + ], + "fallback": 1 + } + } +} diff --git a/config/field_test.yml b/config/field_test.yml index a514f08ab..f7d4c552a 100644 --- a/config/field_test.yml +++ b/config/field_test.yml @@ -28,10 +28,36 @@ ################################################################################ experiments: # NOTE: Our feed strategy testing experiment must begin with "feed_strategy" + feed_strategy_starting_20220719: + # NOTE: Required as we want only want to consider for conversion events that + # occurred on or after the given start_date. + started_at: 2022-07-19 + variants: + - 20220719-variant-a + - 20220719-variant-b + weights: + - 50 + - 50 + goals: + - user_creates_pageview + - user_creates_article_reaction + - user_creates_comment + - user_publishes_post + - user_views_pages_on_at_least_four_different_days_within_a_week + - user_creates_article_reaction_on_four_different_days_within_a_week + - user_creates_comment_on_at_least_four_different_days_within_a_week + - user_publishes_post_on_four_different_days_within_a_week + - user_views_pages_on_at_least_four_different_hours_within_a_day + - user_views_pages_on_at_least_twelve_different_hours_within_five_days + - user_views_pages_on_at_least_nine_different_days_within_two_weeks + - user_publishes_post_at_least_two_times_within_week + - user_publishes_post_at_least_two_times_within_two_weeks feed_strategy_starting_20220712: # NOTE: Required as we want only want to consider for conversion events that # occurred on or after the given start_date. started_at: 2022-07-12 + ended_at: 2022-07-19 + winner: 20220617-variant-a variants: - 20220617-variant-a - 20220712-variant