From 6c45945b11d87cac0d38894579a9c9cf17fb9909 Mon Sep 17 00:00:00 2001 From: Ben Halpern Date: Wed, 29 Nov 2023 13:22:18 -0500 Subject: [PATCH] Add recommended articles to feed experiment (Nov 27 experiment) (#20399) * Add recommended articles to feed experiment * remove changes to initial copy * Adjust query to account for null --- app/models/articles/feeds.rb | 3 +- config/feed-variants/20231127-variant-a.json | 146 +++++++++++++++++++ config/feed-variants/20231127-variant-b.json | 146 +++++++++++++++++++ config/feed-variants/20231127-variant-c.json | 146 +++++++++++++++++++ config/field_test.yml | 28 ++++ 5 files changed, 468 insertions(+), 1 deletion(-) create mode 100644 config/feed-variants/20231127-variant-a.json create mode 100644 config/feed-variants/20231127-variant-b.json create mode 100644 config/feed-variants/20231127-variant-c.json diff --git a/app/models/articles/feeds.rb b/app/models/articles/feeds.rb index 92cf649f4..0215a2712 100644 --- a/app/models/articles/feeds.rb +++ b/app/models/articles/feeds.rb @@ -375,7 +375,8 @@ module Articles user_required: true, select_fragment: "CASE WHEN COUNT(first_matching_list.id) = 0 THEN 0 - WHEN articles.id = ANY(array_agg(first_matching_list.article_ids)) THEN 1 + WHEN articles.id = ANY(array_agg(first_matching_list.article_ids) + FILTER (WHERE first_matching_list.article_ids IS NOT NULL)) THEN 1 ELSE 0 END", joins_fragments: ["LEFT OUTER JOIN diff --git a/config/feed-variants/20231127-variant-a.json b/config/feed-variants/20231127-variant-a.json new file mode 100644 index 000000000..ee34d1ca6 --- /dev/null +++ b/config/feed-variants/20231127-variant-a.json @@ -0,0 +1,146 @@ +{ + "max_days_since_published": 15, + "description": "Built off 20231023-variant-b but light addition of recommended articles.", + "order_by": "final_order_by_feed_success_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.905], + [3, 0.7], + [4, 0.6], + [5, 0.5], + [6, 0.3], + [7, 0.2], + [8, 0.02], + [9, 0.01], + [10, 0.009], + [11, 0.008], + [12, 0.007], + [13, 0.006], + [14, 0.0025] + ], + "fallback": 0.01 + }, + "comments_count": { + "cases": [ + [0, 0.15], + [1, 0.3], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.99 + }, + "following_author": { + "cases": [ + [0, 0.7], + [1, 1] + ], + "fallback": 0.7 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.15 }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.2], + [2, 0.15], + [3, 0.1], + [4, 0.05] + ], + "fallback": 0 + }, + "experience": { + "cases": [ + [0, 1], + [1, 0.97], + [2, 0.95], + [3, 0.88], + [4, 0.77], + [5, 0.55], + [6, 0.35], + [7, 0.3] + ], + "default_user_experience_level": 5, + "fallback": 0.2 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.001], + [1, 0.97] + ], + "fallback": 1 + }, + "matching_positive_tags_intersection_points": { + "cases": [ + [0, 0.001], + [1, 0.45], + [2, 0.5], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.9], + [8, 0.93], + [9, 0.95] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions_score": { + "cases": [ + [0, 0.3], + [1, 0.4], + [2, 0.45], + [3, 0.5], + [4, 0.55], + [5, 0.6], + [6, 0.61], + [7, 0.62], + [8, 0.7], + [9, 0.8], + [10, 0.85], + [11, 0.9], + [12, 0.95] + ], + "fallback": 1.0 + }, + "language_match": { + "cases": [ + [0, 0.1], + [1, 1] + ], + "fallback": 1 + }, + "recommended_articles_match": { + "cases": [ + [0, 0.1], + [1, 1] + ], + "fallback": 1 + } + } +} diff --git a/config/feed-variants/20231127-variant-b.json b/config/feed-variants/20231127-variant-b.json new file mode 100644 index 000000000..b66839570 --- /dev/null +++ b/config/feed-variants/20231127-variant-b.json @@ -0,0 +1,146 @@ +{ + "max_days_since_published": 15, + "description": "Built off 20231023-variant-b but medium addition of recommended articles slightly lower on tag.", + "order_by": "final_order_by_feed_success_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.905], + [3, 0.7], + [4, 0.6], + [5, 0.5], + [6, 0.3], + [7, 0.2], + [8, 0.02], + [9, 0.01], + [10, 0.009], + [11, 0.008], + [12, 0.007], + [13, 0.006], + [14, 0.0025] + ], + "fallback": 0.01 + }, + "comments_count": { + "cases": [ + [0, 0.15], + [1, 0.3], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.99 + }, + "following_author": { + "cases": [ + [0, 0.7], + [1, 1] + ], + "fallback": 0.7 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.15 }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.2], + [2, 0.15], + [3, 0.1], + [4, 0.05] + ], + "fallback": 0 + }, + "experience": { + "cases": [ + [0, 1], + [1, 0.97], + [2, 0.95], + [3, 0.88], + [4, 0.77], + [5, 0.55], + [6, 0.35], + [7, 0.3] + ], + "default_user_experience_level": 5, + "fallback": 0.2 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.001], + [1, 0.97] + ], + "fallback": 1 + }, + "matching_positive_tags_intersection_points": { + "cases": [ + [0, 0.005], + [1, 0.45], + [2, 0.5], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.9], + [8, 0.93], + [9, 0.95] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions_score": { + "cases": [ + [0, 0.3], + [1, 0.4], + [2, 0.45], + [3, 0.5], + [4, 0.55], + [5, 0.6], + [6, 0.61], + [7, 0.62], + [8, 0.7], + [9, 0.8], + [10, 0.85], + [11, 0.9], + [12, 0.95] + ], + "fallback": 1.0 + }, + "language_match": { + "cases": [ + [0, 0.1], + [1, 1] + ], + "fallback": 1 + }, + "recommended_articles_match": { + "cases": [ + [0, 0.01], + [1, 1] + ], + "fallback": 1 + } + } +} diff --git a/config/feed-variants/20231127-variant-c.json b/config/feed-variants/20231127-variant-c.json new file mode 100644 index 000000000..380e81dd7 --- /dev/null +++ b/config/feed-variants/20231127-variant-c.json @@ -0,0 +1,146 @@ +{ + "max_days_since_published": 15, + "description": "Built off 20231023-variant-b but heavy addition of recommended articles and lower on tag.", + "order_by": "final_order_by_feed_success_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.905], + [3, 0.7], + [4, 0.6], + [5, 0.5], + [6, 0.3], + [7, 0.2], + [8, 0.02], + [9, 0.01], + [10, 0.009], + [11, 0.008], + [12, 0.007], + [13, 0.006], + [14, 0.0025] + ], + "fallback": 0.01 + }, + "comments_count": { + "cases": [ + [0, 0.15], + [1, 0.3], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.99 + }, + "following_author": { + "cases": [ + [0, 0.7], + [1, 1] + ], + "fallback": 0.7 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.15 }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.2], + [2, 0.15], + [3, 0.1], + [4, 0.05] + ], + "fallback": 0 + }, + "experience": { + "cases": [ + [0, 1], + [1, 0.97], + [2, 0.95], + [3, 0.88], + [4, 0.77], + [5, 0.55], + [6, 0.35], + [7, 0.3] + ], + "default_user_experience_level": 5, + "fallback": 0.2 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.001], + [1, 0.97] + ], + "fallback": 1 + }, + "matching_positive_tags_intersection_points": { + "cases": [ + [0, 0.01], + [1, 0.45], + [2, 0.5], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.9], + [8, 0.93], + [9, 0.95] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions_score": { + "cases": [ + [0, 0.3], + [1, 0.4], + [2, 0.45], + [3, 0.5], + [4, 0.55], + [5, 0.6], + [6, 0.61], + [7, 0.62], + [8, 0.7], + [9, 0.8], + [10, 0.85], + [11, 0.9], + [12, 0.95] + ], + "fallback": 1.0 + }, + "language_match": { + "cases": [ + [0, 0.1], + [1, 1] + ], + "fallback": 1 + }, + "recommended_articles_match": { + "cases": [ + [0, 0.001], + [1, 1] + ], + "fallback": 1 + } + } +} diff --git a/config/field_test.yml b/config/field_test.yml index 712fd9c59..c55c7d88e 100644 --- a/config/field_test.yml +++ b/config/field_test.yml @@ -28,6 +28,32 @@ ################################################################################ experiments: # NOTE: Our feed strategy testing experiment must begin with "feed_strategy" + feed_strategy_starting_202311127: + # NOTE: Required as we want only want to consider for conversion events that + # occurred on or after the given start_date. + started_at: 2023-11-08 + variants: + - 20231023-variant-b + - 20231127-variant-a + - 20231127-variant-b + - 20231127-variant-c + weights: + - 55 + - 15 + - 15 + - 15 + goals: + - user_creates_pageview + - user_creates_article_reaction + - user_creates_comment + - user_views_pages_on_at_least_two_different_days_within_a_week + - user_views_pages_on_at_least_four_different_days_within_a_week + - user_creates_article_reaction_on_four_different_days_within_a_week + - user_creates_comment_on_at_least_four_different_days_within_a_week + - user_views_pages_on_at_least_three_different_hours_within_a_day + - user_views_pages_on_at_least_four_different_hours_within_a_day + - user_views_pages_on_at_least_twelve_different_hours_within_five_days + - user_views_pages_on_at_least_nine_different_days_within_two_weeks feed_strategy_starting_202311108: # NOTE: Required as we want only want to consider for conversion events that # occurred on or after the given start_date. @@ -36,6 +62,8 @@ experiments: - 20231023-variant-b - 20231108-variant-a - 20231108-variant-b + ended_at: 2023-11-27 + winner: 20231023-variant-b weights: - 70 - 15