diff --git a/app/models/ab_experiment/goal_conversion_handler.rb b/app/models/ab_experiment/goal_conversion_handler.rb index ff7bdd8f1..492a8da0e 100644 --- a/app/models/ab_experiment/goal_conversion_handler.rb +++ b/app/models/ab_experiment/goal_conversion_handler.rb @@ -66,11 +66,21 @@ class AbExperiment if FeatureFlag.accessible?(:field_test_event_single_create_pageview) field_test_converted(experiment, participant: user, goal: goal) # base is someone viewed a page end + pageview_goal(experiment, + [7.days.ago, experiment_start_date].max, + "DATE(created_at)", + 2, + "user_views_pages_on_at_least_two_different_days_within_a_week") pageview_goal(experiment, [7.days.ago, experiment_start_date].max, "DATE(created_at)", 4, "user_views_pages_on_at_least_four_different_days_within_a_week") + pageview_goal(experiment, + [24.hours.ago, experiment_start_date].max, + "DATE_PART('hour', created_at)", + 3, + "user_views_pages_on_at_least_three_different_hours_within_a_day") pageview_goal(experiment, [24.hours.ago, experiment_start_date].max, "DATE_PART('hour', created_at)", diff --git a/config/feed-variants/20220926-variant-a.json b/config/feed-variants/20220926-variant-a.json new file mode 100644 index 000000000..37fcf13ad --- /dev/null +++ b/config/feed-variants/20220926-variant-a.json @@ -0,0 +1,151 @@ +{ + "max_days_since_published": 15, + "description": "Copy of 20220920-variant-b with a sharper daily decay step after day 0", + "order_by": "final_order_by_random_weighted_to_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.91], + [2, 0.905], + [3, 0.9], + [4, 0.89], + [5, 0.88], + [6, 0.87], + [7, 0.86], + [8, 0.85], + [9, 0.84], + [10, 0.83], + [11, 0.82], + [12, 0.81], + [13, 0.8], + [14, 0.79] + ], + "fallback": 0.7 + }, + "comments_count_by_those_followed": { + "cases": [ + [0, 0.95], + [1, 0.98], + [2, 0.99] + ], + "fallback": 0.93 + }, + "comments_count": { + "cases": [ + [0, 0.5], + [1, 0.6], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.98 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.85 }, + "latest_comment": { + "cases": [ + [0, 1], + [1, 0.9988] + ], + "fallback": 0.988 + }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.5], + [2, 0.4], + [3, 0.3], + [4, 0.2] + ], + "fallback": 0 + }, + "matching_positive_tags_intersection_points": { + "cases": [ + [0, 0.55], + [1, 0.6], + [2, 0.65], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.9], + [8, 0.93], + [9, 0.95] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions_score": { + "cases": [ + [0, 0.55], + [1, 0.56], + [2, 0.57], + [3, 0.58], + [4, 0.59], + [5, 0.6], + [6, 0.61], + [7, 0.62], + [8, 0.63], + [9, 0.64], + [10, 0.65], + [11, 0.66], + [12, 0.67], + [13, 0.68], + [14, 0.69], + [15, 0.7], + [16, 0.71], + [17, 0.72], + [18, 0.73], + [19, 0.74], + [20, 0.75], + [21, 0.76], + [22, 0.77], + [23, 0.78], + [24, 0.79], + [25, 0.8], + [26, 0.81], + [27, 0.82], + [28, 0.83], + [29, 0.84], + [30, 0.85], + [31, 0.86], + [32, 0.87], + [33, 0.88], + [34, 0.89], + [35, 0.9], + [36, 0.91], + [37, 0.92], + [38, 0.93], + [39, 0.94], + [40, 0.95], + [41, 0.96], + [42, 0.97], + [43, 0.98], + [44, 0.99] + ], + "fallback": 1 + } + } +} diff --git a/config/feed-variants/20220926-variant-b.json b/config/feed-variants/20220926-variant-b.json new file mode 100644 index 000000000..effee4514 --- /dev/null +++ b/config/feed-variants/20220926-variant-b.json @@ -0,0 +1,151 @@ +{ + "max_days_since_published": 15, + "description": "Copy of 20220920-variant-b with a sharper daily decay step after day 1", + "order_by": "final_order_by_random_weighted_to_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.905], + [3, 0.9], + [4, 0.89], + [5, 0.88], + [6, 0.87], + [7, 0.86], + [8, 0.85], + [9, 0.84], + [10, 0.83], + [11, 0.82], + [12, 0.81], + [13, 0.8], + [14, 0.79] + ], + "fallback": 0.7 + }, + "comments_count_by_those_followed": { + "cases": [ + [0, 0.95], + [1, 0.98], + [2, 0.99] + ], + "fallback": 0.93 + }, + "comments_count": { + "cases": [ + [0, 0.5], + [1, 0.6], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.98 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.85 }, + "latest_comment": { + "cases": [ + [0, 1], + [1, 0.9988] + ], + "fallback": 0.988 + }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.5], + [2, 0.4], + [3, 0.3], + [4, 0.2] + ], + "fallback": 0 + }, + "matching_positive_tags_intersection_points": { + "cases": [ + [0, 0.55], + [1, 0.6], + [2, 0.65], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.9], + [8, 0.93], + [9, 0.95] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions_score": { + "cases": [ + [0, 0.55], + [1, 0.56], + [2, 0.57], + [3, 0.58], + [4, 0.59], + [5, 0.6], + [6, 0.61], + [7, 0.62], + [8, 0.63], + [9, 0.64], + [10, 0.65], + [11, 0.66], + [12, 0.67], + [13, 0.68], + [14, 0.69], + [15, 0.7], + [16, 0.71], + [17, 0.72], + [18, 0.73], + [19, 0.74], + [20, 0.75], + [21, 0.76], + [22, 0.77], + [23, 0.78], + [24, 0.79], + [25, 0.8], + [26, 0.81], + [27, 0.82], + [28, 0.83], + [29, 0.84], + [30, 0.85], + [31, 0.86], + [32, 0.87], + [33, 0.88], + [34, 0.89], + [35, 0.9], + [36, 0.91], + [37, 0.92], + [38, 0.93], + [39, 0.94], + [40, 0.95], + [41, 0.96], + [42, 0.97], + [43, 0.98], + [44, 0.99] + ], + "fallback": 1 + } + } +} diff --git a/config/feed-variants/20220926-variant-c.json b/config/feed-variants/20220926-variant-c.json new file mode 100644 index 000000000..8156fbe01 --- /dev/null +++ b/config/feed-variants/20220926-variant-c.json @@ -0,0 +1,151 @@ +{ + "max_days_since_published": 15, + "description": "Copy of 20220920-variant-b with a sharper daily decay step after day 2", + "order_by": "final_order_by_random_weighted_to_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.98], + [3, 0.9], + [4, 0.89], + [5, 0.88], + [6, 0.87], + [7, 0.86], + [8, 0.85], + [9, 0.84], + [10, 0.83], + [11, 0.82], + [12, 0.81], + [13, 0.8], + [14, 0.79] + ], + "fallback": 0.7 + }, + "comments_count_by_those_followed": { + "cases": [ + [0, 0.95], + [1, 0.98], + [2, 0.99] + ], + "fallback": 0.93 + }, + "comments_count": { + "cases": [ + [0, 0.5], + [1, 0.6], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.98 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.85 }, + "latest_comment": { + "cases": [ + [0, 1], + [1, 0.9988] + ], + "fallback": 0.988 + }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.5], + [2, 0.4], + [3, 0.3], + [4, 0.2] + ], + "fallback": 0 + }, + "matching_positive_tags_intersection_points": { + "cases": [ + [0, 0.55], + [1, 0.6], + [2, 0.65], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.9], + [8, 0.93], + [9, 0.95] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions_score": { + "cases": [ + [0, 0.55], + [1, 0.56], + [2, 0.57], + [3, 0.58], + [4, 0.59], + [5, 0.6], + [6, 0.61], + [7, 0.62], + [8, 0.63], + [9, 0.64], + [10, 0.65], + [11, 0.66], + [12, 0.67], + [13, 0.68], + [14, 0.69], + [15, 0.7], + [16, 0.71], + [17, 0.72], + [18, 0.73], + [19, 0.74], + [20, 0.75], + [21, 0.76], + [22, 0.77], + [23, 0.78], + [24, 0.79], + [25, 0.8], + [26, 0.81], + [27, 0.82], + [28, 0.83], + [29, 0.84], + [30, 0.85], + [31, 0.86], + [32, 0.87], + [33, 0.88], + [34, 0.89], + [35, 0.9], + [36, 0.91], + [37, 0.92], + [38, 0.93], + [39, 0.94], + [40, 0.95], + [41, 0.96], + [42, 0.97], + [43, 0.98], + [44, 0.99] + ], + "fallback": 1 + } + } +} diff --git a/config/field_test.yml b/config/field_test.yml index c28f57ee1..b14effb73 100644 --- a/config/field_test.yml +++ b/config/field_test.yml @@ -28,10 +28,42 @@ ################################################################################ experiments: # NOTE: Our feed strategy testing experiment must begin with "feed_strategy" + feed_strategy_starting_20220926: + # NOTE: Required as we want only want to consider for conversion events that + # occurred on or after the given start_date. + started_at: 2022-09-26 + variants: + - 20220920-variant-b + - 20220926-variant-a + - 20220926-variant-b + - 20220926-variant-c + weights: + - 55 + - 15 + - 15 + - 15 + goals: + - user_creates_pageview + - user_creates_article_reaction + - user_creates_comment + - user_publishes_post + - user_views_pages_on_at_least_two_different_days_within_a_week + - user_views_pages_on_at_least_four_different_days_within_a_week + - user_creates_article_reaction_on_four_different_days_within_a_week + - user_creates_comment_on_at_least_four_different_days_within_a_week + - user_publishes_post_on_four_different_days_within_a_week + - user_views_pages_on_at_least_three_different_hours_within_a_day + - user_views_pages_on_at_least_four_different_hours_within_a_day + - user_views_pages_on_at_least_twelve_different_hours_within_five_days + - user_views_pages_on_at_least_nine_different_days_within_two_weeks + - user_publishes_post_at_least_two_times_within_week + - user_publishes_post_at_least_two_times_within_two_weeks feed_strategy_starting_20220920: # NOTE: Required as we want only want to consider for conversion events that # occurred on or after the given start_date. started_at: 2022-09-20 + ended_at: 2022-09-26 + winner: 20220920-variant-b variants: - 20220915-variant-a - 20220920-variant-a diff --git a/spec/models/ab_experiment/goal_conversion_handler_spec.rb b/spec/models/ab_experiment/goal_conversion_handler_spec.rb index 5f4250359..3a97cfce2 100644 --- a/spec/models/ab_experiment/goal_conversion_handler_spec.rb +++ b/spec/models/ab_experiment/goal_conversion_handler_spec.rb @@ -133,6 +133,17 @@ RSpec.describe AbExperiment::GoalConversionHandler do .to include(goal) end + it "records user_views_pages_on_at_least_two_different_days_within_a_week field test conversion", + :aggregate_failures do + 3.times do |n| + create(:page_view, user_id: user.id, created_at: n.days.ago) + end + handler.call + expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s) + expect(FieldTest::Event.pluck(:name)) + .to include("user_views_pages_on_at_least_two_different_days_within_a_week") + end + it "records user_views_pages_on_at_least_four_different_days_within_a_week field test conversion", :aggregate_failures do 7.times do |n| @@ -176,6 +187,17 @@ RSpec.describe AbExperiment::GoalConversionHandler do .not_to include("user_views_pages_on_at_least_twelve_different_hours_within_five_days") end + it "records user_views_pages_on_at_least_three_different_hours_within_a_day field test conversionn", + :aggregate_failures do + 3.times do |n| + create(:page_view, user_id: user.id, created_at: n.hours.ago) + end + handler.call + expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s) + expect(FieldTest::Event.pluck(:name)) + .to eq([goal, "user_views_pages_on_at_least_three_different_hours_within_a_day"]) + end + it "records user_views_pages_on_at_least_four_different_hours_within_a_day field test conversionn", :aggregate_failures do 7.times do |n| @@ -184,7 +206,8 @@ RSpec.describe AbExperiment::GoalConversionHandler do handler.call expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s) expect(FieldTest::Event.pluck(:name)) - .to eq([goal, "user_views_pages_on_at_least_four_different_hours_within_a_day"]) + .to eq([goal, "user_views_pages_on_at_least_three_different_hours_within_a_day", + "user_views_pages_on_at_least_four_different_hours_within_a_day"]) end it "does not record user_views_article_four_hours_in_day field test conversion for non-qualifying activity" do