maverick/docbrown
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feed experiment for 09-26-2022 (#18501)

Browse source 
* Feed experiment for 09-26-2022

* Fix variant

* Add proper tests
This commit is contained in:
Ben Halpern 2022-09-27 10:48:15 -04:00 • committed by GitHub
parent 3972147204
commit cd5c86c760
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 519 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
app/models/ab_experiment/goal_conversion_handler.rb
View file

@ -66,11 +66,21 @@ class AbExperiment
if FeatureFlag.accessible?(:field_test_event_single_create_pageview)
field_test_converted(experiment, participant: user, goal: goal) # base is someone viewed a page
end
pageview_goal(experiment,
[7.days.ago, experiment_start_date].max,
"DATE(created_at)",
2,
"user_views_pages_on_at_least_two_different_days_within_a_week")
pageview_goal(experiment,
[7.days.ago, experiment_start_date].max,
"DATE(created_at)",
4,
"user_views_pages_on_at_least_four_different_days_within_a_week")
pageview_goal(experiment,
[24.hours.ago, experiment_start_date].max,
"DATE_PART('hour', created_at)",
3,
"user_views_pages_on_at_least_three_different_hours_within_a_day")
pageview_goal(experiment,
[24.hours.ago, experiment_start_date].max,
"DATE_PART('hour', created_at)",

151
config/feed-variants/20220926-variant-a.json Normal file
View file

@ -0,0 +1,151 @@
{
"max_days_since_published": 15,
"description": "Copy of 20220920-variant-b with a sharper daily decay step after day 0",
"order_by": "final_order_by_random_weighted_to_score",
"reseed_randomizer_on_each_request": false,
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.91],
[2, 0.905],
[3, 0.9],
[4, 0.89],
[5, 0.88],
[6, 0.87],
[7, 0.86],
[8, 0.85],
[9, 0.84],
[10, 0.83],
[11, 0.82],
[12, 0.81],
[13, 0.8],
[14, 0.79]
],
"fallback": 0.7
},
"comments_count_by_those_followed": {
"cases": [
[0, 0.95],
[1, 0.98],
[2, 0.99]
],
"fallback": 0.93
},
"comments_count": {
"cases": [
[0, 0.5],
[1, 0.6],
[2, 0.66],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.88],
[8, 0.9],
[9, 0.92],
[12, 1.0],
[18, 1.0],
[22, 1.0],
[25, 1.0],
[30, 1.0],
[35, 1.0],
[40, 1.0],
[45, 1.0]
],
"fallback": 0.98
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.85 },
"latest_comment": {
"cases": [
[0, 1],
[1, 0.9988]
],
"fallback": 0.988
},
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.5],
[2, 0.4],
[3, 0.3],
[4, 0.2]
],
"fallback": 0
},
"matching_positive_tags_intersection_points": {
"cases": [
[0, 0.55],
[1, 0.6],
[2, 0.65],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.9],
[8, 0.93],
[9, 0.95]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -9,
"positive_reaction_threshold": 4
},
"public_reactions_score": {
"cases": [
[0, 0.55],
[1, 0.56],
[2, 0.57],
[3, 0.58],
[4, 0.59],
[5, 0.6],
[6, 0.61],
[7, 0.62],
[8, 0.63],
[9, 0.64],
[10, 0.65],
[11, 0.66],
[12, 0.67],
[13, 0.68],
[14, 0.69],
[15, 0.7],
[16, 0.71],
[17, 0.72],
[18, 0.73],
[19, 0.74],
[20, 0.75],
[21, 0.76],
[22, 0.77],
[23, 0.78],
[24, 0.79],
[25, 0.8],
[26, 0.81],
[27, 0.82],
[28, 0.83],
[29, 0.84],
[30, 0.85],
[31, 0.86],
[32, 0.87],
[33, 0.88],
[34, 0.89],
[35, 0.9],
[36, 0.91],
[37, 0.92],
[38, 0.93],
[39, 0.94],
[40, 0.95],
[41, 0.96],
[42, 0.97],
[43, 0.98],
[44, 0.99]
],
"fallback": 1
}
}
}

151
config/feed-variants/20220926-variant-b.json Normal file
View file

@ -0,0 +1,151 @@
{
"max_days_since_published": 15,
"description": "Copy of 20220920-variant-b with a sharper daily decay step after day 1",
"order_by": "final_order_by_random_weighted_to_score",
"reseed_randomizer_on_each_request": false,
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.99],
[2, 0.905],
[3, 0.9],
[4, 0.89],
[5, 0.88],
[6, 0.87],
[7, 0.86],
[8, 0.85],
[9, 0.84],
[10, 0.83],
[11, 0.82],
[12, 0.81],
[13, 0.8],
[14, 0.79]
],
"fallback": 0.7
},
"comments_count_by_those_followed": {
"cases": [
[0, 0.95],
[1, 0.98],
[2, 0.99]
],
"fallback": 0.93
},
"comments_count": {
"cases": [
[0, 0.5],
[1, 0.6],
[2, 0.66],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.88],
[8, 0.9],
[9, 0.92],
[12, 1.0],
[18, 1.0],
[22, 1.0],
[25, 1.0],
[30, 1.0],
[35, 1.0],
[40, 1.0],
[45, 1.0]
],
"fallback": 0.98
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.85 },
"latest_comment": {
"cases": [
[0, 1],
[1, 0.9988]
],
"fallback": 0.988
},
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.5],
[2, 0.4],
[3, 0.3],
[4, 0.2]
],
"fallback": 0
},
"matching_positive_tags_intersection_points": {
"cases": [
[0, 0.55],
[1, 0.6],
[2, 0.65],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.9],
[8, 0.93],
[9, 0.95]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -9,
"positive_reaction_threshold": 4
},
"public_reactions_score": {
"cases": [
[0, 0.55],
[1, 0.56],
[2, 0.57],
[3, 0.58],
[4, 0.59],
[5, 0.6],
[6, 0.61],
[7, 0.62],
[8, 0.63],
[9, 0.64],
[10, 0.65],
[11, 0.66],
[12, 0.67],
[13, 0.68],
[14, 0.69],
[15, 0.7],
[16, 0.71],
[17, 0.72],
[18, 0.73],
[19, 0.74],
[20, 0.75],
[21, 0.76],
[22, 0.77],
[23, 0.78],
[24, 0.79],
[25, 0.8],
[26, 0.81],
[27, 0.82],
[28, 0.83],
[29, 0.84],
[30, 0.85],
[31, 0.86],
[32, 0.87],
[33, 0.88],
[34, 0.89],
[35, 0.9],
[36, 0.91],
[37, 0.92],
[38, 0.93],
[39, 0.94],
[40, 0.95],
[41, 0.96],
[42, 0.97],
[43, 0.98],
[44, 0.99]
],
"fallback": 1
}
}
}

151
config/feed-variants/20220926-variant-c.json Normal file
View file

@ -0,0 +1,151 @@
{
"max_days_since_published": 15,
"description": "Copy of 20220920-variant-b with a sharper daily decay step after day 2",
"order_by": "final_order_by_random_weighted_to_score",
"reseed_randomizer_on_each_request": false,
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.99],
[2, 0.98],
[3, 0.9],
[4, 0.89],
[5, 0.88],
[6, 0.87],
[7, 0.86],
[8, 0.85],
[9, 0.84],
[10, 0.83],
[11, 0.82],
[12, 0.81],
[13, 0.8],
[14, 0.79]
],
"fallback": 0.7
},
"comments_count_by_those_followed": {
"cases": [
[0, 0.95],
[1, 0.98],
[2, 0.99]
],
"fallback": 0.93
},
"comments_count": {
"cases": [
[0, 0.5],
[1, 0.6],
[2, 0.66],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.88],
[8, 0.9],
[9, 0.92],
[12, 1.0],
[18, 1.0],
[22, 1.0],
[25, 1.0],
[30, 1.0],
[35, 1.0],
[40, 1.0],
[45, 1.0]
],
"fallback": 0.98
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.85 },
"latest_comment": {
"cases": [
[0, 1],
[1, 0.9988]
],
"fallback": 0.988
},
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.5],
[2, 0.4],
[3, 0.3],
[4, 0.2]
],
"fallback": 0
},
"matching_positive_tags_intersection_points": {
"cases": [
[0, 0.55],
[1, 0.6],
[2, 0.65],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.9],
[8, 0.93],
[9, 0.95]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -9,
"positive_reaction_threshold": 4
},
"public_reactions_score": {
"cases": [
[0, 0.55],
[1, 0.56],
[2, 0.57],
[3, 0.58],
[4, 0.59],
[5, 0.6],
[6, 0.61],
[7, 0.62],
[8, 0.63],
[9, 0.64],
[10, 0.65],
[11, 0.66],
[12, 0.67],
[13, 0.68],
[14, 0.69],
[15, 0.7],
[16, 0.71],
[17, 0.72],
[18, 0.73],
[19, 0.74],
[20, 0.75],
[21, 0.76],
[22, 0.77],
[23, 0.78],
[24, 0.79],
[25, 0.8],
[26, 0.81],
[27, 0.82],
[28, 0.83],
[29, 0.84],
[30, 0.85],
[31, 0.86],
[32, 0.87],
[33, 0.88],
[34, 0.89],
[35, 0.9],
[36, 0.91],
[37, 0.92],
[38, 0.93],
[39, 0.94],
[40, 0.95],
[41, 0.96],
[42, 0.97],
[43, 0.98],
[44, 0.99]
],
"fallback": 1
}
}
}

32
config/field_test.yml
View file

@ -28,10 +28,42 @@
################################################################################
experiments:
# NOTE: Our feed strategy testing experiment must begin with "feed_strategy"
feed_strategy_starting_20220926:
# NOTE: Required as we want only want to consider for conversion events that
# occurred on or after the given start_date.
started_at: 2022-09-26
variants:
- 20220920-variant-b
- 20220926-variant-a
- 20220926-variant-b
- 20220926-variant-c
weights:
- 55
- 15
- 15
- 15
goals:
- user_creates_pageview
- user_creates_article_reaction
- user_creates_comment
- user_publishes_post
- user_views_pages_on_at_least_two_different_days_within_a_week
- user_views_pages_on_at_least_four_different_days_within_a_week
- user_creates_article_reaction_on_four_different_days_within_a_week
- user_creates_comment_on_at_least_four_different_days_within_a_week
- user_publishes_post_on_four_different_days_within_a_week
- user_views_pages_on_at_least_three_different_hours_within_a_day
- user_views_pages_on_at_least_four_different_hours_within_a_day
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
- user_publishes_post_at_least_two_times_within_week
- user_publishes_post_at_least_two_times_within_two_weeks
feed_strategy_starting_20220920:
# NOTE: Required as we want only want to consider for conversion events that
# occurred on or after the given start_date.
started_at: 2022-09-20
ended_at: 2022-09-26
winner: 20220920-variant-b
variants:
- 20220915-variant-a
- 20220920-variant-a

25
spec/models/ab_experiment/goal_conversion_handler_spec.rb
View file

@ -133,6 +133,17 @@ RSpec.describe AbExperiment::GoalConversionHandler do
.to include(goal)
end
it "records user_views_pages_on_at_least_two_different_days_within_a_week field test conversion",
:aggregate_failures do
3.times do |n|
create(:page_view, user_id: user.id, created_at: n.days.ago)
end
handler.call
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.pluck(:name))
.to include("user_views_pages_on_at_least_two_different_days_within_a_week")
end
it "records user_views_pages_on_at_least_four_different_days_within_a_week field test conversion",
:aggregate_failures do
7.times do |n|
@ -176,6 +187,17 @@ RSpec.describe AbExperiment::GoalConversionHandler do
.not_to include("user_views_pages_on_at_least_twelve_different_hours_within_five_days")
end
it "records user_views_pages_on_at_least_three_different_hours_within_a_day field test conversionn",
:aggregate_failures do
3.times do |n|
create(:page_view, user_id: user.id, created_at: n.hours.ago)
end
handler.call
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.pluck(:name))
.to eq([goal, "user_views_pages_on_at_least_three_different_hours_within_a_day"])
end
it "records user_views_pages_on_at_least_four_different_hours_within_a_day field test conversionn",
:aggregate_failures do
7.times do |n|
@ -184,7 +206,8 @@ RSpec.describe AbExperiment::GoalConversionHandler do
handler.call
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.pluck(:name))
.to eq([goal, "user_views_pages_on_at_least_four_different_hours_within_a_day"])
.to eq([goal, "user_views_pages_on_at_least_three_different_hours_within_a_day",
"user_views_pages_on_at_least_four_different_hours_within_a_day"])
end
it "does not record user_views_article_four_hours_in_day field test conversion for non-qualifying activity" do