Feed experiment for 11-07-2022 (#18702)

This commit is contained in:
Ben Halpern 2022-11-08 13:06:58 -05:00 committed by GitHub
parent 05b01461d9
commit 43c522b58a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 589 additions and 0 deletions

View file

@ -0,0 +1,135 @@
{
"max_days_since_published": 15,
"description": "Built off 20221031-variant-c but with even softer cap on score.",
"order_by": "final_order_by_random_weighted_to_score",
"reseed_randomizer_on_each_request": false,
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.99],
[2, 0.905],
[3, 0.9],
[4, 0.89],
[5, 0.85],
[6, 0.84],
[7, 0.8],
[8, 0.69],
[9, 0.68],
[10, 0.67],
[11, 0.66],
[12, 0.65],
[13, 0.6],
[14, 0.55]
],
"fallback": 0.25
},
"comments_count": {
"cases": [
[0, 0.15],
[1, 0.3],
[2, 0.66],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.88],
[8, 0.9],
[9, 0.92],
[12, 1.0],
[18, 1.0],
[22, 1.0],
[25, 1.0],
[30, 1.0],
[35, 1.0],
[40, 1.0],
[45, 1.0]
],
"fallback": 0.99
},
"following_author": {
"cases": [
[0, 0.7],
[1, 1]
],
"fallback": 0.7
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.78 },
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.2],
[2, 0.15],
[3, 0.1],
[4, 0.05]
],
"fallback": 0
},
"experience": {
"cases": [
[0, 1],
[1, 0.98],
[2, 0.96],
[3, 0.95],
[4, 0.93],
[5, 0.88],
[6, 0.5],
[7, 0.4]
],
"default_user_experience_level": 5,
"fallback": 0.3
},
"matching_positive_tags_intersection_count": {
"cases": [
[0, 0.25],
[1, 0.97]
],
"fallback": 1
},
"matching_positive_tags_intersection_points": {
"cases": [
[0, 0.3],
[1, 0.45],
[2, 0.5],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.9],
[8, 0.93],
[9, 0.95]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -9,
"positive_reaction_threshold": 4
},
"public_reactions_score": {
"cases": [
[0, 0.25],
[1, 0.4],
[2, 0.45],
[3, 0.5],
[4, 0.55],
[5, 0.6],
[6, 0.61],
[7, 0.62],
[8, 0.64],
[9, 0.68],
[10, 0.7],
[11, 0.75],
[12, 0.8],
[13, 0.85],
[14, 0.9],
[15, 0.95]
],
"fallback": 1.0
}
}
}

View file

@ -0,0 +1,140 @@
{
"max_days_since_published": 15,
"description": "Built off 20221031-variant-c but with steeper daily decay after 4 days.",
"order_by": "final_order_by_random_weighted_to_score",
"reseed_randomizer_on_each_request": false,
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.99],
[2, 0.905],
[3, 0.9],
[4, 0.89],
[5, 0.4],
[6, 0.3],
[7, 0.2],
[8, 0.15],
[9, 0.1],
[10, 0.09],
[11, 0.08],
[12, 0.07],
[13, 0.06],
[14, 0.05]
],
"fallback": 0.01
},
"comments_count": {
"cases": [
[0, 0.15],
[1, 0.3],
[2, 0.66],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.88],
[8, 0.9],
[9, 0.92],
[12, 1.0],
[18, 1.0],
[22, 1.0],
[25, 1.0],
[30, 1.0],
[35, 1.0],
[40, 1.0],
[45, 1.0]
],
"fallback": 0.99
},
"following_author": {
"cases": [
[0, 0.7],
[1, 1]
],
"fallback": 0.7
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.78 },
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.2],
[2, 0.15],
[3, 0.1],
[4, 0.05]
],
"fallback": 0
},
"experience": {
"cases": [
[0, 1],
[1, 0.98],
[2, 0.96],
[3, 0.95],
[4, 0.93],
[5, 0.88],
[6, 0.5],
[7, 0.4]
],
"default_user_experience_level": 5,
"fallback": 0.3
},
"matching_positive_tags_intersection_count": {
"cases": [
[0, 0.25],
[1, 0.97]
],
"fallback": 1
},
"matching_positive_tags_intersection_points": {
"cases": [
[0, 0.3],
[1, 0.45],
[2, 0.5],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.9],
[8, 0.93],
[9, 0.95]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -9,
"positive_reaction_threshold": 4
},
"public_reactions_score": {
"cases": [
[0, 0.3],
[1, 0.4],
[2, 0.45],
[3, 0.5],
[4, 0.55],
[5, 0.6],
[6, 0.61],
[7, 0.62],
[8, 0.63],
[9, 0.64],
[10, 0.65],
[11, 0.69],
[12, 0.72],
[13, 0.75],
[14, 0.79],
[15, 0.82],
[16, 0.85],
[17, 0.88],
[18, 0.9],
[19, 0.93],
[20, 0.95]
],
"fallback": 1.0
}
}
}

View file

@ -0,0 +1,140 @@
{
"max_days_since_published": 15,
"description": "Built off 20221031-variant-c but with steeper daily decay after 7 days.",
"order_by": "final_order_by_random_weighted_to_score",
"reseed_randomizer_on_each_request": false,
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.99],
[2, 0.905],
[3, 0.9],
[4, 0.89],
[5, 0.85],
[6, 0.82],
[7, 0.79],
[8, 0.15],
[9, 0.1],
[10, 0.09],
[11, 0.08],
[12, 0.07],
[13, 0.06],
[14, 0.05]
],
"fallback": 0.25
},
"comments_count": {
"cases": [
[0, 0.15],
[1, 0.3],
[2, 0.66],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.88],
[8, 0.9],
[9, 0.92],
[12, 1.0],
[18, 1.0],
[22, 1.0],
[25, 1.0],
[30, 1.0],
[35, 1.0],
[40, 1.0],
[45, 1.0]
],
"fallback": 0.99
},
"following_author": {
"cases": [
[0, 0.7],
[1, 1]
],
"fallback": 0.7
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.78 },
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.2],
[2, 0.15],
[3, 0.1],
[4, 0.05]
],
"fallback": 0
},
"experience": {
"cases": [
[0, 1],
[1, 0.98],
[2, 0.96],
[3, 0.95],
[4, 0.93],
[5, 0.88],
[6, 0.5],
[7, 0.4]
],
"default_user_experience_level": 5,
"fallback": 0.3
},
"matching_positive_tags_intersection_count": {
"cases": [
[0, 0.25],
[1, 0.97]
],
"fallback": 1
},
"matching_positive_tags_intersection_points": {
"cases": [
[0, 0.3],
[1, 0.45],
[2, 0.5],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.9],
[8, 0.93],
[9, 0.95]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -9,
"positive_reaction_threshold": 4
},
"public_reactions_score": {
"cases": [
[0, 0.3],
[1, 0.4],
[2, 0.45],
[3, 0.5],
[4, 0.55],
[5, 0.6],
[6, 0.61],
[7, 0.62],
[8, 0.63],
[9, 0.64],
[10, 0.65],
[11, 0.69],
[12, 0.72],
[13, 0.75],
[14, 0.79],
[15, 0.82],
[16, 0.85],
[17, 0.88],
[18, 0.9],
[19, 0.93],
[20, 0.95]
],
"fallback": 1.0
}
}
}

View file

@ -0,0 +1,140 @@
{
"max_days_since_published": 15,
"description": "Built off 20221031-variant-c but with steeper daily decay after 2 days.",
"order_by": "final_order_by_random_weighted_to_score",
"reseed_randomizer_on_each_request": false,
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.99],
[2, 0.905],
[3, 0.5],
[4, 0.38],
[5, 0.3],
[6, 0.25],
[7, 0.2],
[8, 0.15],
[9, 0.1],
[10, 0.09],
[11, 0.08],
[12, 0.07],
[13, 0.06],
[14, 0.05]
],
"fallback": 0.01
},
"comments_count": {
"cases": [
[0, 0.15],
[1, 0.3],
[2, 0.66],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.88],
[8, 0.9],
[9, 0.92],
[12, 1.0],
[18, 1.0],
[22, 1.0],
[25, 1.0],
[30, 1.0],
[35, 1.0],
[40, 1.0],
[45, 1.0]
],
"fallback": 0.99
},
"following_author": {
"cases": [
[0, 0.7],
[1, 1]
],
"fallback": 0.7
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.78 },
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.2],
[2, 0.15],
[3, 0.1],
[4, 0.05]
],
"fallback": 0
},
"experience": {
"cases": [
[0, 1],
[1, 0.98],
[2, 0.96],
[3, 0.95],
[4, 0.93],
[5, 0.88],
[6, 0.5],
[7, 0.4]
],
"default_user_experience_level": 5,
"fallback": 0.3
},
"matching_positive_tags_intersection_count": {
"cases": [
[0, 0.25],
[1, 0.97]
],
"fallback": 1
},
"matching_positive_tags_intersection_points": {
"cases": [
[0, 0.3],
[1, 0.45],
[2, 0.5],
[3, 0.7],
[4, 0.75],
[5, 0.8],
[6, 0.85],
[7, 0.9],
[8, 0.93],
[9, 0.95]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -9,
"positive_reaction_threshold": 4
},
"public_reactions_score": {
"cases": [
[0, 0.3],
[1, 0.4],
[2, 0.45],
[3, 0.5],
[4, 0.55],
[5, 0.6],
[6, 0.61],
[7, 0.62],
[8, 0.63],
[9, 0.64],
[10, 0.65],
[11, 0.69],
[12, 0.72],
[13, 0.75],
[14, 0.79],
[15, 0.82],
[16, 0.85],
[17, 0.88],
[18, 0.9],
[19, 0.93],
[20, 0.95]
],
"fallback": 1.0
}
}
}

View file

@ -28,10 +28,44 @@
################################################################################ ################################################################################
experiments: experiments:
# NOTE: Our feed strategy testing experiment must begin with "feed_strategy" # NOTE: Our feed strategy testing experiment must begin with "feed_strategy"
feed_strategy_starting_20221107:
# NOTE: Required as we want only want to consider for conversion events that
# occurred on or after the given start_date.
started_at: 2022-11-07
variants:
- 20221031-variant-c
- 20221107-variant-a
- 20221107-variant-b
- 20221107-variant-c
- 20221107-variant-d
weights:
- 52
- 12
- 12
- 12
- 12
goals:
- user_creates_pageview
- user_creates_article_reaction
- user_creates_comment
- user_publishes_post
- user_views_pages_on_at_least_two_different_days_within_a_week
- user_views_pages_on_at_least_four_different_days_within_a_week
- user_creates_article_reaction_on_four_different_days_within_a_week
- user_creates_comment_on_at_least_four_different_days_within_a_week
- user_publishes_post_on_four_different_days_within_a_week
- user_views_pages_on_at_least_three_different_hours_within_a_day
- user_views_pages_on_at_least_four_different_hours_within_a_day
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
- user_publishes_post_at_least_two_times_within_week
- user_publishes_post_at_least_two_times_within_two_weeks
feed_strategy_starting_20221031: feed_strategy_starting_20221031:
# NOTE: Required as we want only want to consider for conversion events that # NOTE: Required as we want only want to consider for conversion events that
# occurred on or after the given start_date. # occurred on or after the given start_date.
started_at: 2022-10-31 started_at: 2022-10-31
ended_at: 2022-11-07
winner: 20221031-variant-c
variants: variants:
- 20221025-variant-b - 20221025-variant-b
- 20221031-variant-a - 20221031-variant-a