New a/b tests for Feb 14 (#20632)
This commit is contained in:
Ben Halpern
GitHub
parent
8eb41eef8e
commit
097057a8dc
5 changed files with 478 additions and 0 deletions
|
|
@ -111,6 +111,11 @@ module Articles
|
|||
order_by_fragment:
|
||||
"(articles.feed_success_score - articles.clickbait_score) *
|
||||
article_relevancies.randomized_value DESC")
|
||||
order_by_lever(:final_order_by_feed_success_score_minus_half_of_clickbait_score_with_small_randomness,
|
||||
label: "Order by feed success score minus half of clickbait score with a randomization factor",
|
||||
order_by_fragment:
|
||||
"(articles.feed_success_score - (articles.clickbait_score / 2)) +
|
||||
(article_relevancies.randomized_value / 3) DESC")
|
||||
order_by_lever(:final_order_by_feed_success_score_and_primary_score,
|
||||
label: "Order by feed success score and primary score",
|
||||
order_by_fragment: "((articles.feed_success_score + 0.01) * (articles.score / 10)) DESC")
|
||||
|
|
|
|||
146
config/feed-variants/20240214-variant-a.json
Normal file
146
config/feed-variants/20240214-variant-a.json
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
{
|
||||
"max_days_since_published": 15,
|
||||
"description": "Built off 20240205-variant-c but with more subtly random final order",
|
||||
"order_by": "final_order_by_feed_success_score_minus_half_of_clickbait_score_with_small_randomness",
|
||||
"reseed_randomizer_on_each_request": true,
|
||||
"levers": {
|
||||
"daily_decay": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.99],
|
||||
[2, 0.905],
|
||||
[3, 0.7],
|
||||
[4, 0.6],
|
||||
[5, 0.5],
|
||||
[6, 0.3],
|
||||
[7, 0.2],
|
||||
[8, 0.02],
|
||||
[9, 0.01],
|
||||
[10, 0.009],
|
||||
[11, 0.008],
|
||||
[12, 0.007],
|
||||
[13, 0.006],
|
||||
[14, 0.0025]
|
||||
],
|
||||
"fallback": 0.01
|
||||
},
|
||||
"comments_count": {
|
||||
"cases": [
|
||||
[0, 0.15],
|
||||
[1, 0.3],
|
||||
[2, 0.66],
|
||||
[3, 0.7],
|
||||
[4, 0.75],
|
||||
[5, 0.8],
|
||||
[6, 0.85],
|
||||
[7, 0.88],
|
||||
[8, 0.9],
|
||||
[9, 0.92],
|
||||
[12, 1.0],
|
||||
[18, 1.0],
|
||||
[22, 1.0],
|
||||
[25, 1.0],
|
||||
[30, 1.0],
|
||||
[35, 1.0],
|
||||
[40, 1.0],
|
||||
[45, 1.0]
|
||||
],
|
||||
"fallback": 0.99
|
||||
},
|
||||
"following_author": {
|
||||
"cases": [
|
||||
[0, 0.1],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 0.7
|
||||
},
|
||||
"featured_article": { "cases": [[1, 1]], "fallback": 0.15 },
|
||||
"matching_negative_tags_intersection_count": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.2],
|
||||
[2, 0.15],
|
||||
[3, 0.1],
|
||||
[4, 0.05]
|
||||
],
|
||||
"fallback": 0
|
||||
},
|
||||
"experience": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.97],
|
||||
[2, 0.95],
|
||||
[3, 0.88],
|
||||
[4, 0.77],
|
||||
[5, 0.55],
|
||||
[6, 0.35],
|
||||
[7, 0.3]
|
||||
],
|
||||
"default_user_experience_level": 5,
|
||||
"fallback": 0.2
|
||||
},
|
||||
"matching_positive_tags_intersection_count": {
|
||||
"cases": [
|
||||
[0, 0.001],
|
||||
[1, 0.97]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"matching_positive_tags_intersection_points": {
|
||||
"cases": [
|
||||
[0, 0.01],
|
||||
[1, 0.45],
|
||||
[2, 0.5],
|
||||
[3, 0.7],
|
||||
[4, 0.75],
|
||||
[5, 0.8],
|
||||
[6, 0.85],
|
||||
[7, 0.9],
|
||||
[8, 0.93],
|
||||
[9, 0.95]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"privileged_user_reaction": {
|
||||
"cases": [
|
||||
[-1, 0.2],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 0.95,
|
||||
"negative_reaction_threshold": -9,
|
||||
"positive_reaction_threshold": 4
|
||||
},
|
||||
"public_reactions_score": {
|
||||
"cases": [
|
||||
[0, 0.3],
|
||||
[1, 0.4],
|
||||
[2, 0.45],
|
||||
[3, 0.5],
|
||||
[4, 0.55],
|
||||
[5, 0.6],
|
||||
[6, 0.61],
|
||||
[7, 0.62],
|
||||
[8, 0.7],
|
||||
[9, 0.8],
|
||||
[10, 0.85],
|
||||
[11, 0.9],
|
||||
[12, 0.95]
|
||||
],
|
||||
"fallback": 1.0
|
||||
},
|
||||
"language_match": {
|
||||
"cases": [
|
||||
[0, 0.1],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"recommended_articles_match": {
|
||||
"cases": [
|
||||
[0, 0.001],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
146
config/feed-variants/20240214-variant-b.json
Normal file
146
config/feed-variants/20240214-variant-b.json
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
{
|
||||
"max_days_since_published": 15,
|
||||
"description": "Built off 20240205-variant-c but with more subtly random final order and more weight on the author",
|
||||
"order_by": "final_order_by_feed_success_score_minus_half_of_clickbait_score_with_small_randomness",
|
||||
"reseed_randomizer_on_each_request": true,
|
||||
"levers": {
|
||||
"daily_decay": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.99],
|
||||
[2, 0.905],
|
||||
[3, 0.7],
|
||||
[4, 0.6],
|
||||
[5, 0.5],
|
||||
[6, 0.3],
|
||||
[7, 0.2],
|
||||
[8, 0.02],
|
||||
[9, 0.01],
|
||||
[10, 0.009],
|
||||
[11, 0.008],
|
||||
[12, 0.007],
|
||||
[13, 0.006],
|
||||
[14, 0.0025]
|
||||
],
|
||||
"fallback": 0.01
|
||||
},
|
||||
"comments_count": {
|
||||
"cases": [
|
||||
[0, 0.15],
|
||||
[1, 0.3],
|
||||
[2, 0.66],
|
||||
[3, 0.7],
|
||||
[4, 0.75],
|
||||
[5, 0.8],
|
||||
[6, 0.85],
|
||||
[7, 0.88],
|
||||
[8, 0.9],
|
||||
[9, 0.92],
|
||||
[12, 1.0],
|
||||
[18, 1.0],
|
||||
[22, 1.0],
|
||||
[25, 1.0],
|
||||
[30, 1.0],
|
||||
[35, 1.0],
|
||||
[40, 1.0],
|
||||
[45, 1.0]
|
||||
],
|
||||
"fallback": 0.99
|
||||
},
|
||||
"following_author": {
|
||||
"cases": [
|
||||
[0, 0.01],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 0.7
|
||||
},
|
||||
"featured_article": { "cases": [[1, 1]], "fallback": 0.15 },
|
||||
"matching_negative_tags_intersection_count": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.2],
|
||||
[2, 0.15],
|
||||
[3, 0.1],
|
||||
[4, 0.05]
|
||||
],
|
||||
"fallback": 0
|
||||
},
|
||||
"experience": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.97],
|
||||
[2, 0.95],
|
||||
[3, 0.88],
|
||||
[4, 0.77],
|
||||
[5, 0.55],
|
||||
[6, 0.35],
|
||||
[7, 0.3]
|
||||
],
|
||||
"default_user_experience_level": 5,
|
||||
"fallback": 0.2
|
||||
},
|
||||
"matching_positive_tags_intersection_count": {
|
||||
"cases": [
|
||||
[0, 0.001],
|
||||
[1, 0.97]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"matching_positive_tags_intersection_points": {
|
||||
"cases": [
|
||||
[0, 0.01],
|
||||
[1, 0.45],
|
||||
[2, 0.5],
|
||||
[3, 0.7],
|
||||
[4, 0.75],
|
||||
[5, 0.8],
|
||||
[6, 0.85],
|
||||
[7, 0.9],
|
||||
[8, 0.93],
|
||||
[9, 0.95]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"privileged_user_reaction": {
|
||||
"cases": [
|
||||
[-1, 0.2],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 0.95,
|
||||
"negative_reaction_threshold": -9,
|
||||
"positive_reaction_threshold": 4
|
||||
},
|
||||
"public_reactions_score": {
|
||||
"cases": [
|
||||
[0, 0.3],
|
||||
[1, 0.4],
|
||||
[2, 0.45],
|
||||
[3, 0.5],
|
||||
[4, 0.55],
|
||||
[5, 0.6],
|
||||
[6, 0.61],
|
||||
[7, 0.62],
|
||||
[8, 0.7],
|
||||
[9, 0.8],
|
||||
[10, 0.85],
|
||||
[11, 0.9],
|
||||
[12, 0.95]
|
||||
],
|
||||
"fallback": 1.0
|
||||
},
|
||||
"language_match": {
|
||||
"cases": [
|
||||
[0, 0.1],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"recommended_articles_match": {
|
||||
"cases": [
|
||||
[0, 0.001],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
155
config/feed-variants/20240214-variant-c.json
Normal file
155
config/feed-variants/20240214-variant-c.json
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
{
|
||||
"max_days_since_published": 15,
|
||||
"description": "Built off 20240205-variant-c but with more subtly random final order and more weight on score",
|
||||
"order_by": "final_order_by_feed_success_score_minus_half_of_clickbait_score_with_small_randomness",
|
||||
"reseed_randomizer_on_each_request": true,
|
||||
"levers": {
|
||||
"daily_decay": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.99],
|
||||
[2, 0.905],
|
||||
[3, 0.7],
|
||||
[4, 0.6],
|
||||
[5, 0.5],
|
||||
[6, 0.3],
|
||||
[7, 0.2],
|
||||
[8, 0.02],
|
||||
[9, 0.01],
|
||||
[10, 0.009],
|
||||
[11, 0.008],
|
||||
[12, 0.007],
|
||||
[13, 0.006],
|
||||
[14, 0.0025]
|
||||
],
|
||||
"fallback": 0.01
|
||||
},
|
||||
"comments_count": {
|
||||
"cases": [
|
||||
[0, 0.15],
|
||||
[1, 0.3],
|
||||
[2, 0.66],
|
||||
[3, 0.7],
|
||||
[4, 0.75],
|
||||
[5, 0.8],
|
||||
[6, 0.85],
|
||||
[7, 0.88],
|
||||
[8, 0.9],
|
||||
[9, 0.92],
|
||||
[12, 1.0],
|
||||
[18, 1.0],
|
||||
[22, 1.0],
|
||||
[25, 1.0],
|
||||
[30, 1.0],
|
||||
[35, 1.0],
|
||||
[40, 1.0],
|
||||
[45, 1.0]
|
||||
],
|
||||
"fallback": 0.99
|
||||
},
|
||||
"following_author": {
|
||||
"cases": [
|
||||
[0, 0.1],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 0.7
|
||||
},
|
||||
"featured_article": { "cases": [[1, 1]], "fallback": 0.15 },
|
||||
"matching_negative_tags_intersection_count": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.2],
|
||||
[2, 0.15],
|
||||
[3, 0.1],
|
||||
[4, 0.05]
|
||||
],
|
||||
"fallback": 0
|
||||
},
|
||||
"experience": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.97],
|
||||
[2, 0.95],
|
||||
[3, 0.88],
|
||||
[4, 0.77],
|
||||
[5, 0.55],
|
||||
[6, 0.35],
|
||||
[7, 0.3]
|
||||
],
|
||||
"default_user_experience_level": 5,
|
||||
"fallback": 0.2
|
||||
},
|
||||
"matching_positive_tags_intersection_count": {
|
||||
"cases": [
|
||||
[0, 0.001],
|
||||
[1, 0.97]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"matching_positive_tags_intersection_points": {
|
||||
"cases": [
|
||||
[0, 0.01],
|
||||
[1, 0.45],
|
||||
[2, 0.5],
|
||||
[3, 0.7],
|
||||
[4, 0.75],
|
||||
[5, 0.8],
|
||||
[6, 0.85],
|
||||
[7, 0.9],
|
||||
[8, 0.93],
|
||||
[9, 0.95]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"privileged_user_reaction": {
|
||||
"cases": [
|
||||
[-1, 0.2],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 0.95,
|
||||
"negative_reaction_threshold": -9,
|
||||
"positive_reaction_threshold": 4
|
||||
},
|
||||
"public_reactions_score": {
|
||||
"cases": [
|
||||
[0, 0.01],
|
||||
[1, 0.15],
|
||||
[2, 0.2],
|
||||
[3, 0.25],
|
||||
[4, 0.3],
|
||||
[5, 0.35],
|
||||
[6, 0.4],
|
||||
[7, 0.45],
|
||||
[8, 0.5],
|
||||
[9, 0.55],
|
||||
[10, 0.6],
|
||||
[11, 0.65],
|
||||
[12, 0.7],
|
||||
[13, 0.75],
|
||||
[14, 0.8],
|
||||
[15, 0.85],
|
||||
[16, 0.9],
|
||||
[17, 0.95],
|
||||
[18, 0.96],
|
||||
[19, 0.97],
|
||||
[20, 0.98],
|
||||
[21, 0.99]
|
||||
],
|
||||
"fallback": 1.0
|
||||
},
|
||||
"language_match": {
|
||||
"cases": [
|
||||
[0, 0.1],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"recommended_articles_match": {
|
||||
"cases": [
|
||||
[0, 0.001],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -55,6 +55,30 @@ experiments:
|
|||
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
|
||||
|
||||
# NOTE: Our feed strategy testing experiment must begin with "feed_strategy"
|
||||
feed_strategy_starting_20240214:
|
||||
started_at: 2024-02-14
|
||||
variants:
|
||||
- 20240205-variant-c
|
||||
- 20240214-variant-a
|
||||
- 20240214-variant-b
|
||||
- 20240214-variant-c
|
||||
weights:
|
||||
- 67
|
||||
- 11
|
||||
- 11
|
||||
- 11
|
||||
goals:
|
||||
- user_creates_pageview
|
||||
- user_creates_article_reaction
|
||||
- user_creates_comment
|
||||
- user_views_pages_on_at_least_two_different_days_within_a_week
|
||||
- user_views_pages_on_at_least_four_different_days_within_a_week
|
||||
- user_creates_article_reaction_on_four_different_days_within_a_week
|
||||
- user_creates_comment_on_at_least_four_different_days_within_a_week
|
||||
- user_views_pages_on_at_least_three_different_hours_within_a_day
|
||||
- user_views_pages_on_at_least_four_different_hours_within_a_day
|
||||
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
|
||||
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
|
||||
feed_strategy_starting_20240205:
|
||||
started_at: 2024-02-05
|
||||
variants:
|
||||
|
|
@ -62,6 +86,8 @@ experiments:
|
|||
- 20240205-variant-a
|
||||
- 20240205-variant-b
|
||||
- 20240205-variant-c
|
||||
ended_at: 2024-02-14
|
||||
winner: 20240205-variant-c
|
||||
weights:
|
||||
- 67
|
||||
- 11
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue