8fa5c31cb8
This involves declaring the 20220603-variant-a the winner of the past experiment; note in the past experiment there was not a statistically significant winner. This new experiment adds a variant that deviates from the new incumbentby way of reintroducing the follows an author and adjusting the tag follow weights. Closes forem/forem#18082 ```shell $ diff config/feed-variants/20220617-variant-a.json \ config/feed-variants/20220712-variant.json ``` Results in: ```shell 3c3 < "description": "Blend of 20220603-variant-b and 20220603-variant-a", --- > "description": "Builds from 20220617-variant-a (see https://github.com/forem/forem/issues/18082)", 50a51,57 > "following_author": { > "cases": [ > [0, 0.8], > [1, 1] > ], > "fallback": 0.8 > }, 70,73c77,80 < [0, 0.5], < [1, 0.85], < [2, 0.9], < [3, 0.95], --- > [0, 0.3], > [1, 0.9], > [2, 0.95], > [3, 0.98], ```
216 lines
9.2 KiB
YAML
216 lines
9.2 KiB
YAML
# See AbExperiment namespace for details of goals and variants
|
|
#
|
|
################################################################################
|
|
################################################################################
|
|
#
|
|
# Checklist for creating a new feed experiment:
|
|
#
|
|
# - [ ] Create and/or reference the forem/forem issue with desired configuration
|
|
# - [ ] Identify the winner for the past experiment by adding the `winner:
|
|
# <variant-name>` attribute to the past experiment.
|
|
# - [ ] Set the past experiments `ended_at` to the new experiments started_at.
|
|
# - [ ] Copy the previous experiment and prepend and rename to
|
|
# `feed_strategy_starting_CCYYMMDD`
|
|
# - [ ] Update the `started_at` to `CCYY-MM-DD`
|
|
# - [ ] Assign the variants; the winner of the previous experiment SHOULD be in
|
|
# this array.
|
|
# - [ ] Remove the winner attribute from the newly copied
|
|
# `feed_strategy_starting_CCYYMMDD`
|
|
# - [ ] Assign weights and make the corresponding
|
|
# `./config/feed-variants/*.json` files where * is the new named variants.
|
|
# - [ ] Each newly named variant should have a description; typically briefly
|
|
# explaining the difference between it and the winner of the last experiment.
|
|
# - [ ] Run `bin/rspec spec/services/articles/feeds/variant_query_spec.rb` to
|
|
# verify each newly added variant's SQL
|
|
# - [ ] Submit a pull request and ping the Forem Core (in Slack) for a review.
|
|
#
|
|
################################################################################
|
|
################################################################################
|
|
experiments:
|
|
# NOTE: Our feed strategy testing experiment must begin with "feed_strategy"
|
|
feed_strategy_starting_20220712:
|
|
# NOTE: Required as we want only want to consider for conversion events that
|
|
# occurred on or after the given start_date.
|
|
started_at: 2022-07-12
|
|
variants:
|
|
- 20220617-variant-a
|
|
- 20220712-variant
|
|
weights:
|
|
- 50
|
|
- 50
|
|
goals:
|
|
- user_creates_pageview
|
|
- user_creates_article_reaction
|
|
- user_creates_comment
|
|
- user_publishes_post
|
|
- user_views_pages_on_at_least_four_different_days_within_a_week
|
|
- user_creates_article_reaction_on_four_different_days_within_a_week
|
|
- user_creates_comment_on_at_least_four_different_days_within_a_week
|
|
- user_publishes_post_on_four_different_days_within_a_week
|
|
- user_views_pages_on_at_least_four_different_hours_within_a_day
|
|
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
|
|
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
|
|
- user_publishes_post_at_least_two_times_within_week
|
|
- user_publishes_post_at_least_two_times_within_two_weeks
|
|
feed_strategy_starting_20220617:
|
|
# NOTE: Required as we want only want to consider for conversion events that
|
|
# occurred on or after the given start_date.
|
|
started_at: 2022-06-17
|
|
ended_at: 2022-07-12
|
|
winner: 20220617-variant-a
|
|
variants:
|
|
- 20220617-variant-a
|
|
- 20220617-variant-b
|
|
- 20220617-variant-c
|
|
weights:
|
|
- 50
|
|
- 25
|
|
- 25
|
|
goals:
|
|
- user_creates_pageview
|
|
- user_creates_article_reaction
|
|
- user_creates_comment
|
|
- user_publishes_post
|
|
- user_views_pages_on_at_least_four_different_days_within_a_week
|
|
- user_creates_article_reaction_on_four_different_days_within_a_week
|
|
- user_creates_comment_on_at_least_four_different_days_within_a_week
|
|
- user_publishes_post_on_four_different_days_within_a_week
|
|
feed_strategy_starting_20220603:
|
|
# B won with some statistical signficance but A had a strong showing as
|
|
# well. So we're melding those two together.
|
|
winner: 20220603-variant-b
|
|
# NOTE: Required as we want only want to consider for conversion events that
|
|
# occurred on or after the given start_date.
|
|
started_at: 2022-06-03
|
|
ended_at: 2022-06-17
|
|
variants:
|
|
- 20220518-variant
|
|
- 20220603-variant-a
|
|
- 20220603-variant-b
|
|
weights:
|
|
- 50
|
|
- 25
|
|
- 25
|
|
goals:
|
|
- user_creates_pageview
|
|
- user_creates_article_reaction
|
|
- user_creates_comment
|
|
- user_publishes_post
|
|
- user_views_pages_on_at_least_four_different_days_within_a_week
|
|
- user_creates_article_reaction_on_four_different_days_within_a_week
|
|
- user_creates_comment_on_at_least_four_different_days_within_a_week
|
|
- user_publishes_post_on_four_different_days_within_a_week
|
|
# These are historical and may be userful
|
|
- user_views_pages_on_at_least_four_different_hours_within_a_day
|
|
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
|
|
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
|
|
- user_publishes_post_at_least_two_times_within_week
|
|
- user_publishes_post_at_least_two_times_within_two_weeks
|
|
feed_strategy_starting_20220526:
|
|
winner: 20220518-variant
|
|
# NOTE: Required as we want only want to consider for conversion events that
|
|
# occurred on or after the given start_date.
|
|
started_at: 2022-05-26
|
|
ended_at: 2022-06-03
|
|
variants:
|
|
- 20220518-variant
|
|
- 20220525-variant
|
|
weights:
|
|
- 80
|
|
- 20
|
|
goals:
|
|
- user_creates_pageview
|
|
- user_creates_article_reaction
|
|
- user_creates_comment
|
|
- user_publishes_post
|
|
- user_views_pages_on_at_least_four_different_days_within_a_week
|
|
- user_creates_article_reaction_on_four_different_days_within_a_week
|
|
- user_creates_comment_on_at_least_four_different_days_within_a_week
|
|
- user_publishes_post_on_four_different_days_within_a_week
|
|
# These are historical and may be userful
|
|
- user_views_pages_on_at_least_four_different_hours_within_a_day
|
|
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
|
|
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
|
|
- user_publishes_post_at_least_two_times_within_week
|
|
- user_publishes_post_at_least_two_times_within_two_weeks
|
|
feed_strategy_starting_20220518:
|
|
winner: 20220518-variant
|
|
# NOTE: Required as we want only want to consider for conversion events that
|
|
# occurred on or after the given start_date.
|
|
started_at: 2022-05-18
|
|
ended_at: 2022-06-26
|
|
variants:
|
|
- 20220422-variant
|
|
- 20220518-variant
|
|
weights:
|
|
- 80
|
|
- 20
|
|
goals:
|
|
- user_creates_pageview
|
|
- user_creates_article_reaction
|
|
- user_creates_comment
|
|
- user_publishes_post
|
|
- user_views_pages_on_at_least_four_different_days_within_a_week
|
|
- user_creates_article_reaction_on_four_different_days_within_a_week
|
|
- user_creates_comment_on_at_least_four_different_days_within_a_week
|
|
- user_publishes_post_on_four_different_days_within_a_week
|
|
# These are historical and may be userful
|
|
- user_views_pages_on_at_least_four_different_hours_within_a_day
|
|
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
|
|
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
|
|
- user_publishes_post_at_least_two_times_within_week
|
|
feed_strategy_starting_20220509:
|
|
winner: 20220422-variant
|
|
# NOTE: Required as we want only want to consider for conversion events that
|
|
# occurred on or after the given start_date.
|
|
started_at: 2022-05-09
|
|
ended_at: 2022-05-18
|
|
variants:
|
|
- 20220422-variant
|
|
- 20220509-variant
|
|
weights:
|
|
- 80
|
|
- 20
|
|
goals:
|
|
- user_creates_pageview
|
|
- user_creates_article_reaction
|
|
- user_creates_comment
|
|
- user_publishes_post
|
|
- user_views_pages_on_at_least_four_different_days_within_a_week
|
|
- user_creates_article_reaction_on_four_different_days_within_a_week
|
|
- user_creates_comment_on_at_least_four_different_days_within_a_week
|
|
- user_publishes_post_on_four_different_days_within_a_week
|
|
# These are historical and may be userful
|
|
- user_views_pages_on_at_least_four_different_hours_within_a_day
|
|
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
|
|
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
|
|
- user_publishes_post_at_least_two_times_within_week
|
|
feed_strategy_starting_20220422:
|
|
winner: 20220422-jennie-variant
|
|
# NOTE: Required as we want only want to consider for conversion events that
|
|
# occurred on or after the given start_date.
|
|
started_at: 2022-04-22
|
|
ended_at: 2022-05-09
|
|
variants:
|
|
- 20220415-incumbent
|
|
- 20220422-jennie-variant
|
|
weights:
|
|
- 80
|
|
- 20
|
|
goals:
|
|
- user_creates_comment
|
|
- user_creates_comment_on_at_least_four_different_days_within_a_week
|
|
- user_views_pages_on_at_least_four_different_days_within_a_week
|
|
- user_views_pages_on_at_least_four_different_hours_within_a_day
|
|
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
|
|
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
|
|
- user_publishes_post
|
|
- user_publishes_post_at_least_two_times_within_week
|
|
- user_publishes_post_at_least_two_times_within_two_weeks
|
|
# Dashboard performance
|
|
cache: true
|
|
precision: 2
|
|
|
|
# For logged-in users we'll explicitly pass participant.
|
|
# If we experiment on non-logged in, we don't want to collect cookies anyway.
|
|
cookies: false
|