maverick/docbrown
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adding feed experiment for 2022-06-03 (#17824)

Browse source 
This commit contains two separate but related changes:

1. A check-list for creating a new experiment.
2. A new experiment along with declaring a winner for the previous experiment.

Below are the diffs of the two new variants versus the incumbent for the
experiment:

```
❯ diff config/feed-variants/20220603-variant-b.json config/feed-variants/20220518-variant.json
3,4c3
<   "description": "As 202205518-variant but with modificiation to `order_by` lever.",
<   "order_by": "final_order_by_random_weighted_to_score",
---
>   "order_by": "relevancy_score_and_publication_date",
```

```
❯ diff config/feed-variants/20220603-variant-a.json config/feed-variants/20220518-variant.json
3d2
<   "description": "As 202205518-variant but with modificiation to `matching_positive_tags_intersection_count`.",
69c68
<         [0, 0.5],
---
>         [0, 0.7],
```

Closes forem/forem#17822
This commit is contained in:
Jeremy Friesen 2022-06-03 13:45:10 -04:00 • committed by GitHub
parent 1679fbf47d
commit 3118f27197
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 245 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
config/feed-variants/20220603-variant-a.json Normal file
View file

@ -0,0 +1,96 @@
{
"max_days_since_published": 15,
"description": "As 202205518-variant but with modificiation to `matching_positive_tags_intersection_count`.",
"order_by": "relevancy_score_and_publication_date",
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.99],
[2, 0.985],
[3, 0.98],
[4, 0.975],
[5, 0.97],
[6, 0.965],
[7, 0.96],
[8, 0.955],
[9, 0.95],
[10, 0.945],
[11, 0.94],
[12, 0.935],
[13, 0.93],
[14, 0.925]
],
"fallback": 0.9
},
"comments_count_by_those_followed": {
"cases": [
[0, 0.95],
[1, 0.98],
[2, 0.99]
],
"fallback": 0.93
},
"comments_count": {
"cases": [
[0, 0.8],
[1, 0.82],
[2, 0.84],
[3, 0.86],
[4, 0.88],
[5, 0.9],
[6, 0.92],
[7, 0.94],
[8, 0.96],
[9, 0.98]
],
"fallback": 1
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.85 },
"latest_comment": {
"cases": [
[0, 1],
[1, 0.9988]
],
"fallback": 0.988
},
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.5],
[2, 0.4],
[3, 0.3],
[4, 0.2]
],
"fallback": 0
},
"matching_positive_tags_intersection_count": {
"cases": [
[0, 0.5],
[1, 0.85],
[2, 0.9],
[3, 0.95],
[4, 1]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -10,
"positive_reaction_threshold": 10
},
"public_reactions": {
"cases": [
[0, 0.9988],
[1, 0.9988],
[2, 0.9988],
[3, 0.9988]
],
"fallback": 1
}
}
}

96
config/feed-variants/20220603-variant-b.json Normal file
View file

@ -0,0 +1,96 @@
{
"max_days_since_published": 15,
"description": "As 202205518-variant but with modificiation to `order_by` lever.",
"order_by": "final_order_by_random_weighted_to_score",
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.99],
[2, 0.985],
[3, 0.98],
[4, 0.975],
[5, 0.97],
[6, 0.965],
[7, 0.96],
[8, 0.955],
[9, 0.95],
[10, 0.945],
[11, 0.94],
[12, 0.935],
[13, 0.93],
[14, 0.925]
],
"fallback": 0.9
},
"comments_count_by_those_followed": {
"cases": [
[0, 0.95],
[1, 0.98],
[2, 0.99]
],
"fallback": 0.93
},
"comments_count": {
"cases": [
[0, 0.8],
[1, 0.82],
[2, 0.84],
[3, 0.86],
[4, 0.88],
[5, 0.9],
[6, 0.92],
[7, 0.94],
[8, 0.96],
[9, 0.98]
],
"fallback": 1
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.85 },
"latest_comment": {
"cases": [
[0, 1],
[1, 0.9988]
],
"fallback": 0.988
},
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.5],
[2, 0.4],
[3, 0.3],
[4, 0.2]
],
"fallback": 0
},
"matching_positive_tags_intersection_count": {
"cases": [
[0, 0.7],
[1, 0.85],
[2, 0.9],
[3, 0.95],
[4, 1]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -10,
"positive_reaction_threshold": 10
},
"public_reactions": {
"cases": [
[0, 0.9988],
[1, 0.9988],
[2, 0.9988],
[3, 0.9988]
],
"fallback": 1
}
}
}

53
config/field_test.yml
View file

@ -1,7 +1,60 @@
# See AbExperiment namespace for details of goals and variants
#
################################################################################
################################################################################
#
# Checklist for creating a new feed experiment:
#
# - [ ] Create and/or reference the forem/forem issue with desired configuration
# - [ ] Identifyer the winner for the past experiment by adding the `winner:
# <variant-name>` attribute to the past experiment.
# - [ ] Copy the previous experiment and prepend and rename to
# `feed_strategy_starting_CCYYMMDD`
# - [ ] Update the `start_date` to `CCYY-MM-DD`
# - [ ] Assign the variants; the winner of the previous experiment SHOULD be in
# this array.
# - [ ] Remove the winner from the newly copied `feed_strategy_starting_CCYYMMDD`
# - [ ] Assign weights and make the corresponding
# `./config/feed-variants/*.json` files where * is the new named variants.
# - [ ] Each newly named variant should have a description; typically briefly
# explaining the difference between it and the winner of the last experiment.
# - [ ] Run `bin/rspec spec/services/articles/feeds/variant_query_spec.rb` to
# verify each newly added variant's SQL
# - [ ] Submit a pull request and ping the Forem Core (in Slack) for a review.
#
################################################################################
################################################################################
experiments:
# NOTE: Our feed strategy testing experiment must begin with "feed_strategy"
feed_strategy_starting_20220603:
# NOTE: Required as we want only want to consider for conversion events that
# occurred on or after the given start_date.
start_date: 2022-06-03
variants:
- 20220518-variant
- 20220603-variant-a
- 20220603-variant-b
weights:
- 50
- 25
- 25
goals:
- user_creates_pageview
- user_creates_article_reaction
- user_creates_comment
- user_publishes_post
- user_views_pages_on_at_least_four_different_days_within_a_week
- user_creates_article_reaction_on_four_different_days_within_a_week
- user_creates_comment_on_at_least_four_different_days_within_a_week
- user_publishes_post_on_four_different_days_within_a_week
# These are historical and may be userful
- user_views_pages_on_at_least_four_different_hours_within_a_day
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
- user_publishes_post_at_least_two_times_within_week
- user_publishes_post_at_least_two_times_within_two_weeks
feed_strategy_starting_20220526:
winner: 20220518-variant
# NOTE: Required as we want only want to consider for conversion events that
# occurred on or after the given start_date.
start_date: 2022-05-09