Starting 2022-07-12 Experiment (#18083)

This involves declaring the 20220603-variant-a the winner of the past
experiment; note in the past experiment there was not a statistically
significant winner.

This new experiment adds a variant that deviates from the new
incumbentby way of reintroducing the follows an author and adjusting the
tag follow weights.

Closes forem/forem#18082

```shell
$ diff config/feed-variants/20220617-variant-a.json \
  config/feed-variants/20220712-variant.json
```

Results in:

```shell
3c3
<   "description": "Blend of 20220603-variant-b and 20220603-variant-a",
---
>   "description": "Builds from 20220617-variant-a (see https://github.com/forem/forem/issues/18082)",
50a51,57
>     "following_author": {
>       "cases": [
>         [0, 0.8],
>         [1, 1]
>       ],
>       "fallback": 0.8
>     },
70,73c77,80
<         [0, 0.5],
<         [1, 0.85],
<         [2, 0.9],
<         [3, 0.95],
---
>         [0, 0.3],
>         [1, 0.9],
>         [2, 0.95],
>         [3, 0.98],
```
This commit is contained in:
Jeremy Friesen 2022-07-12 14:51:47 -04:00 committed by GitHub
parent 62e8512f37
commit 8fa5c31cb8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 130 additions and 0 deletions

View file

@ -0,0 +1,104 @@
{
"max_days_since_published": 15,
"description": "Builds from 20220617-variant-a (see https://github.com/forem/forem/issues/18082)",
"order_by": "final_order_by_random_weighted_to_score",
"reseed_randomizer_on_each_request": false,
"levers": {
"daily_decay": {
"cases": [
[0, 1],
[1, 0.99],
[2, 0.985],
[3, 0.98],
[4, 0.975],
[5, 0.97],
[6, 0.965],
[7, 0.96],
[8, 0.955],
[9, 0.95],
[10, 0.945],
[11, 0.94],
[12, 0.935],
[13, 0.93],
[14, 0.925]
],
"fallback": 0.9
},
"comments_count_by_those_followed": {
"cases": [
[0, 0.95],
[1, 0.98],
[2, 0.99]
],
"fallback": 0.93
},
"comments_count": {
"cases": [
[0, 0.8],
[1, 0.82],
[2, 0.84],
[3, 0.86],
[4, 0.88],
[5, 0.9],
[6, 0.92],
[7, 0.94],
[8, 0.96],
[9, 0.98]
],
"fallback": 1
},
"featured_article": { "cases": [[1, 1]], "fallback": 0.85 },
"following_author": {
"cases": [
[0, 0.8],
[1, 1]
],
"fallback": 0.8
},
"latest_comment": {
"cases": [
[0, 1],
[1, 0.9988]
],
"fallback": 0.988
},
"matching_negative_tags_intersection_count": {
"cases": [
[0, 1],
[1, 0.5],
[2, 0.4],
[3, 0.3],
[4, 0.2]
],
"fallback": 0
},
"matching_positive_tags_intersection_count": {
"cases": [
[0, 0.3],
[1, 0.9],
[2, 0.95],
[3, 0.98],
[4, 1]
],
"fallback": 1
},
"privileged_user_reaction": {
"cases": [
[-1, 0.2],
[1, 1]
],
"fallback": 0.95,
"negative_reaction_threshold": -10,
"positive_reaction_threshold": 10
},
"public_reactions": {
"cases": [
[0, 0.9988],
[1, 0.9988],
[2, 0.9988],
[3, 0.9988]
],
"fallback": 1
}
}
}

View file

@ -28,10 +28,36 @@
################################################################################
experiments:
# NOTE: Our feed strategy testing experiment must begin with "feed_strategy"
feed_strategy_starting_20220712:
# NOTE: Required as we want only want to consider for conversion events that
# occurred on or after the given start_date.
started_at: 2022-07-12
variants:
- 20220617-variant-a
- 20220712-variant
weights:
- 50
- 50
goals:
- user_creates_pageview
- user_creates_article_reaction
- user_creates_comment
- user_publishes_post
- user_views_pages_on_at_least_four_different_days_within_a_week
- user_creates_article_reaction_on_four_different_days_within_a_week
- user_creates_comment_on_at_least_four_different_days_within_a_week
- user_publishes_post_on_four_different_days_within_a_week
- user_views_pages_on_at_least_four_different_hours_within_a_day
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
- user_publishes_post_at_least_two_times_within_week
- user_publishes_post_at_least_two_times_within_two_weeks
feed_strategy_starting_20220617:
# NOTE: Required as we want only want to consider for conversion events that
# occurred on or after the given start_date.
started_at: 2022-06-17
ended_at: 2022-07-12
winner: 20220617-variant-a
variants:
- 20220617-variant-a
- 20220617-variant-b