Adding new relevancy lever for privileged reactions (#17598)
* Adding new relevancy lever for privileged reactions
This PR includes four changes:
1. Renaming the variant to remove Jennie's name
2. Starting a new experiment
3. Adding a new relevancy lever
4. Adding a new variant that uses the relevancy lever
1 and 2 is somewhat straight-forward.
For 3, we're looking at having a total of 5 "slots" for privileged user
reactions. Hence the four boundary values.
For 4, we're configuring the granular levers with the following:
- `[-∞..-5)` should have a 0.05
- `[-5..0)` should have a 0.5
- `[0..5)` should have a 0.9
- `[5..10)` should have a 0.98
- `[10..∞)` should have a 1.0
The 20220509-variant.json is a copy of 20220422-variant.json but
replaces the `privileged_user_reaction` lever with the
`privileged_user_reaction_granular` lever.
Below is the diff to highlight the similarities and differences between
the two variants.
```shell
❯ diff config/feed-variants/20220509-variant.json config/feed-variants/20220422-variant.json
90c90
< "privileged_user_reaction_granular": {
---
> "privileged_user_reaction": {
92,96c92,93
< [-2, 0.05],
< [-1, 0.5],
< [0, 0.9],
< [1, 0.98],
< [2, 1]
---
> [-1, 0.2],
> [1, 1]
98,102c95,97
< "fallback": 0.9,
< "very_negative_reaction_threshold": -5,
< "negative_reaction_threshold": 0,
< "positive_reaction_threshold": 5,
< "very_positive_reaction_threshold": 10
---
> "fallback": 0.95,
> "negative_reaction_threshold": -10,
> "positive_reaction_threshold": 10
```
Closes forem/forem#17584
* Update app/models/articles/feeds.rb
Co-authored-by: Josh Puetz <josh@dev.to>
* Adjusting testing logic to reflect latest experiment
* Adjusting negative threshold
Co-authored-by: Josh Puetz <josh@dev.to>
This commit is contained in:
Jeremy Friesen
GitHub
parent
27373c6cf8
commit
27b82996ce
6 changed files with 153 additions and 9 deletions
|
|
@ -17,10 +17,11 @@ class AbExperiment
|
|||
new(...).call
|
||||
end
|
||||
|
||||
def initialize(user:, goal:, experiments:)
|
||||
def initialize(user:, goal:, experiments:, start_date: nil)
|
||||
@user = user
|
||||
@goal = goal
|
||||
@experiments = experiments
|
||||
@start_date = start_date&.beginning_of_day
|
||||
end
|
||||
|
||||
attr_reader :experiments, :user, :goal
|
||||
|
|
@ -30,7 +31,7 @@ class AbExperiment
|
|||
return if experiments.nil?
|
||||
|
||||
experiments.each do |key, data|
|
||||
experiment_start_date = data.fetch("start_date").beginning_of_day
|
||||
experiment_start_date = @start_date || data.fetch("start_date").beginning_of_day
|
||||
experiment = key.to_sym
|
||||
convert(experiment: experiment, experiment_start_date: experiment_start_date)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -260,6 +260,33 @@ module Articles
|
|||
group_by_fragment: "articles.privileged_users_reaction_points_sum",
|
||||
query_parameter_names: %i[negative_reaction_threshold positive_reaction_threshold])
|
||||
|
||||
# Note the symmetry of the < and >=; the smaller value is always "exclusive" and the larger value is "inclusive"
|
||||
relevancy_lever(:privileged_user_reaction_granular,
|
||||
label: "A more granular configuration for privileged user reactions (see select_fragment)",
|
||||
user_required: false,
|
||||
range: "[-2..2]",
|
||||
select_fragment: "(CASE
|
||||
--- Very negative
|
||||
WHEN articles.privileged_users_reaction_points_sum < :very_negative_reaction_threshold THEN -2
|
||||
--- Negative
|
||||
WHEN articles.privileged_users_reaction_points_sum >= :very_negative_reaction_threshold
|
||||
AND articles.privileged_users_reaction_points_sum < :negative_reaction_threshold THEN -1
|
||||
--- Neutral
|
||||
WHEN articles.privileged_users_reaction_points_sum >= :negative_reaction_threshold
|
||||
AND articles.privileged_users_reaction_points_sum < :positive_reaction_threshold THEN 0
|
||||
--- Positive
|
||||
WHEN articles.privileged_users_reaction_points_sum >= :positive_reaction_threshold
|
||||
AND articles.privileged_users_reaction_points_sum < :very_positive_reaction_threshold THEN 1
|
||||
--- Very Positive
|
||||
WHEN articles.privileged_users_reaction_points_sum >= :very_positive_reaction_threshold THEN 2
|
||||
ELSE 0 END)",
|
||||
group_by_fragment: "articles.privileged_users_reaction_points_sum",
|
||||
query_parameter_names: %i[
|
||||
very_negative_reaction_threshold
|
||||
negative_reaction_threshold
|
||||
very_positive_reaction_threshold
|
||||
positive_reaction_threshold
|
||||
])
|
||||
relevancy_lever(:public_reactions,
|
||||
label: "Weight to give for the number of unicorn, heart, reading list reactions for article.",
|
||||
range: "[0..∞)",
|
||||
|
|
|
|||
114
config/feed-variants/20220509-variant.json
Normal file
114
config/feed-variants/20220509-variant.json
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
{
|
||||
"max_days_since_published": 15,
|
||||
"order_by": "relevancy_score_and_publication_date",
|
||||
"levers": {
|
||||
"daily_decay": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.99],
|
||||
[2, 0.985],
|
||||
[3, 0.98],
|
||||
[4, 0.975],
|
||||
[5, 0.97],
|
||||
[6, 0.965],
|
||||
[7, 0.96],
|
||||
[8, 0.955],
|
||||
[9, 0.95],
|
||||
[10, 0.945],
|
||||
[11, 0.94],
|
||||
[12, 0.935],
|
||||
[13, 0.93],
|
||||
[14, 0.925]
|
||||
],
|
||||
"fallback": 0.9
|
||||
},
|
||||
"comments_count_by_those_followed": {
|
||||
"cases": [
|
||||
[0, 0.95],
|
||||
[1, 0.98],
|
||||
[2, 0.99]
|
||||
],
|
||||
"fallback": 0.93
|
||||
},
|
||||
"comments_count": {
|
||||
"cases": [
|
||||
[0, 0.8],
|
||||
[1, 0.82],
|
||||
[2, 0.84],
|
||||
[3, 0.86],
|
||||
[4, 0.88],
|
||||
[5, 0.9],
|
||||
[6, 0.92],
|
||||
[7, 0.94],
|
||||
[8, 0.96],
|
||||
[9, 0.98]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"featured_article": { "cases": [[1, 1]], "fallback": 0.85 },
|
||||
"following_author": {
|
||||
"cases": [
|
||||
[0, 0.8],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"following_org": {
|
||||
"cases": [
|
||||
[0, 0.95],
|
||||
[1, 1]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"latest_comment": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.9988]
|
||||
],
|
||||
"fallback": 0.988
|
||||
},
|
||||
"matching_negative_tags_intersection_count": {
|
||||
"cases": [
|
||||
[0, 1],
|
||||
[1, 0.8],
|
||||
[2, 0.5],
|
||||
[3, 0.3],
|
||||
[4, 0.2]
|
||||
],
|
||||
"fallback": 0
|
||||
},
|
||||
"matching_positive_tags_intersection_count": {
|
||||
"cases": [
|
||||
[0, 0.7],
|
||||
[1, 0.85],
|
||||
[2, 0.93],
|
||||
[3, 0.98],
|
||||
[4, 1]
|
||||
],
|
||||
"fallback": 1
|
||||
},
|
||||
"privileged_user_reaction_granular": {
|
||||
"cases": [
|
||||
[-2, 0.05],
|
||||
[-1, 0.5],
|
||||
[0, 0.9],
|
||||
[1, 0.98],
|
||||
[2, 1]
|
||||
],
|
||||
"fallback": 0.9,
|
||||
"very_negative_reaction_threshold": -10,
|
||||
"negative_reaction_threshold": 0,
|
||||
"positive_reaction_threshold": 5,
|
||||
"very_positive_reaction_threshold": 10
|
||||
},
|
||||
"public_reactions": {
|
||||
"cases": [
|
||||
[0, 0.9988],
|
||||
[1, 0.9988],
|
||||
[2, 0.9988],
|
||||
[3, 0.9988]
|
||||
],
|
||||
"fallback": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +1,13 @@
|
|||
# See AbExperiment namespace for details of goals and variants
|
||||
experiments:
|
||||
# NOTE: Our feed strategy testing experiment must begin with "feed_strategy"
|
||||
feed_strategy_starting_20220422:
|
||||
feed_strategy_starting_20220509:
|
||||
# NOTE: Required as we want only want to consider for conversion events that
|
||||
# occurred on or after the given start_date.
|
||||
start_date: 2022-04-22
|
||||
start_date: 2022-05-09
|
||||
variants:
|
||||
- 20220415-incumbent
|
||||
- 20220422-jennie-variant
|
||||
- 20220422-variant
|
||||
- 20220509-variant
|
||||
weights:
|
||||
- 80
|
||||
- 20
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@ RSpec.describe AbExperiment::GoalConversionHandler do
|
|||
include FieldTest::Helpers
|
||||
|
||||
describe ".call" do
|
||||
subject(:handler) { described_class.new(user: user, goal: goal, experiments: experiments) }
|
||||
subject(:handler) do
|
||||
described_class.new(user: user, goal: goal, experiments: experiments, start_date: start_date)
|
||||
end
|
||||
|
||||
let(:user) { create(:user) }
|
||||
let(:start_date) { 16.days.ago }
|
||||
let(:goal) { "non_sense" }
|
||||
let(:experiments) { FieldTest.config["experiments"] }
|
||||
|
||||
|
|
@ -21,8 +24,7 @@ RSpec.describe AbExperiment::GoalConversionHandler do
|
|||
context "with experiment that started to soon for some results" do
|
||||
let(:goal) { described_class::USER_CREATES_COMMENT_GOAL }
|
||||
let(:experiment_name) { AbExperiment::CURRENT_FEED_STRATEGY_EXPERIMENT }
|
||||
# NOTE: We're choosing a future date as a logic short-cut for comment tests
|
||||
let(:experiments) { { experiment_name => { "start_date" => 8.days.from_now } } }
|
||||
let(:start_date) { 8.days.from_now }
|
||||
|
||||
before do
|
||||
field_test(experiment_name, participant: user)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue