From 12efe40e63b0c981d68c052befb03ce6c1214a4b Mon Sep 17 00:00:00 2001 From: Ben Halpern Date: Thu, 3 Dec 2020 08:39:04 -0500 Subject: [PATCH] Add a/b test for implicit follow points (#11675) * Add a/b test for implicit follow points * Add field test specs * Add a/b testing docs * Fix tests * Fix tests * Update docs/technical-overview/ab_testing.md Co-authored-by: Molly Struve * Add comment * Update docs/technical-overview/ab_testing.md Co-authored-by: Jacob Herrington * Update config/field_test.yml * Update docs/technical-overview/ab_testing.md Co-authored-by: Julianna Tetreault <32834804+juliannatetreault@users.noreply.github.com> * Update docs/technical-overview/ab_testing.md Co-authored-by: Julianna Tetreault <32834804+juliannatetreault@users.noreply.github.com> Co-authored-by: Molly Struve Co-authored-by: Jacob Herrington Co-authored-by: Julianna Tetreault <32834804+juliannatetreault@users.noreply.github.com> --- app/workers/follows/update_points_worker.rb | 21 +++++++++- config/field_test.yml | 24 ++++++++++++ docs/technical-overview/ab_testing.md | 7 ++++ docs/technical-overview/readme.md | 1 + .../follows/update_points_worker_spec.rb | 39 +++++++++++++++++++ 5 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 docs/technical-overview/ab_testing.md diff --git a/app/workers/follows/update_points_worker.rb b/app/workers/follows/update_points_worker.rb index 3b9322ce8..a23c8475b 100644 --- a/app/workers/follows/update_points_worker.rb +++ b/app/workers/follows/update_points_worker.rb @@ -1,6 +1,7 @@ module Follows class UpdatePointsWorker include Sidekiq::Worker + include FieldTest::Helpers sidekiq_options queue: :low_priority, retry: 10 def perform(article_id, user_id) @@ -34,7 +35,7 @@ module Follows tags = articles.pluck(:cached_tag_list).compact.flat_map { |list| list.split(", ") } occurrences = tags.count(tag.name) bonus = inverse_popularity_bonus(tag) - Math.log(occurrences + bonus + 1) # +1 is purely to avoid log(0) => -infinity + finalized_points(occurrences, bonus, user) end def adjust_other_tag_follows_of_user(user_id) @@ -63,6 +64,24 @@ module Follows top_100_tag_names.index(tag.name) || (top_100_tag_names.size * 1.5) end + def finalized_points(occurrences, bonus, user) + test_variant = field_test(:follow_implicit_points, participant: user) + case test_variant + when "no_implicit_score" + 0 + when "half_weight_after_log" + Math.log(occurrences + bonus + 1) * 0.5 + when "double_weight_after_log" + Math.log(occurrences + bonus + 1) * 2.0 + when "double_bonus_before_log" + Math.log(occurrences + (bonus * 2) + 1) + when "without_weighting_bonus" + Math.log(occurrences + 1) + else # base - Our current "default" implementation + Math.log(occurrences + bonus + 1) # + 1 in all cases is to avoid log(0) => -infinity + end + end + def cached_app_wide_top_tag_names Rails.cache.fetch("top-100-tags") do Tag.order(hotness_score: :desc).limit(100).pluck(:name) diff --git a/config/field_test.yml b/config/field_test.yml index f3ac0a7e5..a3accfff3 100644 --- a/config/field_test.yml +++ b/config/field_test.yml @@ -9,6 +9,30 @@ experiments: - user_creates_reaction - user_views_article_four_days_in_week - user_views_article_four_hours_in_day + follow_implicit_points: # Points given for implicit tag weights + # Implemented here: app/workers/follows/update_points_worker.rb + # Hypotheses: Weighting tags based on implied preference will lead + # to more repeat engagement. + # Recommended duration: 4 weeks (ending at start of January) + variants: + - base + - no_implicit_score + - half_weight_after_log + - double_weight_after_log + - double_bonus_before_log + - without_weighting_bonus + weights: # "base" (what we currently implement) and "no_implicit_score" are what we want the most data on. + - 30 + - 30 + - 10 + - 10 + - 10 + - 10 + goals: + - user_creates_comment + - user_creates_reaction + - user_views_article_four_days_in_week + - user_views_article_four_hours_in_day exclude: bots: true diff --git a/docs/technical-overview/ab_testing.md b/docs/technical-overview/ab_testing.md new file mode 100644 index 000000000..b998e5222 --- /dev/null +++ b/docs/technical-overview/ab_testing.md @@ -0,0 +1,7 @@ +# A/B testing + +We use the [Field Test](https://github.com/ankane/field_test) gem for conducting simple A/B tests. + +If you want to propose an A/B test of a feature, you may make a pull request which defines the hypotheses and what admins should look for to declare a winner. As A/B tests are going to have results that may differ from Forem to Forem, the process is relatively immature. In the future we may have more studies that can return anonymous ecosystem-wide results. + +A/B tests are inherently the most useful in _large_ Forems, where qualitative feedback may be more useful on small Forems. As such, [DEV](https://dev.to) is our largest Forem and therefore can provide us with the most feedback for our existing A/B tests. However, we must keep in mind that DEV results may not apply well to future large Forems. We should seek to re-run useful experiments within the ecosystem after time has passed. diff --git a/docs/technical-overview/readme.md b/docs/technical-overview/readme.md index 74f84e3d7..1bc35f6d7 100644 --- a/docs/technical-overview/readme.md +++ b/docs/technical-overview/readme.md @@ -4,6 +4,7 @@ items: - architecture.md - feed.md - compatibility.md + - ab_testing.md --- # Technical Overview diff --git a/spec/workers/follows/update_points_worker_spec.rb b/spec/workers/follows/update_points_worker_spec.rb index 6560c9ab2..388ff9ae1 100644 --- a/spec/workers/follows/update_points_worker_spec.rb +++ b/spec/workers/follows/update_points_worker_spec.rb @@ -21,6 +21,8 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do end it "calculates scores" do + create(:field_test_membership, + experiment: :follow_implicit_points, variant: :base, participant_id: user.id) follow = Follow.last follow.update_column(:explicit_points, 2.2) worker.perform(reaction.reactable_id, reaction.user_id) @@ -30,6 +32,8 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do end it "has higher score with more long page views" do + create(:field_test_membership, + experiment: :follow_implicit_points, variant: :base, participant_id: user.id) follow = Follow.last worker.perform(reaction.reactable_id, reaction.user_id) follow.reload @@ -43,6 +47,8 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do end it "has higher score with more reactions" do + create(:field_test_membership, + experiment: :follow_implicit_points, variant: :base, participant_id: user.id) follow = Follow.last worker.perform(reaction.reactable_id, reaction.user_id) follow.reload @@ -62,6 +68,8 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do end it "applies inverse bonus to slightly penalize more popular tags" do + create(:field_test_membership, + experiment: :follow_implicit_points, variant: :base, participant_id: user.id) follow = Follow.last tag.update_column(:hotness_score, 1000) second_tag.update_column(:hotness_score, 100) @@ -74,5 +82,36 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do expect(follow.reload.points).to be > original_points # should be higher because tag is now less popular end + + context "with field tests in place" do + # regressions testing a few field test scenarios + it "returns zero if no_implicit_score field test" do + create(:field_test_membership, + experiment: :follow_implicit_points, variant: "no_implicit_score", participant_id: user.id) + follow = Follow.last + follow.update_column(:explicit_points, 2.2) + worker.perform(reaction.reactable_id, reaction.user_id) + follow.reload + expect(follow.implicit_points).to eq 0 + end + + it "returns double if double_weight_after_log field test" do + create(:field_test_membership, + experiment: :follow_implicit_points, variant: "double_weight_after_log", participant_id: user.id) + follow = Follow.last + worker.perform(reaction.reactable_id, reaction.user_id) + follow.reload + expect(follow.implicit_points).to be_within(0.9).of(2) + end + + it "returns double if half_weight_after_log field test" do + create(:field_test_membership, + experiment: :follow_implicit_points, variant: "half_weight_after_log", participant_id: user.id) + follow = Follow.last + worker.perform(reaction.reactable_id, reaction.user_id) + follow.reload + expect(follow.implicit_points).to be_within(0.7).of(0.5) + end + end end end