Add a/b test for implicit follow points (#11675)

* Add a/b test for implicit follow points

* Add field test specs

* Add a/b testing docs

* Fix tests

* Fix tests

* Update docs/technical-overview/ab_testing.md

Co-authored-by: Molly Struve <mollylbs@gmail.com>

* Add comment

* Update docs/technical-overview/ab_testing.md

Co-authored-by: Jacob Herrington <jacobherringtondeveloper@gmail.com>

* Update config/field_test.yml

* Update docs/technical-overview/ab_testing.md

Co-authored-by: Julianna Tetreault <32834804+juliannatetreault@users.noreply.github.com>

* Update docs/technical-overview/ab_testing.md

Co-authored-by: Julianna Tetreault <32834804+juliannatetreault@users.noreply.github.com>

Co-authored-by: Molly Struve <mollylbs@gmail.com>
Co-authored-by: Jacob Herrington <jacobherringtondeveloper@gmail.com>
Co-authored-by: Julianna Tetreault <32834804+juliannatetreault@users.noreply.github.com>

app/workers/follows/update_points_worker.rb

@@ -1,6 +1,7 @@
 module Follows
   class UpdatePointsWorker
     include Sidekiq::Worker
+    include FieldTest::Helpers
     sidekiq_options queue: :low_priority, retry: 10
 
     def perform(article_id, user_id)
@@ -34,7 +35,7 @@ module Follows
       tags = articles.pluck(:cached_tag_list).compact.flat_map { |list| list.split(", ") }
       occurrences = tags.count(tag.name)
       bonus = inverse_popularity_bonus(tag)
-      Math.log(occurrences + bonus + 1) # +1 is purely to avoid log(0) => -infinity
+      finalized_points(occurrences, bonus, user)
     end
 
     def adjust_other_tag_follows_of_user(user_id)
@@ -63,6 +64,24 @@ module Follows
       top_100_tag_names.index(tag.name) || (top_100_tag_names.size * 1.5)
     end
 
+    def finalized_points(occurrences, bonus, user)
+      test_variant = field_test(:follow_implicit_points, participant: user)
+      case test_variant
+      when "no_implicit_score"
+        0
+      when "half_weight_after_log"
+        Math.log(occurrences + bonus + 1) * 0.5
+      when "double_weight_after_log"
+        Math.log(occurrences + bonus + 1) * 2.0
+      when "double_bonus_before_log"
+        Math.log(occurrences + (bonus * 2) + 1)
+      when "without_weighting_bonus"
+        Math.log(occurrences + 1)
+      else # base - Our current "default" implementation
+        Math.log(occurrences + bonus + 1) # + 1 in all cases is to avoid log(0) => -infinity
+      end
+    end
+
     def cached_app_wide_top_tag_names
       Rails.cache.fetch("top-100-tags") do
         Tag.order(hotness_score: :desc).limit(100).pluck(:name)

config/field_test.yml

@@ -9,6 +9,30 @@ experiments:
       - user_creates_reaction
       - user_views_article_four_days_in_week
       - user_views_article_four_hours_in_day
+  follow_implicit_points: # Points given for implicit tag weights
+    # Implemented here: app/workers/follows/update_points_worker.rb
+    # Hypotheses: Weighting tags based on implied preference will lead
+    # to more repeat engagement.
+    # Recommended duration: 4 weeks (ending at start of January)
+    variants:
+      - base
+      - no_implicit_score
+      - half_weight_after_log
+      - double_weight_after_log
+      - double_bonus_before_log
+      - without_weighting_bonus
+    weights: # "base" (what we currently implement) and "no_implicit_score" are what we want the most data on.
+      - 30
+      - 30
+      - 10
+      - 10
+      - 10
+      - 10
+    goals:
+      - user_creates_comment
+      - user_creates_reaction
+      - user_views_article_four_days_in_week
+      - user_views_article_four_hours_in_day
 exclude:
   bots: true
 

docs/technical-overview/ab_testing.md

@@ -0,0 +1,7 @@
+# A/B testing
+
+We use the [Field Test](https://github.com/ankane/field_test) gem for conducting simple A/B tests.
+
+If you want to propose an A/B test of a feature, you may make a pull request which defines the hypotheses and what admins should look for to declare a winner. As A/B tests are going to have results that may differ from Forem to Forem, the process is relatively immature. In the future we may have more studies that can return anonymous ecosystem-wide results.
+
+A/B tests are inherently the most useful in _large_ Forems, where qualitative feedback may be more useful on small Forems. As such, [DEV](https://dev.to) is our largest Forem and therefore can provide us with the most feedback for our existing A/B tests. However, we must keep in mind that DEV results may not apply well to future large Forems. We should seek to re-run useful experiments within the ecosystem after time has passed.

docs/technical-overview/readme.md

@@ -4,6 +4,7 @@ items:
   - architecture.md
   - feed.md
   - compatibility.md
+  - ab_testing.md
 ---
 
 # Technical Overview

spec/workers/follows/update_points_worker_spec.rb

@@ -21,6 +21,8 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do
     end
 
     it "calculates scores" do
+      create(:field_test_membership,
+             experiment: :follow_implicit_points, variant: :base, participant_id: user.id)
       follow = Follow.last
       follow.update_column(:explicit_points, 2.2)
       worker.perform(reaction.reactable_id, reaction.user_id)
@@ -30,6 +32,8 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do
     end
 
     it "has higher score with more long page views" do
+      create(:field_test_membership,
+             experiment: :follow_implicit_points, variant: :base, participant_id: user.id)
       follow = Follow.last
       worker.perform(reaction.reactable_id, reaction.user_id)
       follow.reload
@@ -43,6 +47,8 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do
     end
 
     it "has higher score with more reactions" do
+      create(:field_test_membership,
+             experiment: :follow_implicit_points, variant: :base, participant_id: user.id)
       follow = Follow.last
       worker.perform(reaction.reactable_id, reaction.user_id)
       follow.reload
@@ -62,6 +68,8 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do
     end
 
     it "applies inverse bonus to slightly penalize more popular tags" do
+      create(:field_test_membership,
+             experiment: :follow_implicit_points, variant: :base, participant_id: user.id)
       follow = Follow.last
       tag.update_column(:hotness_score, 1000)
       second_tag.update_column(:hotness_score, 100)
@@ -74,5 +82,36 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do
 
       expect(follow.reload.points).to be > original_points # should be higher because tag is now less popular
     end
+
+    context "with field tests in place" do
+      # regressions testing a few field test scenarios
+      it "returns zero if no_implicit_score field test" do
+        create(:field_test_membership,
+               experiment: :follow_implicit_points, variant: "no_implicit_score", participant_id: user.id)
+        follow = Follow.last
+        follow.update_column(:explicit_points, 2.2)
+        worker.perform(reaction.reactable_id, reaction.user_id)
+        follow.reload
+        expect(follow.implicit_points).to eq 0
+      end
+
+      it "returns double if double_weight_after_log field test" do
+        create(:field_test_membership,
+               experiment: :follow_implicit_points, variant: "double_weight_after_log", participant_id: user.id)
+        follow = Follow.last
+        worker.perform(reaction.reactable_id, reaction.user_id)
+        follow.reload
+        expect(follow.implicit_points).to be_within(0.9).of(2)
+      end
+
+      it "returns double if half_weight_after_log field test" do
+        create(:field_test_membership,
+               experiment: :follow_implicit_points, variant: "half_weight_after_log", participant_id: user.id)
+        follow = Follow.last
+        worker.perform(reaction.reactable_id, reaction.user_id)
+        follow.reload
+        expect(follow.implicit_points).to be_within(0.7).of(0.5)
+      end
+    end
   end
 end