maverick/docbrown
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Test variants for which posts are fetched on initial "hot articles" home feed query. (#11982)

Browse source 
* Declare winner in update points worker

* Remove tests

* Test variants for which posts are fetched on home feed

* Add field test

* Rewrite field tests

* Change user to @user

* Fix tests

* Fix user thing

* Check for existence of user before setting experiment

* Fix experiment -> @experiment

* Change spec

* Fix tests

* Fix tests

* Fix tests

* Fix rubocop issues

* Remove unneeded offset value

* Fix up specs

* Final adjustments

* Add comments

* Update app/services/articles/feeds/large_forem_experimental.rb

Co-authored-by: Molly Struve <mollylbs@gmail.com>

* Update app/services/articles/feeds/large_forem_experimental.rb

Co-authored-by: Molly Struve <mollylbs@gmail.com>

* Update app/services/articles/feeds/large_forem_experimental.rb

Co-authored-by: Fernando Valverde <fdov88@gmail.com>

* Update app/services/articles/feeds/large_forem_experimental.rb

Co-authored-by: Fernando Valverde <fdov88@gmail.com>

* Better initial user query

Co-authored-by: Molly Struve <mollylbs@gmail.com>
Co-authored-by: Fernando Valverde <fdov88@gmail.com>
This commit is contained in:
Ben Halpern 2021-01-05 12:29:46 -05:00 • committed by GitHub
parent 025a46e012
commit 0ec2a40fe0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 174 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/models/comment.rb
View file

@ -305,7 +305,7 @@ class Comment < ApplicationRecord
def record_field_test_event
Users::RecordFieldTestEventWorker
.perform_async(user_id, :follow_implicit_points, "user_creates_comment")
.perform_async(user_id, "user_creates_comment")
end
def notify_slack_channel_about_warned_users

4
app/models/page_view.rb
View file

@ -31,8 +31,6 @@ class PageView < ApplicationRecord
return unless user_id
Users::RecordFieldTestEventWorker
.perform_async(user_id, :follow_implicit_points, "user_views_article_four_days_in_week")
Users::RecordFieldTestEventWorker
.perform_async(user_id, :follow_implicit_points, "user_views_article_four_hours_in_day")
.perform_async(user_id, "user_creates_pageview")
end
end

6
app/models/reaction.rb
View file

@ -36,7 +36,6 @@ class Reaction < ApplicationRecord
after_create :notify_slack_channel_about_vomit_reaction, if: -> { category == "vomit" }
before_destroy :bust_reactable_cache_without_delay
before_destroy :update_reactable_without_delay, unless: :destroyed_by_association
after_create_commit :record_field_test_event
after_commit :async_bust
after_commit :bust_reactable_cache, :update_reactable, on: %i[create update]
@ -138,11 +137,6 @@ class Reaction < ApplicationRecord
negative? && !user.trusted
end
def record_field_test_event
Users::RecordFieldTestEventWorker
.perform_async(user_id, :follow_implicit_points, "user_creates_reaction")
end
def notify_slack_channel_about_vomit_reaction
Slack::Messengers::ReactionVomit.call(reaction: self)
end

70
app/services/articles/feeds/large_forem_experimental.rb
View file

@ -1,10 +1,10 @@
module Articles
module Feeds
class LargeForemExperimental
RANDOM_OFFSET_VALUES = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13].freeze
include FieldTest::Helpers
MINIMUM_SCORE_LATEST_FEED = -20
def initialize(user: nil, number_of_articles: 35, page: 1, tag: nil)
def initialize(user: nil, number_of_articles: 50, page: 1, tag: nil)
@user = user
@number_of_articles = number_of_articles
@page = page
@ -126,26 +126,74 @@ module Articles
end
def globally_hot_articles(user_signed_in)
hot_stories = published_articles_by_tag
.where("score >= ? OR featured = ?", SiteConfig.home_feed_minimum_score, true)
.order(hotness_score: :desc)
featured_story = hot_stories.where.not(main_image: nil).first
if user_signed_in
hot_story_count = hot_stories.count
offset = RANDOM_OFFSET_VALUES.select do |i|
i < hot_story_count
end.sample # random offset, weighted more towards zero
hot_stories = hot_stories.offset(offset)
hot_stories = experimental_hot_story_grab
featured_story = hot_stories.where.not(main_image: nil).first
new_stories = Article.published
.where("score > ?", -15)
.limited_column_select.includes(top_comments: :user).order(published_at: :desc).limit(rand(15..80))
hot_stories = hot_stories.to_a + new_stories.to_a
else
hot_stories = Article.published.limited_column_select
.page(@page).per(@number_of_articles)
.where("score >= ? OR featured = ?", SiteConfig.home_feed_minimum_score, true)
.order(hotness_score: :desc)
featured_story = hot_stories.where.not(main_image: nil).first
end
[featured_story, hot_stories.to_a]
end
private
# Disable complexity cop to allow for variant-driven method
# rubocop:disable Metrics/CyclomaticComplexity
def experimental_hot_story_grab
test_variant = @user ? field_test(:feed_top_articles_query, participant: @user) : "base"
case test_variant
when "base_with_more_articles" # equivalent to current base but with higher "number of articles"
articles = Article.published.limited_column_select.includes(top_comments: :user)
.page(@page).per(75)
.where("score >= ? OR featured = ?", SiteConfig.home_feed_minimum_score, true)
.order(hotness_score: :desc)
when "only_followed_tags" # equivalent to base but only on tags user follows (if user follows enough)
followed_tags = @user.cached_followed_tag_names
articles = Article.published.limited_column_select.includes(top_comments: :user)
.page(@page).per(@number_of_articles)
.where("score >= ? OR featured = ?", SiteConfig.home_feed_minimum_score, true)
.order(hotness_score: :desc)
# We only want to limit the posts to tagged_with if the participant follows enough tags.
articles = articles.tagged_with(followed_tags, any: true) if followed_tags.size > 4
when "top_articles_since_last_pageview_3_days_max" # Top articles since last page view (max 3 days)
start_time = [(@user.page_views.last&.created_at || 3.days.ago) - 12.hours, 3.days.ago].max
articles = Article.published.limited_column_select.includes(top_comments: :user)
.where("published_at > ?", start_time)
.page(@page).per(@number_of_articles)
.order(score: :desc)
when "top_articles_since_last_pageview_7_days_max" # Top articles since last page view (max 7 days)
start_time = [(@user.page_views.last&.created_at || 7.days.ago) - 12.hours, 7.days.ago].max
articles = Article.published.limited_column_select.includes(top_comments: :user)
.where("published_at > ?", start_time)
.page(@page).per(@number_of_articles)
.order(score: :desc)
when "combination_only_tags_followed_and_top_max_7_days" # Top articles since last page view (max 7 days)
start_time = [(@user.page_views.last&.created_at || 7.days.ago) - 12.hours, 7.days.ago].max
followed_tags = @user.cached_followed_tag_names
articles = Article.published.limited_column_select.includes(top_comments: :user)
.where("published_at > ?", start_time)
.page(@page).per(@number_of_articles)
.order(score: :desc)
# We only want to limit the posts to tagged_with if the participant follows enough tags.
articles = articles.tagged_with(followed_tags, any: true) if followed_tags.size > 4
else # "base"
articles = Article.published.limited_column_select.includes(top_comments: :user)
.page(@page).per(@number_of_articles)
.where("score >= ? OR featured = ?", SiteConfig.home_feed_minimum_score, true)
.order(hotness_score: :desc)
end
articles
end
# rubocop:enable Metrics/CyclomaticComplexity
def user_followed_tags
@user_followed_tags ||= (@user&.decorate&.cached_followed_tags || [])
end

1
app/workers/follows/update_points_worker.rb
View file

@ -1,7 +1,6 @@
module Follows
class UpdatePointsWorker
include Sidekiq::Worker
include FieldTest::Helpers
sidekiq_options queue: :low_priority, retry: 10
def perform(article_id, user_id)

48
app/workers/users/record_field_test_event_worker.rb
View file

@ -5,39 +5,45 @@ module Users
sidekiq_options queue: :low_priority, retry: 10
def perform(user_id, experiment, goal)
user = User.find_by(id: user_id)
return unless user
def perform(user_id, goal)
@user = User.find_by(id: user_id)
return unless @user
@experiment = :feed_top_articles_query # Current experiment running
case goal
when "user_views_article_four_days_in_week"
determine_weekly_pageview_goal(user, experiment)
when "user_views_article_four_hours_in_day"
determine_daily_pageview_goal(user, experiment)
# We have special conditional goals for some where we look for past events for commulative wins
# Otherwise we convert the goal as given.
when "user_creates_pageview"
pageview_goal(7.days.ago, "DATE(created_at)", 4, "user_views_article_four_days_in_week")
pageview_goal(24.hours.ago, "DATE_PART('hour', created_at)", 4, "user_views_article_four_hours_in_day")
pageview_goal(14.days.ago, "DATE(created_at)", 9, "user_views_article_nine_days_in_two_week")
pageview_goal(5.days.ago, "DATE_PART('hour', created_at)", 12, "user_views_article_twelve_hours_in_five_days")
when "user_creates_comment" # comments goal. Only page views and comments are currently active.
field_test_converted(@experiment, participant: @user, goal: goal) # base single comment goal.
comment_goal(7.days.ago, "DATE(created_at)", 4, "user_creates_comment_four_days_in_week")
else
field_test_converted(experiment, participant: user, goal: goal)
field_test_converted(@experiment, participant: @user, goal: goal) # base single comment goal.
end
end
private
def determine_weekly_pageview_goal(user, experiment)
past_week_page_view_counts = user.page_views.where("created_at > ?", 7.days.ago)
.group("DATE(created_at)").count.values
past_week_page_view_counts.delete(0)
return unless past_week_page_view_counts.size > 3
def pageview_goal(time_start, group_value, min_count, goal_name)
page_view_counts = @user.page_views.where("created_at > ?", time_start)
.group(group_value).count.values
page_view_counts.delete(0)
return unless page_view_counts.size >= min_count
field_test_converted(experiment, participant: user, goal: "user_views_article_four_days_in_week")
field_test_converted(@experiment, participant: @user, goal: goal_name)
end
# Almost repeat of above method, but rule of threes dictates this is fine duplication for now.
def determine_daily_pageview_goal(user, experiment)
past_day_page_view_counts = user.page_views.where("created_at > ?", 24.hours.ago)
.group("DATE_PART('hour', created_at)").count.values
past_day_page_view_counts.delete(0)
return unless past_day_page_view_counts.size > 3
def comment_goal(time_start, group_value, min_count, goal_name)
comment_counts = @user.comments.where("created_at > ?", time_start)
.group(group_value).count.values
comment_counts.delete(0)
return unless comment_counts.size >= min_count
field_test_converted(experiment, participant: user, goal: "user_views_article_four_hours_in_day")
field_test_converted(@experiment, participant: @user, goal: goal_name)
end
end
end

57
config/field_test.yml
View file

@ -1,41 +1,36 @@
experiments:
follow_implicit_points: # Points given for implicit tag weights
# TEST FINISHED
# Hypotheses: Weighting tags based on implied preference will lead
# to more repeat engagement.
# Recommended duration: 4 weeks (ending at start of January)
# OUTCOME:
# It was observed that "base" has been consistently ahead in comments and
# "four hours of article reads in day" by 10% to 20% over no_implicit_score
# but over time this began regressing towards the mean and seeming more random.
# In inspecting the ways we are measuring goals here, anything with a low barrier
# will eventually be reached (aka leave one reaction) by most users, so over time
# these goals demonstrate less value.
# So this test is being cut off on the observation of a clear winner early on
# with the note that we need tests which can map better to the duration of the test.
# This experiment is no longer running in production, but can remain configured here
# while there are no other experiments running.
feed_top_articles_query:
# This tests how we fetch "globally hot articles" — Testing some variants of
# more focus on the particpant's tags, some other alternatives which take more into account
# when the particpant was last on the site and viewed a post.
# -> Recommended experiment duration: 2 weeks.
# This experiment has new goals measuring consistently of coming back and clocking a page view
# and two weeks should be enough to see results.
# Hypotheses. This will be more impactful than past experiments which were too modest.
# We'll get more info on bigger levers on experiments, and should be able to pit the winner
# against some more variation.
# combination_only_tags_followed_and_top_max_7_days will be interesting to observe I think.
variants:
- base
- no_implicit_score
- half_weight_after_log
- double_weight_after_log
- double_bonus_before_log
- without_weighting_bonus
weights: # "base" (what we currently implement) and "no_implicit_score" are what we want the most data on.
- 30
- 30
- 10
- 10
- 10
- 10
- base_with_more_articles
- only_followed_tags
- top_articles_since_last_pageview_3_days_max
- top_articles_since_last_pageview_7_days_max
- combination_only_tags_followed_and_top_max_7_days
weights: # "base" gets a few more rolls of the dice
- 20
- 16
- 16
- 16
- 16
- 16
goals:
- user_creates_comment
- user_creates_reaction
- user_creates_comment_four_days_in_week
- user_views_article_four_days_in_week
- user_views_article_four_hours_in_day
- user_views_article_nine_days_in_two_week
- user_views_article_twelve_hours_in_five_days
exclude:
bots: true

2
spec/requests/comments_spec.rb
View file

@ -318,7 +318,7 @@ RSpec.describe "Comments", type: :request do
it "converts field test" do
post "/comments", params: base_comment_params
expected_args = [user.id, :follow_implicit_points, "user_creates_comment"]
expected_args = [user.id, "user_creates_comment"]
expect(Users::RecordFieldTestEventWorker).to have_received(:perform_async).with(*expected_args)
end
end

2
spec/requests/page_views_spec.rb
View file

@ -56,7 +56,7 @@ RSpec.describe "PageViews", type: :request do
referrer: "test"
}
expect(Users::RecordFieldTestEventWorker).to have_received(:perform_async)
.with(user.id, :follow_implicit_points, "user_views_article_four_days_in_week")
.with(user.id, "user_creates_pageview")
end
end

14
spec/requests/reactions_spec.rb
View file

@ -316,20 +316,6 @@ RSpec.describe "Reactions", type: :request do
end
end
context "when part of field test" do
before do
sign_in user
allow(Users::RecordFieldTestEventWorker).to receive(:perform_async)
end
it "converts field test" do
post "/reactions", params: article_params
expect(Users::RecordFieldTestEventWorker).to have_received(:perform_async).with(user.id,
:follow_implicit_points,
"user_creates_reaction")
end
end
context "when signed out" do
it "returns an unauthorized error" do
expect { post "/reactions", params: article_params }.to raise_error(Pundit::NotAuthorizedError)

18
spec/services/articles/feeds/large_forem_experimental_spec.rb
View file

@ -15,6 +15,7 @@ RSpec.describe Articles::Feeds::LargeForemExperimental, type: :service do
let(:tagged_article) { create(:article, tags: tag) }
it "returns published articles" do
result = feed.published_articles_by_tag
expect(result).to include article
expect(result).not_to include unpublished_article
@ -126,6 +127,23 @@ RSpec.describe Articles::Feeds::LargeForemExperimental, type: :service do
expect(stories).to include(new_story)
end
end
context "when experiment is running" do
it "works with every variant" do
# Basic test to see that these all work.
%i[base
base_with_more_articles
only_followed_tags
top_articles_since_last_pageview_3_days_max
top_articles_since_last_pageview_7_days_max
combination_only_tags_followed_and_top_max_7_days].each do |experiment|
create(:field_test_membership,
experiment: experiment, variant: "base", participant_id: user.id)
stories = feed.default_home_feed(user_signed_in: true)
expect(stories.size).to be > 0
end
end
end
end
describe "more_comments_minimal_weight_randomized_at_end" do

8
spec/workers/follows/update_points_worker_spec.rb
View file

@ -21,8 +21,6 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do
end
it "calculates scores" do
create(:field_test_membership,
experiment: :follow_implicit_points, variant: :base, participant_id: user.id)
follow = Follow.last
follow.update_column(:explicit_points, 2.2)
worker.perform(reaction.reactable_id, reaction.user_id)
@ -32,8 +30,6 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do
end
it "has higher score with more long page views" do
create(:field_test_membership,
experiment: :follow_implicit_points, variant: :base, participant_id: user.id)
follow = Follow.last
worker.perform(reaction.reactable_id, reaction.user_id)
follow.reload
@ -47,8 +43,6 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do
end
it "has higher score with more reactions" do
create(:field_test_membership,
experiment: :follow_implicit_points, variant: :base, participant_id: user.id)
follow = Follow.last
worker.perform(reaction.reactable_id, reaction.user_id)
follow.reload
@ -68,8 +62,6 @@ RSpec.describe Follows::UpdatePointsWorker, type: :worker do
end
it "applies inverse bonus to slightly penalize more popular tags" do
create(:field_test_membership,
experiment: :follow_implicit_points, variant: :base, participant_id: user.id)
follow = Follow.last
tag.update_column(:hotness_score, 1000)
second_tag.update_column(:hotness_score, 100)

52
spec/workers/users/record_field_test_event_worker_spec.rb
View file

@ -11,35 +11,63 @@ RSpec.describe Users::RecordFieldTestEventWorker, type: :worker do
context "with user who is part of field test" do
before do
field_test(:follow_implicit_points, participant: user)
field_test(:feed_top_articles_query, participant: user)
end
it "records user_creates_reaction field test conversion" do
worker.perform(user.id, "follow_implicit_points", "user_creates_reaction")
worker.perform(user.id, "user_creates_reaction")
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.last.name).to eq("user_creates_reaction")
end
it "records user_creates_comment field test conversion" do
worker.perform(user.id, "follow_implicit_points", "user_creates_comment")
worker.perform(user.id, "user_creates_comment")
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.last.name).to eq("user_creates_comment")
end
it "records user_creates_comment_four_days_in_week field test conversion if qualifies" do
7.times do |n|
create(:comment, user_id: user.id, created_at: n.days.ago)
end
worker.perform(user.id, "user_creates_comment")
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.last.name).to eq("user_creates_comment_four_days_in_week")
end
it "records user_views_article_four_days_in_week field test conversion if qualifies" do
7.times do |n|
create(:page_view, user_id: user.id, created_at: n.days.ago)
end
worker.perform(user.id, "follow_implicit_points", "user_views_article_four_days_in_week")
worker.perform(user.id, "user_creates_pageview")
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.last.name).to eq("user_views_article_four_days_in_week")
expect(FieldTest::Event.pluck(:name)).to include("user_views_article_four_days_in_week")
end
it "records user_views_article_nine_days_in_two_week field test conversion if qualifies" do
10.times do |n|
create(:page_view, user_id: user.id, created_at: n.days.ago)
end
worker.perform(user.id, "user_creates_pageview")
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.pluck(:name)).to include("user_views_article_nine_days_in_two_week")
end
it "records user_views_article_twelve_hours_in_five_days field test conversion if qualifies" do
15.times do |n|
create(:page_view, user_id: user.id, created_at: n.hours.ago)
end
worker.perform(user.id, "user_creates_pageview")
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.pluck(:name)).to include("user_views_article_twelve_hours_in_five_days")
end
it "does not record user_views_article_four_days_in_week field test conversion if not qualifying" do
2.times do |n|
create(:page_view, user_id: user.id, created_at: n.days.ago)
end
worker.perform(user.id, "follow_implicit_points", "user_views_article_four_days_in_week")
worker.perform(user.id, "user_creates_pageview")
expect(FieldTest::Event.all.size).to be(0)
end
@ -47,28 +75,28 @@ RSpec.describe Users::RecordFieldTestEventWorker, type: :worker do
7.times do |n|
create(:page_view, user_id: user.id, created_at: n.hours.ago)
end
worker.perform(user.id, "follow_implicit_points", "user_views_article_four_hours_in_day")
worker.perform(user.id, "user_creates_pageview")
expect(FieldTest::Event.last.field_test_membership.participant_id).to eq(user.id.to_s)
expect(FieldTest::Event.last.name).to eq("user_views_article_four_hours_in_day")
expect(FieldTest::Event.pluck(:name)).to include("user_views_article_four_hours_in_day")
end
it "does not record user_views_article_four_hours_in_day field test conversion if not qualifying" do
2.times do |n|
create(:page_view, user_id: user.id, created_at: n.hours.ago)
end
worker.perform(user.id, "follow_implicit_points", "user_views_article_four_hours_in_day")
worker.perform(user.id, "user_creates_pageview")
expect(FieldTest::Event.all.size).to be(0)
end
end
context "with user who is not part of field test" do
it "records user_creates_reaction field test conversion" do
worker.perform(user.id, "follow_implicit_points", "user_creates_reaction")
worker.perform(user.id, "user_creates_reaction")
expect(FieldTest::Event.all.size).to be(0)
end
it "records user_creates_comment field test conversion" do
worker.perform(user.id, "follow_implicit_points", "user_creates_comment")
worker.perform(user.id, "user_creates_comment")
expect(FieldTest::Event.all.size).to be(0)
end
@ -83,7 +111,7 @@ RSpec.describe Users::RecordFieldTestEventWorker, type: :worker do
context "without a user" do
it "does not raise an error" do
expect do
worker.perform(user.id + 1000, "follow_implicit_points", "user_creates_reaction")
worker.perform(user.id + 1000, "user_creates_reaction")
end.not_to raise_error
end
end