From 108d753d6884964681021131ddb790962166d97b Mon Sep 17 00:00:00 2001 From: Ben Halpern Date: Fri, 5 Jan 2024 09:37:53 -0500 Subject: [PATCH] Add articles clickbait_score as factor in final feed ordering (#20493) * Add articles clickbait_score * Add attributes * Fix field test config --- .../concerns/api/articles_controller.rb | 1 + app/models/article.rb | 1 + app/models/articles/feeds.rb | 10 +- app/services/articles/attributes.rb | 2 +- config/feed-variants/20240104-variant-a.json | 146 ++++++++++++++++++ config/feed-variants/20240104-variant-b.json | 146 ++++++++++++++++++ config/feed-variants/20240104-variant-c.json | 146 ++++++++++++++++++ config/field_test.yml | 26 ++++ ...223627_add_clickbait_factor_to_articles.rb | 6 + db/schema.rb | 3 +- spec/requests/api/v1/articles_spec.rb | 15 ++ 11 files changed, 499 insertions(+), 3 deletions(-) create mode 100644 config/feed-variants/20240104-variant-a.json create mode 100644 config/feed-variants/20240104-variant-b.json create mode 100644 config/feed-variants/20240104-variant-c.json create mode 100644 db/migrate/20240103223627_add_clickbait_factor_to_articles.rb diff --git a/app/controllers/concerns/api/articles_controller.rb b/app/controllers/concerns/api/articles_controller.rb index b26abb779..794741779 100644 --- a/app/controllers/concerns/api/articles_controller.rb +++ b/app/controllers/concerns/api/articles_controller.rb @@ -151,6 +151,7 @@ module Api :published_at ] allowed_params << :organization_id if params.dig("article", "organization_id") && allowed_to_change_org_id? + allowed_params << :clickbait_score if @user.super_admin? params.require(:article).permit(allowed_params) end diff --git a/app/models/article.rb b/app/models/article.rb index 627d5c3de..d533396b6 100644 --- a/app/models/article.rb +++ b/app/models/article.rb @@ -178,6 +178,7 @@ class Article < ApplicationRecord validates :video_source_url, url: { allow_blank: true, schemes: ["https"] } validates :video_state, inclusion: { in: %w[PROGRESSING COMPLETED] }, allow_nil: true validates :video_thumbnail_url, url: { allow_blank: true, schemes: %w[https http] } + validates :clickbait_score, numericality: { greater_than_or_equal_to: 0.0, less_than_or_equal_to: 1.0 } validate :future_or_current_published_at, on: :create validate :correct_published_at?, on: :update, unless: :admin_update diff --git a/app/models/articles/feeds.rb b/app/models/articles/feeds.rb index 0215a2712..d06ad8e5f 100644 --- a/app/models/articles/feeds.rb +++ b/app/models/articles/feeds.rb @@ -92,7 +92,15 @@ module Articles order_by_lever(:final_order_by_feed_success_score, label: "Order by feed success score", order_by_fragment: "articles.feed_success_score DESC") - + order_by_lever(:final_order_by_feed_success_score_minus_clickbait_score, + label: "Order by feed success score minus clickbait score", + order_by_fragment: "articles.feed_success_score - articles.clickbait_score DESC") + order_by_lever(:final_order_by_feed_success_score_minus_half_of_clickbait_score, + label: "Order by feed success score minus half of clickbait score", + order_by_fragment: "articles.feed_success_score - (articles.clickbait_score / 2) DESC") + order_by_lever(:final_order_by_feed_success_score_minus_one_tenth_of_clickbait_score, + label: "Order by feed success score minus one tenth of clickbait score", + order_by_fragment: "articles.feed_success_score - (articles.clickbait_score / 10) DESC") order_by_lever(:final_order_by_feed_success_score_and_primary_score, label: "Order by feed success score and primary score", order_by_fragment: "((articles.feed_success_score + 0.01) * (articles.score / 10)) DESC") diff --git a/app/services/articles/attributes.rb b/app/services/articles/attributes.rb index fef8a6a57..fc3e888b7 100644 --- a/app/services/articles/attributes.rb +++ b/app/services/articles/attributes.rb @@ -1,7 +1,7 @@ module Articles class Attributes ATTRIBUTES = %i[archived body_markdown canonical_url description - edited_at main_image organization_id user_id published + edited_at main_image organization_id user_id published clickbait_score title video_thumbnail_url published_at co_author_ids_list].freeze attr_reader :attributes, :article_user diff --git a/config/feed-variants/20240104-variant-a.json b/config/feed-variants/20240104-variant-a.json new file mode 100644 index 000000000..6e46de190 --- /dev/null +++ b/config/feed-variants/20240104-variant-a.json @@ -0,0 +1,146 @@ +{ + "max_days_since_published": 15, + "description": "Built off 20231127-variant-c but minus clickbait score for final order.", + "order_by": "final_order_by_feed_success_score_minus_clickbait_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.905], + [3, 0.7], + [4, 0.6], + [5, 0.5], + [6, 0.3], + [7, 0.2], + [8, 0.02], + [9, 0.01], + [10, 0.009], + [11, 0.008], + [12, 0.007], + [13, 0.006], + [14, 0.0025] + ], + "fallback": 0.01 + }, + "comments_count": { + "cases": [ + [0, 0.15], + [1, 0.3], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.99 + }, + "following_author": { + "cases": [ + [0, 0.7], + [1, 1] + ], + "fallback": 0.7 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.15 }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.2], + [2, 0.15], + [3, 0.1], + [4, 0.05] + ], + "fallback": 0 + }, + "experience": { + "cases": [ + [0, 1], + [1, 0.97], + [2, 0.95], + [3, 0.88], + [4, 0.77], + [5, 0.55], + [6, 0.35], + [7, 0.3] + ], + "default_user_experience_level": 5, + "fallback": 0.2 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.001], + [1, 0.97] + ], + "fallback": 1 + }, + "matching_positive_tags_intersection_points": { + "cases": [ + [0, 0.01], + [1, 0.45], + [2, 0.5], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.9], + [8, 0.93], + [9, 0.95] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions_score": { + "cases": [ + [0, 0.3], + [1, 0.4], + [2, 0.45], + [3, 0.5], + [4, 0.55], + [5, 0.6], + [6, 0.61], + [7, 0.62], + [8, 0.7], + [9, 0.8], + [10, 0.85], + [11, 0.9], + [12, 0.95] + ], + "fallback": 1.0 + }, + "language_match": { + "cases": [ + [0, 0.1], + [1, 1] + ], + "fallback": 1 + }, + "recommended_articles_match": { + "cases": [ + [0, 0.001], + [1, 1] + ], + "fallback": 1 + } + } +} diff --git a/config/feed-variants/20240104-variant-b.json b/config/feed-variants/20240104-variant-b.json new file mode 100644 index 000000000..13edf35c2 --- /dev/null +++ b/config/feed-variants/20240104-variant-b.json @@ -0,0 +1,146 @@ +{ + "max_days_since_published": 15, + "description": "Built off 20231127-variant-c but minus clickbait score.", + "order_by": "final_order_by_feed_success_score_minus_clickbait_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.905], + [3, 0.7], + [4, 0.6], + [5, 0.5], + [6, 0.3], + [7, 0.2], + [8, 0.02], + [9, 0.01], + [10, 0.009], + [11, 0.008], + [12, 0.007], + [13, 0.006], + [14, 0.0025] + ], + "fallback": 0.01 + }, + "comments_count": { + "cases": [ + [0, 0.15], + [1, 0.3], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.99 + }, + "following_author": { + "cases": [ + [0, 0.7], + [1, 1] + ], + "fallback": 0.7 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.15 }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.2], + [2, 0.15], + [3, 0.1], + [4, 0.05] + ], + "fallback": 0 + }, + "experience": { + "cases": [ + [0, 1], + [1, 0.97], + [2, 0.95], + [3, 0.88], + [4, 0.77], + [5, 0.55], + [6, 0.35], + [7, 0.3] + ], + "default_user_experience_level": 5, + "fallback": 0.2 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.001], + [1, 0.97] + ], + "fallback": 1 + }, + "matching_positive_tags_intersection_points": { + "cases": [ + [0, 0.01], + [1, 0.45], + [2, 0.5], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.9], + [8, 0.93], + [9, 0.95] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions_score": { + "cases": [ + [0, 0.3], + [1, 0.4], + [2, 0.45], + [3, 0.5], + [4, 0.55], + [5, 0.6], + [6, 0.61], + [7, 0.62], + [8, 0.7], + [9, 0.8], + [10, 0.85], + [11, 0.9], + [12, 0.95] + ], + "fallback": 1.0 + }, + "language_match": { + "cases": [ + [0, 0.1], + [1, 1] + ], + "fallback": 1 + }, + "recommended_articles_match": { + "cases": [ + [0, 0.001], + [1, 1] + ], + "fallback": 1 + } + } +} diff --git a/config/feed-variants/20240104-variant-c.json b/config/feed-variants/20240104-variant-c.json new file mode 100644 index 000000000..41fa8ad8d --- /dev/null +++ b/config/feed-variants/20240104-variant-c.json @@ -0,0 +1,146 @@ +{ + "max_days_since_published": 15, + "description": "Built off 20231127-variant-c but minus one tenth of clickbait score for final order.", + "order_by": "final_order_by_feed_success_score_minus_one_tenth_of_clickbait_score", + "reseed_randomizer_on_each_request": false, + "levers": { + "daily_decay": { + "cases": [ + [0, 1], + [1, 0.99], + [2, 0.905], + [3, 0.7], + [4, 0.6], + [5, 0.5], + [6, 0.3], + [7, 0.2], + [8, 0.02], + [9, 0.01], + [10, 0.009], + [11, 0.008], + [12, 0.007], + [13, 0.006], + [14, 0.0025] + ], + "fallback": 0.01 + }, + "comments_count": { + "cases": [ + [0, 0.15], + [1, 0.3], + [2, 0.66], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.88], + [8, 0.9], + [9, 0.92], + [12, 1.0], + [18, 1.0], + [22, 1.0], + [25, 1.0], + [30, 1.0], + [35, 1.0], + [40, 1.0], + [45, 1.0] + ], + "fallback": 0.99 + }, + "following_author": { + "cases": [ + [0, 0.7], + [1, 1] + ], + "fallback": 0.7 + }, + "featured_article": { "cases": [[1, 1]], "fallback": 0.15 }, + "matching_negative_tags_intersection_count": { + "cases": [ + [0, 1], + [1, 0.2], + [2, 0.15], + [3, 0.1], + [4, 0.05] + ], + "fallback": 0 + }, + "experience": { + "cases": [ + [0, 1], + [1, 0.97], + [2, 0.95], + [3, 0.88], + [4, 0.77], + [5, 0.55], + [6, 0.35], + [7, 0.3] + ], + "default_user_experience_level": 5, + "fallback": 0.2 + }, + "matching_positive_tags_intersection_count": { + "cases": [ + [0, 0.001], + [1, 0.97] + ], + "fallback": 1 + }, + "matching_positive_tags_intersection_points": { + "cases": [ + [0, 0.01], + [1, 0.45], + [2, 0.5], + [3, 0.7], + [4, 0.75], + [5, 0.8], + [6, 0.85], + [7, 0.9], + [8, 0.93], + [9, 0.95] + ], + "fallback": 1 + }, + "privileged_user_reaction": { + "cases": [ + [-1, 0.2], + [1, 1] + ], + "fallback": 0.95, + "negative_reaction_threshold": -9, + "positive_reaction_threshold": 4 + }, + "public_reactions_score": { + "cases": [ + [0, 0.3], + [1, 0.4], + [2, 0.45], + [3, 0.5], + [4, 0.55], + [5, 0.6], + [6, 0.61], + [7, 0.62], + [8, 0.7], + [9, 0.8], + [10, 0.85], + [11, 0.9], + [12, 0.95] + ], + "fallback": 1.0 + }, + "language_match": { + "cases": [ + [0, 0.1], + [1, 1] + ], + "fallback": 1 + }, + "recommended_articles_match": { + "cases": [ + [0, 0.001], + [1, 1] + ], + "fallback": 1 + } + } +} diff --git a/config/field_test.yml b/config/field_test.yml index c62f0a04d..ef26cb41a 100644 --- a/config/field_test.yml +++ b/config/field_test.yml @@ -28,6 +28,30 @@ ################################################################################ experiments: # NOTE: Our feed strategy testing experiment must begin with "feed_strategy" + feed_strategy_starting_20240104: + started_at: 2024-01-04 + variants: + - 20231127-variant-c + - 20240104-variant-a + - 20240104-variant-b + - 20240104-variant-c + weights: + - 64 + - 12 + - 12 + - 12 + goals: + - user_creates_pageview + - user_creates_article_reaction + - user_creates_comment + - user_views_pages_on_at_least_two_different_days_within_a_week + - user_views_pages_on_at_least_four_different_days_within_a_week + - user_creates_article_reaction_on_four_different_days_within_a_week + - user_creates_comment_on_at_least_four_different_days_within_a_week + - user_views_pages_on_at_least_three_different_hours_within_a_day + - user_views_pages_on_at_least_four_different_hours_within_a_day + - user_views_pages_on_at_least_twelve_different_hours_within_five_days + - user_views_pages_on_at_least_nine_different_days_within_two_weeks feed_strategy_starting_20231205: # NOTE: Required as we want only want to consider for conversion events that # occurred on or after the given start_date. @@ -37,6 +61,8 @@ experiments: - 20231205-variant-a - 20231205-variant-b - 20231205-variant-c + ended_at: 2024-01-04 + winner: 20231127-variant-c weights: - 70 - 10 diff --git a/db/migrate/20240103223627_add_clickbait_factor_to_articles.rb b/db/migrate/20240103223627_add_clickbait_factor_to_articles.rb new file mode 100644 index 000000000..ee43a1d0f --- /dev/null +++ b/db/migrate/20240103223627_add_clickbait_factor_to_articles.rb @@ -0,0 +1,6 @@ +class AddClickbaitFactorToArticles < ActiveRecord::Migration[7.0] + def change + # Clickbait score is a number between 0 and 1 that represents how clickbaity the article is — higher is more clickbaity + add_column :articles, :clickbait_score, :float, default: 0.0 + end +end diff --git a/db/schema.rb b/db/schema.rb index f3e715918..5308ba191 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_01_02_154708) do +ActiveRecord::Schema[7.0].define(version: 2024_01_03_223627) do # These are extensions that must be enabled in order to support this database enable_extension "citext" enable_extension "ltree" @@ -94,6 +94,7 @@ ActiveRecord::Schema[7.0].define(version: 2024_01_02_154708) do t.string "cached_user_name" t.string "cached_user_username" t.string "canonical_url" + t.float "clickbait_score", default: 0.0 t.bigint "co_author_ids", default: [], array: true t.bigint "collection_id" t.integer "comment_score", default: 0 diff --git a/spec/requests/api/v1/articles_spec.rb b/spec/requests/api/v1/articles_spec.rb index ba91165a8..abee27188 100644 --- a/spec/requests/api/v1/articles_spec.rb +++ b/spec/requests/api/v1/articles_spec.rb @@ -982,6 +982,21 @@ RSpec.describe "Api::V1::Articles" do expect(response).to have_http_status(:ok) end + it "lets a super admin update an article's clickbait_score" do + user.add_role(:super_admin) + article = create(:article, user: create(:user)) + params = { article: { title: "foobar", clickbait_score: 0.3 } }.to_json + put "/api/articles/#{article.id}", params: params, headers: auth_headers + expect(article.reload.clickbait_score).to eq(0.3) + end + + it "does not update clickbait_score for non super-admins" do + article = create(:article, user: create(:user)) + params = { article: { title: "foobar", clickbait_score: 0.3 } }.to_json + put "/api/articles/#{article.id}", params: params, headers: auth_headers + expect(article.reload.clickbait_score).not_to eq(0.3) + end + it "does not update title if only given a title because the article has a front matter" do put_article(title: Faker::Book.title) expect(response).to have_http_status(:ok)