From 0d4f7c7e19eddece9e7feb2901adbb7ebe472d5d Mon Sep 17 00:00:00 2001 From: Ben Halpern Date: Mon, 11 Mar 2024 13:02:19 -0400 Subject: [PATCH] Add new simple a/b test for Digest subject and adjust config and design details (#20754) * Adjust Digest Email design and configuration * Adjust some tests * Update tests * More adjustments to markup and tests * More adjustments to markup and tests * Digest email feature flag * Test title variants --- app/javascript/packs/billboard.js | 7 ++ app/mailers/digest_mailer.rb | 24 +++- .../email_digest_article_collector.rb | 15 +-- app/views/layouts/mailer.html.erb | 6 +- .../digest_mailer/digest_email.html.erb | 114 ++++++++++-------- config/field_test.yml | 28 +++++ config/initializers/ahoy_email.rb | 2 +- spec/mailers/devise_mailer_spec.rb | 4 +- spec/mailers/digest_mailer_spec.rb | 13 ++ spec/mailers/notify_mailer_spec.rb | 52 ++------ 10 files changed, 158 insertions(+), 107 deletions(-) diff --git a/app/javascript/packs/billboard.js b/app/javascript/packs/billboard.js index 24f63cb2f..92d028e2d 100644 --- a/app/javascript/packs/billboard.js +++ b/app/javascript/packs/billboard.js @@ -26,8 +26,15 @@ async function generateBillboard(element) { asyncUrl += `${asyncUrl.includes('?') ? '&' : '?'}cookies_allowed=true`; } + if (asyncUrl) { try { + // When context is digest we don't show this billboard + // This is a hardcoded feature which should become more dynamic later. + if (asyncUrl.includes('post_fixed_bottom') && currentParams.includes('context=digest')) { + return; + } + const response = await window.fetch(asyncUrl); const htmlContent = await response.text(); const generatedElement = document.createElement('div'); diff --git a/app/mailers/digest_mailer.rb b/app/mailers/digest_mailer.rb index f73035553..c61ae3ff2 100644 --- a/app/mailers/digest_mailer.rb +++ b/app/mailers/digest_mailer.rb @@ -1,4 +1,5 @@ class DigestMailer < ApplicationMailer + include FieldTest::Helpers default from: -> { email_from(I18n.t("mailers.digest_mailer.from")) } def digest_email @@ -18,10 +19,31 @@ class DigestMailer < ApplicationMailer mail(to: @user.email, subject: subject) end + def title_test_variant(user) + field_test(:digest_title_03_11, participant: user) + end + private def generate_title - "#{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase} #{random_emoji}" + base = "#{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase} #{random_emoji}" + return base unless FeatureFlag.enabled?(:digest_subject_testing) + + title_variant = title_test_variant(@user) + case title_variant + when "base" + base + when "base_with_no_emoji" + "#{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase}" + when "base_with_start_with_dev_digest" + "DEV Digest: #{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase} #{random_emoji}" + when "base_with_start_with_dev_digest_and_no_emoji" + "DEV Digest: #{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase}" + when "just_first_title" + @articles.first.title + when "just_first_title_and_dev_digest" + "#{@articles.first.title} | DEV Digest" + end end def adjusted_title(article) diff --git a/app/services/email_digest_article_collector.rb b/app/services/email_digest_article_collector.rb index 0266c0512..2514c20a0 100644 --- a/app/services/email_digest_article_collector.rb +++ b/app/services/email_digest_article_collector.rb @@ -9,13 +9,14 @@ class EmailDigestArticleCollector def articles_to_send # rubocop:disable Metrics/BlockLength + order = Arel.sql("((score * (feed_success_score + 0.1)) - clickbait_score) DESC") instrument ARTICLES_TO_SEND, tags: { user_id: @user.id } do return [] unless should_receive_email? articles = if user_has_followings? experience_level_rating = (@user.setting.experience_level || 5) - experience_level_rating_min = experience_level_rating - 3.6 - experience_level_rating_max = experience_level_rating + 3.6 + experience_level_rating_min = experience_level_rating - 4 + experience_level_rating_max = experience_level_rating + 4 @user.followed_articles .select(:title, :description, :path) @@ -23,10 +24,10 @@ class EmailDigestArticleCollector .where("published_at > ?", cutoff_date) .where(email_digest_eligible: true) .not_authored_by(@user.id) - .where("score > ?", 12) + .where("score > ?", 8) .where("experience_level_rating > ? AND experience_level_rating < ?", experience_level_rating_min, experience_level_rating_max) - .order(score: :desc) + .order(order) .limit(6) else Article.select(:title, :description, :path) @@ -35,8 +36,8 @@ class EmailDigestArticleCollector .featured .where(email_digest_eligible: true) .not_authored_by(@user.id) - .where("score > ?", 25) - .order(score: :desc) + .where("score > ?", 15) + .order(order) .limit(6) end @@ -61,7 +62,7 @@ class EmailDigestArticleCollector end def cutoff_date - a_few_days_ago = 4.days.ago.utc + a_few_days_ago = 7.days.ago.utc return a_few_days_ago unless last_email_sent [a_few_days_ago, last_email_sent].max diff --git a/app/views/layouts/mailer.html.erb b/app/views/layouts/mailer.html.erb index dbd2b3697..8ddd3a26f 100644 --- a/app/views/layouts/mailer.html.erb +++ b/app/views/layouts/mailer.html.erb @@ -1,6 +1,6 @@ - - + +
<% if @user %> -
<%= yield %> @@ -8,7 +8,7 @@
+ <%= signed_up_with(@user) %>

diff --git a/app/views/mailers/digest_mailer/digest_email.html.erb b/app/views/mailers/digest_mailer/digest_email.html.erb index 0bb17435a..ce100bc69 100644 --- a/app/views/mailers/digest_mailer/digest_email.html.erb +++ b/app/views/mailers/digest_mailer/digest_email.html.erb @@ -1,50 +1,68 @@ -

- <%= community_name %> Digest -

-

Recent posts you might find valuable based on your interests ❤️

-
    - <% @articles.each do |article| %> -
  • - <%= article.title.strip %> <%= truncate(article.description, length: 120) %> -
    • - <% end %> -
    -
+ + + + +
+ + + + + -
- - <% if @user.setting.experience_level.nil? %> - - You can now add your experience level to your account in order to receive more relevant content. - -

- Visit your settings to enter your experience level on a scale of 1-10. -

- You can change it any time in the future. - <% elsif @user.following_users_count == 0 %> - - <%= community_name %> Digest is a new periodic email featuring the best posts from our community of <%= community_members_label %>. - -

- Your digest is not yet customized. Follow <%= community_members_label %> on - <%= community_name %> that interest you in order to receive the most relevant posts in your inbox. - <% else %> - <% tip_rand = rand(5) %> - <% if tip_rand == 0 %> - Thanks for being a valued member of the community. - <% elsif tip_rand == 1 %> - Follow the authors you want to see more of! - <% elsif tip_rand == 2 %> - And there's so much more <%= community_name %> goodness to discover. - <% elsif tip_rand == 3 %> - You're a better <%= Settings::Community.member_label %> today than you were yesterday. - <% elsif tip_rand == 4 %> - <% if @user.articles_count > 0 %> - Can't wait to see your next <%= community_name %> post! - <% else %> - Can't wait to see your first <%= community_name %> post! + +
+ + + + + <% @articles.each do |article| %> + + + <% end %> - <% end %> - <% end %> - - + <% if ForemInstance.dev_to? %> + + + + <% end %> + + + + + + + + + + + + +
+

<%= community_name %> Digest

+
+ Recent community posts you might find valuable +
+ <%= article.title.strip %> +

+ <%= truncate(article.description, length: 150) %> +

+
+

+ Happy coding ❤️ +

+
+
+
+

How to make your Digest better

+ <% if @user.setting.experience_level.nil? %> +

+ Visit your settings to enter your experience level to get more relevant posts for you. +

+ <% end %> +

+ Visit the tags page to choose which types of content will be featured. +

+
+
+
+
\ No newline at end of file diff --git a/config/field_test.yml b/config/field_test.yml index 7dcd5a1de..092974890 100644 --- a/config/field_test.yml +++ b/config/field_test.yml @@ -27,6 +27,34 @@ ################################################################################ ################################################################################ experiments: + digest_title_03_11: + started_at: 2024-03-11 + variants: + - base + - base_with_no_emoji + - base_with_start_with_dev_digest + - base_with_start_with_dev_digest_and_no_emoji + - just_first_title + - just_first_title_and_dev_digest + weights: + - 50 + - 10 + - 10 + - 10 + - 10 + - 10 + goals: + - user_creates_pageview + - user_creates_article_reaction + - user_creates_comment + - user_views_pages_on_at_least_two_different_days_within_a_week + - user_views_pages_on_at_least_four_different_days_within_a_week + - user_creates_article_reaction_on_four_different_days_within_a_week + - user_creates_comment_on_at_least_four_different_days_within_a_week + - user_views_pages_on_at_least_three_different_hours_within_a_day + - user_views_pages_on_at_least_four_different_hours_within_a_day + - user_views_pages_on_at_least_twelve_different_hours_within_five_days + - user_views_pages_on_at_least_nine_different_days_within_two_weeks feed_style_20240220: started_at: 2024-02-20 variants: diff --git a/config/initializers/ahoy_email.rb b/config/initializers/ahoy_email.rb index 2ae16eeb8..5305d0935 100644 --- a/config/initializers/ahoy_email.rb +++ b/config/initializers/ahoy_email.rb @@ -1,5 +1,5 @@ # enable tracking for open, click and UTM params AhoyEmail.api = true -AhoyEmail.default_options[:click] = false +AhoyEmail.default_options[:click] = true AhoyEmail.default_options[:utm_params] = false AhoyEmail.default_options[:message] = true diff --git a/spec/mailers/devise_mailer_spec.rb b/spec/mailers/devise_mailer_spec.rb index 403102117..6dcc52af6 100644 --- a/spec/mailers/devise_mailer_spec.rb +++ b/spec/mailers/devise_mailer_spec.rb @@ -40,7 +40,7 @@ RSpec.describe DeviseMailer do end it "renders proper URL" do - expect(email.body.to_s).to include("/users/confirmation?confirmation_token=faketoken") + expect(email.body.to_s).to include("confirmation_token%3Dfaketoken") # encoded URL end end @@ -52,7 +52,7 @@ RSpec.describe DeviseMailer do end it "renders proper URL" do - expect(email.body.to_s).to include("/users/confirmation?confirmation_token=faketoken") + expect(email.body.to_s).to include("confirmation_token%3Dfaketoken") end end end diff --git a/spec/mailers/digest_mailer_spec.rb b/spec/mailers/digest_mailer_spec.rb index 61b2e7ebd..5d57739fe 100644 --- a/spec/mailers/digest_mailer_spec.rb +++ b/spec/mailers/digest_mailer_spec.rb @@ -25,6 +25,19 @@ RSpec.describe DigestMailer do expect(email["from"].value).to eq(expected_from) end + it "works with all field_test variants", :aggregate_failures do + allow(FeatureFlag).to receive(:enabled?).with(:digest_subject_testing).and_return(true) + variants = %w[base base_with_no_emoji base_with_start_with_dev_digest + base_with_start_with_dev_digest_and_no_emoji just_first_title just_first_title_and_dev_digest] + variants.each do |variant| + allow(described_class).to receive(:title_test_variant) + .with(user).and_return(variant) + email = described_class.with(user: user, articles: [article]).digest_email + + expect(email.subject).not_to be_nil + end + end + it "includes the correct X-SMTPAPI header for SendGrid", :aggregate_failures do email = described_class.with(user: user, articles: [article]).digest_email.deliver_now diff --git a/spec/mailers/notify_mailer_spec.rb b/spec/mailers/notify_mailer_spec.rb index 820d64aef..043b1e345 100644 --- a/spec/mailers/notify_mailer_spec.rb +++ b/spec/mailers/notify_mailer_spec.rb @@ -122,47 +122,9 @@ RSpec.describe NotifyMailer do expect(email.to).to eq([user.email]) end - context "when rendering the HTML email for badge with credits" do - it "includes the listings URL" do - expect(email_with_credits.html_part.body).to include( - Rails.application.routes.url_helpers.listings_url(host: Settings::General.app_domain), - ) - end - - it "includes the about listings URL" do - expect(email_with_credits.html_part.body).to include( - Rails.application.routes.url_helpers.about_listings_url(host: Settings::General.app_domain), - ) - end - - it "includes number of credits" do - expect(email_with_credits.html_part.body).to include("7 new credits") - end - end - - context "when rendering the text email for badge with credits" do - it "includes the listings URL" do - expect(email_with_credits.text_part.body).not_to include( - CGI.escape( - Rails.application.routes.url_helpers.listings_url(host: Settings::General.app_domain), - ), - ) - end - - it "includes the about listings URL" do - expect(email_with_credits.text_part.body).not_to include( - CGI.escape(Rails.application.routes.url_helpers.about_listings_url(host: Settings::General.app_domain)), - ) - end - - it "includes number of credits" do - expect(email_with_credits.text_part.body).to include("7 new credits") - end - end - context "when rendering the HTML email for badge w/o credits" do it "includes the user URL" do - expect(email.html_part.body).to include(URL.user(user)) + expect(email.html_part.body).to include("%2F#{user.username}") end it "doesn't include the listings URL" do @@ -181,21 +143,21 @@ RSpec.describe NotifyMailer do it "includes the rewarding_context_message in the email" do expect(email.html_part.body).to include("Hello