maverick/docbrown
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add new simple a/b test for Digest subject and adjust config and design details (#20754)

Browse source 
* Adjust Digest Email design and configuration

* Adjust some tests

* Update tests

* More adjustments to markup and tests

* More adjustments to markup and tests

* Digest email feature flag

* Test title variants
This commit is contained in:
Ben Halpern 2024-03-11 13:02:19 -04:00 • committed by GitHub
parent 04fb62a6f5
commit 0d4f7c7e19
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 158 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
app/javascript/packs/billboard.js
View file

@ -26,8 +26,15 @@ async function generateBillboard(element) {
asyncUrl += `${asyncUrl.includes('?') ? '&' : '?'}cookies_allowed=true`;
}
if (asyncUrl) {
try {
// When context is digest we don't show this billboard
// This is a hardcoded feature which should become more dynamic later.
if (asyncUrl.includes('post_fixed_bottom') && currentParams.includes('context=digest')) {
return;
}
const response = await window.fetch(asyncUrl);
const htmlContent = await response.text();
const generatedElement = document.createElement('div');

24
app/mailers/digest_mailer.rb
View file

@ -1,4 +1,5 @@
class DigestMailer < ApplicationMailer
include FieldTest::Helpers
default from: -> { email_from(I18n.t("mailers.digest_mailer.from")) }
def digest_email
@ -18,10 +19,31 @@ class DigestMailer < ApplicationMailer
mail(to: @user.email, subject: subject)
end
def title_test_variant(user)
field_test(:digest_title_03_11, participant: user)
end
private
def generate_title
"#{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase} #{random_emoji}"
base = "#{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase} #{random_emoji}"
return base unless FeatureFlag.enabled?(:digest_subject_testing)
title_variant = title_test_variant(@user)
case title_variant
when "base"
base
when "base_with_no_emoji"
"#{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase}"
when "base_with_start_with_dev_digest"
"DEV Digest: #{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase} #{random_emoji}"
when "base_with_start_with_dev_digest_and_no_emoji"
"DEV Digest: #{adjusted_title(@articles.first)} + #{@articles.size - 1} #{email_end_phrase}"
when "just_first_title"
@articles.first.title
when "just_first_title_and_dev_digest"
"#{@articles.first.title} | DEV Digest"
end
end
def adjusted_title(article)

15
app/services/email_digest_article_collector.rb
View file

@ -9,13 +9,14 @@ class EmailDigestArticleCollector
def articles_to_send
# rubocop:disable Metrics/BlockLength
order = Arel.sql("((score * (feed_success_score + 0.1)) - clickbait_score) DESC")
instrument ARTICLES_TO_SEND, tags: { user_id: @user.id } do
return [] unless should_receive_email?
articles = if user_has_followings?
experience_level_rating = (@user.setting.experience_level || 5)
experience_level_rating_min = experience_level_rating - 3.6
experience_level_rating_max = experience_level_rating + 3.6
experience_level_rating_min = experience_level_rating - 4
experience_level_rating_max = experience_level_rating + 4
@user.followed_articles
.select(:title, :description, :path)
@ -23,10 +24,10 @@ class EmailDigestArticleCollector
.where("published_at > ?", cutoff_date)
.where(email_digest_eligible: true)
.not_authored_by(@user.id)
.where("score > ?", 12)
.where("score > ?", 8)
.where("experience_level_rating > ? AND experience_level_rating < ?",
experience_level_rating_min, experience_level_rating_max)
.order(score: :desc)
.order(order)
.limit(6)
else
Article.select(:title, :description, :path)
@ -35,8 +36,8 @@ class EmailDigestArticleCollector
.featured
.where(email_digest_eligible: true)
.not_authored_by(@user.id)
.where("score > ?", 25)
.order(score: :desc)
.where("score > ?", 15)
.order(order)
.limit(6)
end
@ -61,7 +62,7 @@ class EmailDigestArticleCollector
end
def cutoff_date
a_few_days_ago = 4.days.ago.utc
a_few_days_ago = 7.days.ago.utc
return a_few_days_ago unless last_email_sent
[a_few_days_ago, last_email_sent].max

6
app/views/layouts/mailer.html.erb
View file

@ -1,6 +1,6 @@
<html>
<body style='font-family: "HelveticaNeue-Light", "Helvetica Neue Light", "Helvetica Neue", Helvetica, Arial, sans-serif;'>
<table style="width:100%;max-width:700px;margin:auto;font-size:18px">
<body style='font-family: "HelveticaNeue-Light", "Helvetica Neue Light", "Helvetica Neue", Helvetica, Arial, sans-serif;background:#fcfcfc;'>
<table style="width:100%;max-width:700px;margin:20px auto;font-size:18px;border:solid 1px #e6e6e6;border-radius:12px;display:block;padding:2px 12px;text-decoration:none;background:white;">
<tr>
<td style="padding:4%;">
<%= yield %>
@ -8,7 +8,7 @@
</tr>
<% if @user %>
<tr>
<td style="padding:3% 12% 2% 4%;font-size:18px;line-height:22px;color:#7d7d7d">
<td style="padding:1% 12% 2% 4%;16px;line-height:calc(20px + 0.2vw);color:#202121">
<%= signed_up_with(@user) %>
<br /><br />

114
app/views/mailers/digest_mailer/digest_email.html.erb
View file

@ -1,50 +1,68 @@
<h1 style="text-align: center;">
<%= community_name %> Digest
</h1>
<h4 style="text-align: center;">Recent posts you might find valuable based on your interests ❤️</h4>
<ul>
<% @articles.each do |article| %>
<li style="margin:18px auto">
<a href="<%= article_url(article) %>" style="font-weight: bold;color:#0045ff;font-size:calc(0.93em + 0.2vw);"><%= article.title.strip %></a> <%= truncate(article.description, length: 120) %>
</li>
<% end %>
<hr />
</ul>
<table width="100%" cellpadding="0" cellspacing="0">
<tr>
<td align="center">
<table style="width:100%;" cellpadding="0" cellspacing="0">
<!-- Header -->
<tr>
<td align="center" bgcolor="#FFFFFF" style="padding: 25px 2px 35px;">
<h1 style="margin: 0; font-size: 24px; color: #333;"> <%= community_name %> Digest </h1>
</td>
</tr>
<center style="margin-top:50px;">
<em>
<% if @user.setting.experience_level.nil? %>
<b>
You can now add your experience level to your account in order to receive more relevant content.
</b>
<br /><br />
Visit <b><a href="<%= app_url(user_settings_path(:customization)) %>" style="text-decoration: none;">your settings</a></b> to enter your experience level on a scale of 1-10.
<br /><br />
You can change it any time in the future.
<% elsif @user.following_users_count == 0 %>
<b>
<%= community_name %> Digest is a new periodic email featuring the best posts from our community of <%= community_members_label %>.
</b>
<br /><br />
Your digest is not yet customized. Follow <%= community_members_label %> on
<a href="<%= app_url %>" style="text-decoration: none;"><%= community_name %></a> that interest you in order to receive the most relevant posts in your inbox.
<% else %>
<% tip_rand = rand(5) %>
<% if tip_rand == 0 %>
Thanks for being a valued member of the community.
<% elsif tip_rand == 1 %>
Follow the authors you want to see more of!
<% elsif tip_rand == 2 %>
And there's so much more <%= community_name %> goodness to discover.
<% elsif tip_rand == 3 %>
You're a better <%= Settings::Community.member_label %> today than you were yesterday.
<% elsif tip_rand == 4 %>
<% if @user.articles_count > 0 %>
Can't wait to see <b>your</b> next <%= community_name %> post!
<% else %>
Can't wait to see <b>your</b> first <%= community_name %> post!
<!-- Subheader -->
<tr>
<td bgcolor="#F8F8F8" style="padding: 8px; text-align: center; color: #3a3c3d; font-size: 16px;border-radius: 12px;">
Recent community posts you might find valuable
</td>
</tr>
<!-- Articles List -->
<% @articles.each do |article| %>
<tr>
<td bgcolor="#FFFFFF" style="padding: 25px 0px 0px; text-align: left;">
<a href="<%= article_url(article) %>?context=digest" style="font-weight: bold; color:#0a0a0a; font-size: 16px;"><%= article.title.strip %></a>
<p style="margin-top: 5px; font-size: 14px; color: #808080;">
<%= truncate(article.description, length: 150) %>
</p>
</td>
</tr>
<% end %>
<% end %>
<% end %>
</em>
</center>
<% if ForemInstance.dev_to? %>
<tr>
<td bgcolor="#FFFFFF" style="padding: 30px 0 8px;">
<p style="margin: 0; font-weight: bold; font-size: 20px;">
Happy coding ❤️
</p>
</td>
</tr>
<% end %>
<tr>
<td bgcolor="#FFFFFF" style="padding: 20px 0;">
<hr style="border-top: 3px solid #0a0a0a;"/>
</td>
</tr>
<!-- Footer -->
<tr>
<td bgcolor="#FFFFFF" style="padding: 15px 0; font-size: 16px;">
<h2>How to make your Digest better</h2>
<% if @user.setting.experience_level.nil? %>
<p style="margin: 22px 0; font-size: 14px;">
Visit <b><a href="<%= app_url(user_settings_path(:customization)) %>" style="color: #0a0a0a; text-decoration: underline;">your settings</a></b> to enter your experience level to get more relevant posts for you.
</p>
<% end %>
<p style="margin: 22px 0 10px; font-size: 14px;">
Visit the <b><a href="<%= app_url(tags_path) %>" style="color: #0a0a0a; text-decoration: underline;">tags page</a></b> to choose which types of content will be featured.
</p>
</td>
</tr>
<tr>
<td bgcolor="#FFFFFF" style="padding: 10px 0;">
<hr style="border-top: 3px solid #0a0a0a;"/>
</td>
</tr>
</table>
</td>
</tr>
</table>

28
config/field_test.yml
View file

@ -27,6 +27,34 @@
################################################################################
################################################################################
experiments:
digest_title_03_11:
started_at: 2024-03-11
variants:
- base
- base_with_no_emoji
- base_with_start_with_dev_digest
- base_with_start_with_dev_digest_and_no_emoji
- just_first_title
- just_first_title_and_dev_digest
weights:
- 50
- 10
- 10
- 10
- 10
- 10
goals:
- user_creates_pageview
- user_creates_article_reaction
- user_creates_comment
- user_views_pages_on_at_least_two_different_days_within_a_week
- user_views_pages_on_at_least_four_different_days_within_a_week
- user_creates_article_reaction_on_four_different_days_within_a_week
- user_creates_comment_on_at_least_four_different_days_within_a_week
- user_views_pages_on_at_least_three_different_hours_within_a_day
- user_views_pages_on_at_least_four_different_hours_within_a_day
- user_views_pages_on_at_least_twelve_different_hours_within_five_days
- user_views_pages_on_at_least_nine_different_days_within_two_weeks
feed_style_20240220:
started_at: 2024-02-20
variants:

2
config/initializers/ahoy_email.rb
View file

@ -1,5 +1,5 @@
# enable tracking for open, click and UTM params
AhoyEmail.api = true
AhoyEmail.default_options[:click] = false
AhoyEmail.default_options[:click] = true
AhoyEmail.default_options[:utm_params] = false
AhoyEmail.default_options[:message] = true

4
spec/mailers/devise_mailer_spec.rb
View file

@ -40,7 +40,7 @@ RSpec.describe DeviseMailer do
end
it "renders proper URL" do
expect(email.body.to_s).to include("/users/confirmation?confirmation_token=faketoken")
expect(email.body.to_s).to include("confirmation_token%3Dfaketoken") # encoded URL
end
end
@ -52,7 +52,7 @@ RSpec.describe DeviseMailer do
end
it "renders proper URL" do
expect(email.body.to_s).to include("/users/confirmation?confirmation_token=faketoken")
expect(email.body.to_s).to include("confirmation_token%3Dfaketoken")
end
end
end

13
spec/mailers/digest_mailer_spec.rb
View file

@ -25,6 +25,19 @@ RSpec.describe DigestMailer do
expect(email["from"].value).to eq(expected_from)
end
it "works with all field_test variants", :aggregate_failures do
allow(FeatureFlag).to receive(:enabled?).with(:digest_subject_testing).and_return(true)
variants = %w[base base_with_no_emoji base_with_start_with_dev_digest
base_with_start_with_dev_digest_and_no_emoji just_first_title just_first_title_and_dev_digest]
variants.each do |variant|
allow(described_class).to receive(:title_test_variant)
.with(user).and_return(variant)
email = described_class.with(user: user, articles: [article]).digest_email
expect(email.subject).not_to be_nil
end
end
it "includes the correct X-SMTPAPI header for SendGrid", :aggregate_failures do
email = described_class.with(user: user, articles: [article]).digest_email.deliver_now

52
spec/mailers/notify_mailer_spec.rb
View file

@ -122,47 +122,9 @@ RSpec.describe NotifyMailer do
expect(email.to).to eq([user.email])
end
context "when rendering the HTML email for badge with credits" do
it "includes the listings URL" do
expect(email_with_credits.html_part.body).to include(
Rails.application.routes.url_helpers.listings_url(host: Settings::General.app_domain),
)
end
it "includes the about listings URL" do
expect(email_with_credits.html_part.body).to include(
Rails.application.routes.url_helpers.about_listings_url(host: Settings::General.app_domain),
)
end
it "includes number of credits" do
expect(email_with_credits.html_part.body).to include("7 new credits")
end
end
context "when rendering the text email for badge with credits" do
it "includes the listings URL" do
expect(email_with_credits.text_part.body).not_to include(
CGI.escape(
Rails.application.routes.url_helpers.listings_url(host: Settings::General.app_domain),
),
)
end
it "includes the about listings URL" do
expect(email_with_credits.text_part.body).not_to include(
CGI.escape(Rails.application.routes.url_helpers.about_listings_url(host: Settings::General.app_domain)),
)
end
it "includes number of credits" do
expect(email_with_credits.text_part.body).to include("7 new credits")
end
end
context "when rendering the HTML email for badge w/o credits" do
it "includes the user URL" do
expect(email.html_part.body).to include(URL.user(user))
expect(email.html_part.body).to include("%2F#{user.username}")
end
it "doesn't include the listings URL" do
@ -181,21 +143,21 @@ RSpec.describe NotifyMailer do
it "includes the rewarding_context_message in the email" do
expect(email.html_part.body).to include("Hello <a")
expect(email.html_part.body).to include(URL.url("/hey"))
expect(email.html_part.body).to include("%2Fhey")
end
it "does not include the nil rewarding_context_message in the email" do
allow(badge_achievement).to receive(:rewarding_context_message).and_return(nil)
expect(email.html_part.body).not_to include("Hello <a")
expect(email.html_part.body).not_to include(URL.url("/hey"))
expect(email.html_part.body).not_to include("%2Fhey")
end
it "does not include the empty rewarding_context_message in the email" do
allow(badge_achievement).to receive(:rewarding_context_message).and_return("")
expect(email.html_part.body).not_to include("Hello <a")
expect(email.html_part.body).not_to include(URL.url("/hey"))
expect(email.html_part.body).not_to include("%2Fhey")
end
end
@ -218,21 +180,21 @@ RSpec.describe NotifyMailer do
it "includes the rewarding_context_message in the email" do
expect(email.text_part.body).to include("Hello Yoho")
expect(email.text_part.body).not_to include(URL.url("/hey"))
expect(email.text_part.body).not_to include("%2Fhey")
end
it "does not include the nil rewarding_context_message in the email" do
allow(badge_achievement).to receive(:rewarding_context_message).and_return(nil)
expect(email.text_part.body).not_to include("Hello Yoho")
expect(email.text_part.body).not_to include(URL.url("/hey"))
expect(email.text_part.body).not_to include("%2Fhey")
end
it "does not include the empty rewarding_context_message in the email" do
allow(badge_achievement).to receive(:rewarding_context_message).and_return("")
expect(email.text_part.body).not_to include("Hello Yoho")
expect(email.text_part.body).not_to include(URL.url("/hey"))
expect(email.text_part.body).not_to include("%2Fhey")
end
end
end