From 9db13cbc4ca09076a8ce02bdffcfc32d229a98db Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Tue, 25 May 2021 14:25:45 -0400 Subject: [PATCH] Run user setting DUS per-user via Sidekiq (#13848) * Run user setting DUS per-user via Sidekiq Doing it as a single query trips the query timeout beyond a certain quantity of users. * Run DataUpdateScripts::MigrateRelevantFieldsFromUsersToUsersNotificationSettings asynchronously --- ...grate_user_notification_settings_worker.rb | 51 ++++++++++++ app/workers/migrate_user_settings_worker.rb | 82 +++++++++++++++++++ ...ant_fields_from_users_to_users_settings.rb | 81 +----------------- ...om_users_to_users_notification_settings.rb | 50 +---------- ...ers_to_users_notification_settings_spec.rb | 2 +- ...ields_from_users_to_users_settings_spec.rb | 2 +- spec/rails_helper.rb | 13 +++ 7 files changed, 152 insertions(+), 129 deletions(-) create mode 100644 app/workers/migrate_user_notification_settings_worker.rb create mode 100644 app/workers/migrate_user_settings_worker.rb diff --git a/app/workers/migrate_user_notification_settings_worker.rb b/app/workers/migrate_user_notification_settings_worker.rb new file mode 100644 index 000000000..05131e304 --- /dev/null +++ b/app/workers/migrate_user_notification_settings_worker.rb @@ -0,0 +1,51 @@ +class MigrateUserNotificationSettingsWorker + include Sidekiq::Worker + + SQL = <<~SQL.freeze + WITH notification_settings_data AS ( + SELECT + users.id AS user_id, + COALESCE(email_badge_notifications, true), + COALESCE(email_comment_notifications, true), + COALESCE(email_community_mod_newsletter, false), + COALESCE(email_connect_messages, true), + COALESCE(email_digest_periodic, false), + COALESCE(email_follower_notifications, true), + COALESCE(email_membership_newsletter, false), + COALESCE(email_mention_notifications, true), + COALESCE(email_newsletter, false), + COALESCE(email_tag_mod_newsletter, false), + COALESCE(email_unread_notifications, true), + COALESCE(mobile_comment_notifications, true), + COALESCE(mod_roundrobin_notifications, true), + COALESCE(reaction_notifications, true), + COALESCE(welcome_notifications, true), + NOW(), + NOW() + FROM users + ) + INSERT INTO users_notification_settings (user_id, email_badge_notifications, email_comment_notifications, email_community_mod_newsletter, email_connect_messages, email_digest_periodic, email_follower_notifications, email_membership_newsletter, email_mention_notifications, email_newsletter, email_tag_mod_newsletter, email_unread_notifications, mobile_comment_notifications, mod_roundrobin_notifications, reaction_notifications, welcome_notifications, created_at, updated_at) + (SELECT * FROM notification_settings_data WHERE user_id = $1) + ON CONFLICT (user_id) DO UPDATE + SET email_badge_notifications = EXCLUDED.email_badge_notifications, + email_comment_notifications = EXCLUDED.email_comment_notifications, + email_community_mod_newsletter = EXCLUDED.email_community_mod_newsletter, + email_connect_messages = EXCLUDED.email_connect_messages, + email_digest_periodic = EXCLUDED.email_digest_periodic, + email_follower_notifications = EXCLUDED.email_follower_notifications, + email_membership_newsletter = EXCLUDED.email_membership_newsletter, + email_mention_notifications = EXCLUDED.email_mention_notifications, + email_newsletter = EXCLUDED.email_newsletter, + email_tag_mod_newsletter = EXCLUDED.email_tag_mod_newsletter, + email_unread_notifications = EXCLUDED.email_unread_notifications, + mobile_comment_notifications = EXCLUDED.mobile_comment_notifications, + mod_roundrobin_notifications = EXCLUDED.mod_roundrobin_notifications, + reaction_notifications = EXCLUDED.reaction_notifications, + welcome_notifications = EXCLUDED.welcome_notifications, + updated_at = NOW() + SQL + + def perform(user_id) + Users::Setting.connection.exec_insert SQL, nil, [user_id] + end +end diff --git a/app/workers/migrate_user_settings_worker.rb b/app/workers/migrate_user_settings_worker.rb new file mode 100644 index 000000000..040b943c3 --- /dev/null +++ b/app/workers/migrate_user_settings_worker.rb @@ -0,0 +1,82 @@ +class MigrateUserSettingsWorker + include Sidekiq::Worker + + SQL = <<~SQL.freeze + WITH settings_data AS ( + SELECT + data ->> 'brand_color1' AS brand_color1, + data ->> 'brand_color2' AS brand_color2, + CASE WHEN config_font='default' THEN 0 + WHEN config_font='comic_sans' THEN 1 + WHEN config_font='monospace' THEN 2 + WHEN config_font='open_dyslexic' THEN 3 + WHEN config_font='sans_serif' THEN 4 + WHEN config_font='serif' THEN 5 + ELSE 0 + END + config_font, + CASE WHEN config_navbar='default_navbar' THEN 0 + WHEN config_navbar='static_navbar' THEN 1 + ELSE 0 + END + config_navbar, + CASE WHEN config_theme='default_theme' THEN 0 + WHEN config_theme='minimal_light_theme' THEN 1 + WHEN config_theme='night_theme' THEN 2 + WHEN config_theme='pink_theme' THEN 3 + WHEN config_theme='ten_x_hacker_theme' THEN 4 + ELSE 0 + END + config_theme, + COALESCE(display_announcements, true), + COALESCE((data ->> 'display_email_on_profile')::boolean, false) as display_email_on_profile, + COALESCE(display_sponsors, true), + CASE WHEN editor_version='v2' THEN 0 + WHEN editor_version='v1' THEN 1 + ELSE 0 + END + editor_version, + experience_level, + COALESCE(feed_mark_canonical, false), + COALESCE(feed_referential_link, true), + feed_url, + inbox_guidelines, + CASE WHEN inbox_type='private' THEN 0 + WHEN inbox_type='open' THEN 1 + ELSE 0 + END + inbox_type, + COALESCE(permit_adjacent_sponsors, true), + users.id AS user_id, + NOW(), + NOW() + FROM users + JOIN profiles + ON profiles.user_id = users.id + ) + INSERT INTO users_settings (brand_color1, brand_color2, config_font, config_navbar, config_theme, display_announcements, display_email_on_profile, display_sponsors, editor_version, experience_level, feed_mark_canonical, feed_referential_link, feed_url, inbox_guidelines, inbox_type, permit_adjacent_sponsors, user_id, created_at, updated_at) + (SELECT * FROM settings_data WHERE user_id = $1) + ON CONFLICT (user_id) DO UPDATE + SET brand_color1 = EXCLUDED.brand_color1, + brand_color2 = EXCLUDED.brand_color2, + config_font = EXCLUDED.config_font, + config_navbar = EXCLUDED.config_navbar, + config_theme = EXCLUDED.config_theme, + display_announcements = EXCLUDED.display_announcements, + display_email_on_profile = EXCLUDED.display_email_on_profile, + display_sponsors = EXCLUDED.display_sponsors, + editor_version = EXCLUDED.editor_version, + experience_level = EXCLUDED.experience_level, + feed_mark_canonical = EXCLUDED.feed_mark_canonical, + feed_referential_link = EXCLUDED.feed_referential_link, + feed_url = EXCLUDED.feed_url, + inbox_guidelines = EXCLUDED.inbox_guidelines, + inbox_type = EXCLUDED.inbox_type, + permit_adjacent_sponsors = EXCLUDED.permit_adjacent_sponsors, + updated_at = NOW() + SQL + + def perform(user_id) + Users::Setting.connection.exec_insert SQL, nil, [user_id] + end +end diff --git a/lib/data_update_scripts/20210423155327_migrate_relevant_fields_from_users_to_users_settings.rb b/lib/data_update_scripts/20210423155327_migrate_relevant_fields_from_users_to_users_settings.rb index fbca81fb8..5fd1886ce 100644 --- a/lib/data_update_scripts/20210423155327_migrate_relevant_fields_from_users_to_users_settings.rb +++ b/lib/data_update_scripts/20210423155327_migrate_relevant_fields_from_users_to_users_settings.rb @@ -1,86 +1,9 @@ module DataUpdateScripts class MigrateRelevantFieldsFromUsersToUsersSettings def run - # rubocop:disable Metrics/BlockLength(RuboCop) - ActiveRecord::Base.transaction do - ActiveRecord::Base.connection.execute( - <<~SQL.squish, - WITH settings_data AS ( - SELECT - data ->> 'brand_color1' AS brand_color1, - data ->> 'brand_color2' AS brand_color2, - CASE WHEN config_font='default' THEN 0 - WHEN config_font='comic_sans' THEN 1 - WHEN config_font='monospace' THEN 2 - WHEN config_font='open_dyslexic' THEN 3 - WHEN config_font='sans_serif' THEN 4 - WHEN config_font='serif' THEN 5 - ELSE 0 - END - config_font, - CASE WHEN config_navbar='default_navbar' THEN 0 - WHEN config_navbar='static_navbar' THEN 1 - ELSE 0 - END - config_navbar, - CASE WHEN config_theme='default_theme' THEN 0 - WHEN config_theme='minimal_light_theme' THEN 1 - WHEN config_theme='night_theme' THEN 2 - WHEN config_theme='pink_theme' THEN 3 - WHEN config_theme='ten_x_hacker_theme' THEN 4 - ELSE 0 - END - config_theme, - COALESCE(display_announcements, true), - COALESCE((data ->> 'display_email_on_profile')::boolean, false) as display_email_on_profile, - COALESCE(display_sponsors, true), - CASE WHEN editor_version='v2' THEN 0 - WHEN editor_version='v1' THEN 1 - ELSE 0 - END - editor_version, - experience_level, - COALESCE(feed_mark_canonical, false), - COALESCE(feed_referential_link, true), - feed_url, - inbox_guidelines, - CASE WHEN inbox_type='private' THEN 0 - WHEN inbox_type='open' THEN 1 - ELSE 0 - END - inbox_type, - COALESCE(permit_adjacent_sponsors, true), - users.id AS user_id, - NOW(), - NOW() - FROM users - JOIN profiles - ON profiles.user_id = users.id - ) - INSERT INTO users_settings (brand_color1, brand_color2, config_font, config_navbar, config_theme, display_announcements, display_email_on_profile, display_sponsors, editor_version, experience_level, feed_mark_canonical, feed_referential_link, feed_url, inbox_guidelines, inbox_type, permit_adjacent_sponsors, user_id, created_at, updated_at) - SELECT * FROM settings_data - ON CONFLICT (user_id) DO UPDATE - SET brand_color1 = EXCLUDED.brand_color1, - brand_color2 = EXCLUDED.brand_color2, - config_font = EXCLUDED.config_font, - config_navbar = EXCLUDED.config_navbar, - config_theme = EXCLUDED.config_theme, - display_announcements = EXCLUDED.display_announcements, - display_email_on_profile = EXCLUDED.display_email_on_profile, - display_sponsors = EXCLUDED.display_sponsors, - editor_version = EXCLUDED.editor_version, - experience_level = EXCLUDED.experience_level, - feed_mark_canonical = EXCLUDED.feed_mark_canonical, - feed_referential_link = EXCLUDED.feed_referential_link, - feed_url = EXCLUDED.feed_url, - inbox_guidelines = EXCLUDED.inbox_guidelines, - inbox_type = EXCLUDED.inbox_type, - permit_adjacent_sponsors = EXCLUDED.permit_adjacent_sponsors, - updated_at = NOW(); - SQL - ) + User.ids.each do |id| + MigrateUserSettingsWorker.perform_async id end - # rubocop:enable Metrics/BlockLength(RuboCop) end end end diff --git a/lib/data_update_scripts/20210503174302_migrate_relevant_fields_from_users_to_users_notification_settings.rb b/lib/data_update_scripts/20210503174302_migrate_relevant_fields_from_users_to_users_notification_settings.rb index 12cb3c0ce..adb36ba67 100644 --- a/lib/data_update_scripts/20210503174302_migrate_relevant_fields_from_users_to_users_notification_settings.rb +++ b/lib/data_update_scripts/20210503174302_migrate_relevant_fields_from_users_to_users_notification_settings.rb @@ -1,55 +1,9 @@ module DataUpdateScripts class MigrateRelevantFieldsFromUsersToUsersNotificationSettings def run - # rubocop:disable Metrics/BlockLength(RuboCop) - ActiveRecord::Base.transaction do - ActiveRecord::Base.connection.execute( - <<~SQL.squish, - WITH notification_settings_data AS ( - SELECT - users.id AS user_id, - COALESCE(email_badge_notifications, true), - COALESCE(email_comment_notifications, true), - COALESCE(email_community_mod_newsletter, false), - COALESCE(email_connect_messages, true), - COALESCE(email_digest_periodic, false), - COALESCE(email_follower_notifications, true), - COALESCE(email_membership_newsletter, false), - COALESCE(email_mention_notifications, true), - COALESCE(email_newsletter, false), - COALESCE(email_tag_mod_newsletter, false), - COALESCE(email_unread_notifications, true), - COALESCE(mobile_comment_notifications, true), - COALESCE(mod_roundrobin_notifications, true), - COALESCE(reaction_notifications, true), - COALESCE(welcome_notifications, true), - NOW(), - NOW() - FROM users - ) - INSERT INTO users_notification_settings (user_id, email_badge_notifications, email_comment_notifications, email_community_mod_newsletter, email_connect_messages, email_digest_periodic, email_follower_notifications, email_membership_newsletter, email_mention_notifications, email_newsletter, email_tag_mod_newsletter, email_unread_notifications, mobile_comment_notifications, mod_roundrobin_notifications, reaction_notifications, welcome_notifications, created_at, updated_at) - SELECT * FROM notification_settings_data - ON CONFLICT (user_id) DO UPDATE - SET email_badge_notifications = EXCLUDED.email_badge_notifications, - email_comment_notifications = EXCLUDED.email_comment_notifications, - email_community_mod_newsletter = EXCLUDED.email_community_mod_newsletter, - email_connect_messages = EXCLUDED.email_connect_messages, - email_digest_periodic = EXCLUDED.email_digest_periodic, - email_follower_notifications = EXCLUDED.email_follower_notifications, - email_membership_newsletter = EXCLUDED.email_membership_newsletter, - email_mention_notifications = EXCLUDED.email_mention_notifications, - email_newsletter = EXCLUDED.email_newsletter, - email_tag_mod_newsletter = EXCLUDED.email_tag_mod_newsletter, - email_unread_notifications = EXCLUDED.email_unread_notifications, - mobile_comment_notifications = EXCLUDED.mobile_comment_notifications, - mod_roundrobin_notifications = EXCLUDED.mod_roundrobin_notifications, - reaction_notifications = EXCLUDED.reaction_notifications, - welcome_notifications = EXCLUDED.welcome_notifications, - updated_at = NOW(); - SQL - ) + User.ids.each do |id| + MigrateUserNotificationSettingsWorker.perform_async id end - # rubocop:enable Metrics/BlockLength(RuboCop) end end end diff --git a/spec/lib/data_update_scripts/migrate_relevant_fields_from_users_to_users_notification_settings_spec.rb b/spec/lib/data_update_scripts/migrate_relevant_fields_from_users_to_users_notification_settings_spec.rb index c51668508..8e90b3747 100644 --- a/spec/lib/data_update_scripts/migrate_relevant_fields_from_users_to_users_notification_settings_spec.rb +++ b/spec/lib/data_update_scripts/migrate_relevant_fields_from_users_to_users_notification_settings_spec.rb @@ -3,7 +3,7 @@ require Rails.root.join( "lib/data_update_scripts/20210503174302_migrate_relevant_fields_from_users_to_users_notification_settings.rb", ) -describe DataUpdateScripts::MigrateRelevantFieldsFromUsersToUsersNotificationSettings do +describe DataUpdateScripts::MigrateRelevantFieldsFromUsersToUsersNotificationSettings, sidekiq: :inline do let(:users_notification_setting) { Users::NotificationSetting.last } before do diff --git a/spec/lib/data_update_scripts/migrate_relevant_fields_from_users_to_users_settings_spec.rb b/spec/lib/data_update_scripts/migrate_relevant_fields_from_users_to_users_settings_spec.rb index 4e68cea7f..5a97d6a83 100644 --- a/spec/lib/data_update_scripts/migrate_relevant_fields_from_users_to_users_settings_spec.rb +++ b/spec/lib/data_update_scripts/migrate_relevant_fields_from_users_to_users_settings_spec.rb @@ -3,7 +3,7 @@ require Rails.root.join( "lib/data_update_scripts/20210423155327_migrate_relevant_fields_from_users_to_users_settings.rb", ) -describe DataUpdateScripts::MigrateRelevantFieldsFromUsersToUsersSettings do +describe DataUpdateScripts::MigrateRelevantFieldsFromUsersToUsersSettings, sidekiq: :inline do let(:users_setting) { Users::Setting.last } before do diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index e41cecb56..212fc6d20 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -85,6 +85,19 @@ RSpec.configure do |config| Warden::Manager._on_request.clear end + config.around do |example| + case example.metadata[:sidekiq] + when :inline + Sidekiq::Testing.inline! { example.run } + when :fake + Sidekiq::Testing.fake! { example.run } + when :disable + Sidekiq::Testing.disable! { example.run } + else + example.run + end + end + config.before(:suite) do # Set the TZ ENV variable with the current random timezone from zonebie # which we can then use to properly set the browser time for Capybara specs