From 83d37df4d79700d3daaad14df560f221689c8226 Mon Sep 17 00:00:00 2001 From: rhymes Date: Mon, 14 Sep 2020 15:38:53 +0200 Subject: [PATCH] [deploy] Cleanup articles with duplicate user_id, title and body_markdown (#10273) * Add script to delete draft articles * Add script to de-duplicate published articles * Ignore NotificationSubscription, Notification and RatingVote --- ...h_duplicate_user_id_title_body_markdown.rb | 32 ++++++ ...h_duplicate_user_id_title_body_markdown.rb | 103 ++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 lib/data_update_scripts/20200910135958_remove_draft_articles_with_duplicate_user_id_title_body_markdown.rb create mode 100644 lib/data_update_scripts/20200910140109_cleanup_published_articles_with_duplicate_user_id_title_body_markdown.rb diff --git a/lib/data_update_scripts/20200910135958_remove_draft_articles_with_duplicate_user_id_title_body_markdown.rb b/lib/data_update_scripts/20200910135958_remove_draft_articles_with_duplicate_user_id_title_body_markdown.rb new file mode 100644 index 000000000..7a60b9fdc --- /dev/null +++ b/lib/data_update_scripts/20200910135958_remove_draft_articles_with_duplicate_user_id_title_body_markdown.rb @@ -0,0 +1,32 @@ +module DataUpdateScripts + class RemoveDraftArticlesWithDuplicateUserIdTitleBodyMarkdown + def run + # Currently there are duplicate draft articles in the DB with the same body_markdown, title, user_id combination + + # This statement deletes all draft articles in excess found to be duplicate over those three columns + ActiveRecord::Base.connection.execute( + <<~SQL.squish, + WITH duplicates_draft_articles AS + (SELECT id + FROM + (SELECT id, + published, + body_markdown, + title, + user_id, + ROW_NUMBER() OVER(PARTITION BY user_id, title, body_markdown + ORDER BY id ASC) AS row_number + FROM articles) duplicates + WHERE duplicates.row_number > 1 + AND published = 'f' -- drafts + ) + DELETE + FROM articles + WHERE id IN + (SELECT id + FROM duplicates_draft_articles) RETURNING id; + SQL + ) + end + end +end diff --git a/lib/data_update_scripts/20200910140109_cleanup_published_articles_with_duplicate_user_id_title_body_markdown.rb b/lib/data_update_scripts/20200910140109_cleanup_published_articles_with_duplicate_user_id_title_body_markdown.rb new file mode 100644 index 000000000..57a93f969 --- /dev/null +++ b/lib/data_update_scripts/20200910140109_cleanup_published_articles_with_duplicate_user_id_title_body_markdown.rb @@ -0,0 +1,103 @@ +module DataUpdateScripts + class CleanupPublishedArticlesWithDuplicateUserIdTitleBodyMarkdown + def run + # load all published duplicates + rows = ActiveRecord::Base.connection.execute( + <<~SQL.squish, + SELECT * + FROM + (SELECT *, + ROW_NUMBER() OVER(PARTITION BY user_id, title, body_markdown) AS row_number, + COUNT(*) OVER(PARTITION BY user_id, title, body_markdown) AS num_rows_in_partition + FROM articles) duplicates + WHERE published = 't' + AND num_rows_in_partition > 1 -- we select all rows in each partition with duplicates + SQL + ) + + articles_to_delete_ids = [] + ActiveRecord::Base.transaction do + # now that we have all rows in the partition, we group them by the partition columns + # so we can have small groups referring to the same article and graft data of surplus articles + # onto the first of the partition (it doesn't really matter which one we're going to use as the "mold") + groups = rows.group_by { |row| [row["user_id"], row["title"], row["body_markdown"]] } + + # for each group we select the first, and then graft the data of the other ones on to it + + groups.each_value do |group| + article_to_keep = group.first + articles_to_graft = group[1..] + + article = Article.find(article_to_keep["id"]) + + graft_articles(article, articles_to_graft) + + # save all the magic + article.save + article.index_to_elasticsearch_inline + + articles_to_delete_ids += articles_to_graft.map { |a| a["id"] } + end + + # we shouldn't need to call Articles::Destroy as all the data has been moved over by now + Article.where(id: articles_to_delete_ids).destroy_all + end + + return if articles_to_delete_ids.blank? + + # Store deleted IDs temporarily in Redis for safe keeping + Rails.cache.write( + "DataUpdateScripts::CleanupPublishedArticlesWithDuplicateUserIdTitleBodyMarkdown", + articles_to_delete_ids, + expires_in: 2.weeks, + ) + end + + private + + # Main goal here is copy and merge everything from the duplicates to the main one + # NOTE: I'm intentionally ignore the case that two duplicates belong to different orgs + # or different collections as it's highly unlikely + def graft_articles(article_to_keep, articles_to_graft) + article_id = article_to_keep.id + articles_to_graft_ids = articles_to_graft.map { |a| a["id"] } + + # NotificationSubscription, Notification and RatingVote rows will be removed + # Poll is ignored because it's related to the liquid tag inside the article, also user's can't use polls + # TagAdjustment is ignored as there's likely no reason for article to have an adjustment moved over + models_with_a_direct_relation = [ + BufferUpdate, + HtmlVariantSuccess, + HtmlVariantSuccess, + HtmlVariantTrial, + PageView, + ] + models_with_a_direct_relation.each do |model_class| + model_class.where(article_id: articles_to_graft_ids).update_all(article_id: article_id) + end + + Comment + .where(commentable_type: "Article", commentable_id: articles_to_graft_ids) + .update_all(commentable_id: article_id) + + ProfilePin + .where(pinnable_type: "Article", pinnable_id: articles_to_graft_ids) + .update_all(pinnable_id: article_id) + + # we add tags that are not already present in the article from the others + # we don't really need to graft these as the tag objects belonging to the soon to be + # destroyed copies will be removed when we destroy the articles + tag_ids = ActsAsTaggableOn::Tagging + .where(taggable_type: "Article", taggable_id: articles_to_graft_ids) + .pluck(:tag_id) + + tags = Tag.where(id: tag_ids).pluck(:id, :name) + + # if the union of the two sets equals the article tags, then there's nothing to copy over + tag_names_to_maybe_graft = tags.map(&:second) + return if article_to_keep.tag_list == article_to_keep.tag_list | tag_names_to_maybe_graft + + article_to_keep.tag_list.add(*(article_to_keep.tag_list - tag_names_to_maybe_graft)) + end + end +end