diff --git a/lib/data_update_scripts/20200904132553_remove_draft_articles_with_duplicate_feed_source_url.rb b/lib/data_update_scripts/20200904132553_remove_draft_articles_with_duplicate_feed_source_url.rb new file mode 100644 index 000000000..8cd9b54e2 --- /dev/null +++ b/lib/data_update_scripts/20200904132553_remove_draft_articles_with_duplicate_feed_source_url.rb @@ -0,0 +1,80 @@ +module DataUpdateScripts + class RemoveDraftArticlesWithDuplicateFeedSourceUrl + def run + # Currently there are duplicate draft articles in the DB with the same feed_source_url. + + # This statement deletes all draft articles in excess found to be duplicate over feed_source_url, + # excluding those whose body_markdown is different from the other duplicate occurrences + result_same_body = ActiveRecord::Base.connection.execute( + <<~SQL, + WITH duplicates_draft_articles AS + (SELECT id + FROM + (SELECT id, + published, + body_markdown, + LAG(body_markdown, 1) OVER(PARTITION BY feed_source_url + ORDER BY id ASC) AS previous_body_markdown, + ROW_NUMBER() OVER(PARTITION BY feed_source_url + ORDER BY id ASC) AS row_number + FROM articles + WHERE feed_source_url IS NOT NULL ) duplicates + WHERE duplicates.row_number > 1 + AND published = 'f' -- drafts + AND body_markdown = previous_body_markdown -- with the same body + ) + DELETE + FROM articles + WHERE id IN (SELECT id FROM duplicates_draft_articles) RETURNING id; + SQL + ) + + # Now that all duplicates with the same body are gone, we need to deal with duplicate feed source URLs + # with different bodies. + # We thus select the oldest for removal preserving the most recent one + result_different_bodies = ActiveRecord::Base.connection.execute( + <<~SQL, + WITH duplicates_draft_articles AS + (SELECT id + FROM + (SELECT id, + published, + body_markdown, + LAG(body_markdown, 1) OVER(PARTITION BY feed_source_url + ORDER BY created_at DESC) AS previous_body_markdown, + ROW_NUMBER() OVER(PARTITION BY feed_source_url + ORDER BY created_at DESC) AS row_number + FROM articles + WHERE feed_source_url IS NOT NULL ) duplicates + WHERE duplicates.row_number > 1 + AND published = 'f' -- drafts + AND body_markdown != previous_body_markdown -- with different bodies + ) + DELETE + FROM articles + WHERE id IN (SELECT id FROM duplicates_draft_articles) RETURNING id; + SQL + ) + + result_same_body_ids = result_same_body.map { |row| row["id"] } + result_different_bodies_ids = result_different_bodies.map { |row| row["id"] } + + # Remove all articles from Elasticsearch + (result_same_body_ids + result_different_bodies_ids).each do |id| + Search::RemoveFromIndexWorker.perform_in(5.seconds, "Article", id) + end + + # Store deleted IDs temporarily in Redis for safe keeping + Rails.cache.write( + "DataUpdateScripts::RemoveDraftArticlesWithDuplicateFeedSourceUrl::SameBody", + result_same_body_ids, + expires_in: 2.weeks, + ) + Rails.cache.write( + "DataUpdateScripts::RemoveDraftArticlesWithDuplicateFeedSourceUrl::DifferentBodies", + result_same_body_ids, + expires_in: 2.weeks, + ) + end + end +end diff --git a/lib/data_update_scripts/20200904141057_cleanup_articles_with_invalid_feed_source_url.rb b/lib/data_update_scripts/20200904141057_cleanup_articles_with_invalid_feed_source_url.rb new file mode 100644 index 000000000..1c202bb4e --- /dev/null +++ b/lib/data_update_scripts/20200904141057_cleanup_articles_with_invalid_feed_source_url.rb @@ -0,0 +1,16 @@ +module DataUpdateScripts + class CleanupArticlesWithInvalidFeedSourceUrl + def run + # We need this to intercept those strings that would be valid URLs if the scheme were added, + # so I slightly modified the Ruby URL regexp from https://urlregex.com/ to avoid checking for the scheme + almost_url_regexp = %r{\A(?:\S+(?::\S*)?@)?(?:(?!10(?:\.\d{1,3}){3})(?!127(?:\.\d{1,3}){3})(?!169\.254(?:\.\d{1,3}){2})(?!192\.168(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:/[^\s]*)?\z}i # rubocop:disable Layout/LineLength + + Article.where.not(feed_source_url: nil).where("feed_source_url NOT ILIKE ?", "http%").find_each do |article| + fixed_feed_source_url = article.canonical_url.presence || + (almost_url_regexp.match?(article.feed_source_url) ? "https://#{article.feed_source_url}" : nil) + + article.update_columns(feed_source_url: fixed_feed_source_url) + end + end + end +end diff --git a/spec/lib/data_update_scripts/cleanup_articles_with_invalid_feed_source_url_spec.rb b/spec/lib/data_update_scripts/cleanup_articles_with_invalid_feed_source_url_spec.rb new file mode 100644 index 000000000..9ca97397d --- /dev/null +++ b/spec/lib/data_update_scripts/cleanup_articles_with_invalid_feed_source_url_spec.rb @@ -0,0 +1,41 @@ +require "rails_helper" +require Rails.root.join( + "lib/data_update_scripts/20200904141057_cleanup_articles_with_invalid_feed_source_url.rb", +) + +describe DataUpdateScripts::CleanupArticlesWithInvalidFeedSourceUrl do + let(:article) { create(:article) } + + it "sets feed_source_url to canonical_url if the latter is present" do + canonical_url = "https://example.com/article" + article.update_columns(feed_source_url: "", canonical_url: canonical_url) + + described_class.new.run + + expect(article.reload.feed_source_url).to eq(canonical_url) + end + + it "sets empty string feed source url to nil" do + article.update_columns(feed_source_url: "") + + described_class.new.run + + expect(article.reload.feed_source_url).to be(nil) + end + + it "sets totally invalid url to nil" do + article.update_columns(feed_source_url: "not a url") + + described_class.new.run + + expect(article.reload.feed_source_url).to be(nil) + end + + it "sets an 'almost URL' to a https URL" do + article.update_columns(feed_source_url: "dev.to") + + described_class.new.run + + expect(article.reload.feed_source_url).to eq("https://dev.to") + end +end