[deploy] Remove draft articles with duplicate feed source URL and cleanup invalid data: take 2 (#10239)
This commit is contained in:
rhymes
GitHub
parent
729a17a7a3
commit
b185dca976
3 changed files with 137 additions and 0 deletions
|
|
@ -0,0 +1,80 @@
|
|||
module DataUpdateScripts
|
||||
class RemoveDraftArticlesWithDuplicateFeedSourceUrl
|
||||
def run
|
||||
# Currently there are duplicate draft articles in the DB with the same feed_source_url.
|
||||
|
||||
# This statement deletes all draft articles in excess found to be duplicate over feed_source_url,
|
||||
# excluding those whose body_markdown is different from the other duplicate occurrences
|
||||
result_same_body = ActiveRecord::Base.connection.execute(
|
||||
<<~SQL,
|
||||
WITH duplicates_draft_articles AS
|
||||
(SELECT id
|
||||
FROM
|
||||
(SELECT id,
|
||||
published,
|
||||
body_markdown,
|
||||
LAG(body_markdown, 1) OVER(PARTITION BY feed_source_url
|
||||
ORDER BY id ASC) AS previous_body_markdown,
|
||||
ROW_NUMBER() OVER(PARTITION BY feed_source_url
|
||||
ORDER BY id ASC) AS row_number
|
||||
FROM articles
|
||||
WHERE feed_source_url IS NOT NULL ) duplicates
|
||||
WHERE duplicates.row_number > 1
|
||||
AND published = 'f' -- drafts
|
||||
AND body_markdown = previous_body_markdown -- with the same body
|
||||
)
|
||||
DELETE
|
||||
FROM articles
|
||||
WHERE id IN (SELECT id FROM duplicates_draft_articles) RETURNING id;
|
||||
SQL
|
||||
)
|
||||
|
||||
# Now that all duplicates with the same body are gone, we need to deal with duplicate feed source URLs
|
||||
# with different bodies.
|
||||
# We thus select the oldest for removal preserving the most recent one
|
||||
result_different_bodies = ActiveRecord::Base.connection.execute(
|
||||
<<~SQL,
|
||||
WITH duplicates_draft_articles AS
|
||||
(SELECT id
|
||||
FROM
|
||||
(SELECT id,
|
||||
published,
|
||||
body_markdown,
|
||||
LAG(body_markdown, 1) OVER(PARTITION BY feed_source_url
|
||||
ORDER BY created_at DESC) AS previous_body_markdown,
|
||||
ROW_NUMBER() OVER(PARTITION BY feed_source_url
|
||||
ORDER BY created_at DESC) AS row_number
|
||||
FROM articles
|
||||
WHERE feed_source_url IS NOT NULL ) duplicates
|
||||
WHERE duplicates.row_number > 1
|
||||
AND published = 'f' -- drafts
|
||||
AND body_markdown != previous_body_markdown -- with different bodies
|
||||
)
|
||||
DELETE
|
||||
FROM articles
|
||||
WHERE id IN (SELECT id FROM duplicates_draft_articles) RETURNING id;
|
||||
SQL
|
||||
)
|
||||
|
||||
result_same_body_ids = result_same_body.map { |row| row["id"] }
|
||||
result_different_bodies_ids = result_different_bodies.map { |row| row["id"] }
|
||||
|
||||
# Remove all articles from Elasticsearch
|
||||
(result_same_body_ids + result_different_bodies_ids).each do |id|
|
||||
Search::RemoveFromIndexWorker.perform_in(5.seconds, "Article", id)
|
||||
end
|
||||
|
||||
# Store deleted IDs temporarily in Redis for safe keeping
|
||||
Rails.cache.write(
|
||||
"DataUpdateScripts::RemoveDraftArticlesWithDuplicateFeedSourceUrl::SameBody",
|
||||
result_same_body_ids,
|
||||
expires_in: 2.weeks,
|
||||
)
|
||||
Rails.cache.write(
|
||||
"DataUpdateScripts::RemoveDraftArticlesWithDuplicateFeedSourceUrl::DifferentBodies",
|
||||
result_same_body_ids,
|
||||
expires_in: 2.weeks,
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
module DataUpdateScripts
|
||||
class CleanupArticlesWithInvalidFeedSourceUrl
|
||||
def run
|
||||
# We need this to intercept those strings that would be valid URLs if the scheme were added,
|
||||
# so I slightly modified the Ruby URL regexp from https://urlregex.com/ to avoid checking for the scheme
|
||||
almost_url_regexp = %r{\A(?:\S+(?::\S*)?@)?(?:(?!10(?:\.\d{1,3}){3})(?!127(?:\.\d{1,3}){3})(?!169\.254(?:\.\d{1,3}){2})(?!192\.168(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:/[^\s]*)?\z}i # rubocop:disable Layout/LineLength
|
||||
|
||||
Article.where.not(feed_source_url: nil).where("feed_source_url NOT ILIKE ?", "http%").find_each do |article|
|
||||
fixed_feed_source_url = article.canonical_url.presence ||
|
||||
(almost_url_regexp.match?(article.feed_source_url) ? "https://#{article.feed_source_url}" : nil)
|
||||
|
||||
article.update_columns(feed_source_url: fixed_feed_source_url)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
require "rails_helper"
|
||||
require Rails.root.join(
|
||||
"lib/data_update_scripts/20200904141057_cleanup_articles_with_invalid_feed_source_url.rb",
|
||||
)
|
||||
|
||||
describe DataUpdateScripts::CleanupArticlesWithInvalidFeedSourceUrl do
|
||||
let(:article) { create(:article) }
|
||||
|
||||
it "sets feed_source_url to canonical_url if the latter is present" do
|
||||
canonical_url = "https://example.com/article"
|
||||
article.update_columns(feed_source_url: "", canonical_url: canonical_url)
|
||||
|
||||
described_class.new.run
|
||||
|
||||
expect(article.reload.feed_source_url).to eq(canonical_url)
|
||||
end
|
||||
|
||||
it "sets empty string feed source url to nil" do
|
||||
article.update_columns(feed_source_url: "")
|
||||
|
||||
described_class.new.run
|
||||
|
||||
expect(article.reload.feed_source_url).to be(nil)
|
||||
end
|
||||
|
||||
it "sets totally invalid url to nil" do
|
||||
article.update_columns(feed_source_url: "not a url")
|
||||
|
||||
described_class.new.run
|
||||
|
||||
expect(article.reload.feed_source_url).to be(nil)
|
||||
end
|
||||
|
||||
it "sets an 'almost URL' to a https URL" do
|
||||
article.update_columns(feed_source_url: "dev.to")
|
||||
|
||||
described_class.new.run
|
||||
|
||||
expect(article.reload.feed_source_url).to eq("https://dev.to")
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue