216 lines
6.3 KiB
Ruby
216 lines
6.3 KiB
Ruby
require "rss"
|
|
require "open-uri"
|
|
require "nokogiri"
|
|
require "httparty"
|
|
|
|
class RssReader
|
|
def self.get_all_articles
|
|
new.get_all_articles
|
|
end
|
|
|
|
def get_all_articles
|
|
User.where.not(feed_url: nil).each do |u|
|
|
feed_url = u.feed_url.strip
|
|
next if feed_url == ""
|
|
create_articles_for_user(u)
|
|
end
|
|
end
|
|
|
|
def fetch_user(user)
|
|
create_articles_for_user(user)
|
|
end
|
|
|
|
def valid_feed_url?(link)
|
|
true if fetch_rss(link)
|
|
rescue StandardError
|
|
false
|
|
end
|
|
|
|
private
|
|
|
|
def create_articles_for_user(user)
|
|
feed = fetch_rss(user.feed_url)
|
|
feed.entries.reverse_each do |item|
|
|
make_from_rss_item(item, user, feed)
|
|
rescue StandardError => e
|
|
log_error("RssReaderError: occurred while creating article for " \
|
|
"USER: #{user.username} " \
|
|
"FEED-URL: #{user.feed_url} " \
|
|
"ITEM-TITLE: #{item.title || 'no title'} " \
|
|
"ERROR: #{e}")
|
|
end
|
|
rescue StandardError => e
|
|
log_error("RssReaderError: occurred while fetch feed for " \
|
|
"USER: #{user.username} " \
|
|
"FEED-URL: #{user.feed_url} " \
|
|
"ITEM-COUNT: #{get_item_count_error(feed)} " \
|
|
"ERROR: #{e}")
|
|
end
|
|
|
|
def get_item_count_error(feed)
|
|
if feed
|
|
feed.entries ? feed.entries.length : "no count"
|
|
else
|
|
"NIL FEED, INVALID URL"
|
|
end
|
|
end
|
|
|
|
def fetch_rss(url)
|
|
xml = HTTParty.get(url).body
|
|
Feedjira::Feed.parse xml
|
|
end
|
|
|
|
def make_from_rss_item(item, user, feed)
|
|
return if medium_reply?(item) || article_exist?(user, item)
|
|
article_params = {
|
|
feed_source_url: feed_source_url = item.url.strip.split("?source=")[0],
|
|
user_id: user.id,
|
|
published_at: item.published,
|
|
published_from_feed: true,
|
|
show_comments: true,
|
|
body_markdown: assemble_body_markdown(item, user, feed, feed_source_url),
|
|
organization_id: user.organization_id.present? ? user.organization_id : nil
|
|
}
|
|
article = Article.create!(article_params)
|
|
SlackBot.delay.ping(
|
|
"New Article Retrieved via RSS: #{article.title}\nhttps://dev.to#{article.path}",
|
|
channel: "activity",
|
|
username: "article_bot",
|
|
icon_emoji: ":robot_face:",
|
|
)
|
|
end
|
|
|
|
def assemble_body_markdown(item, user, feed, feed_source_url)
|
|
body = <<~HEREDOC
|
|
---
|
|
title: #{item.title.strip}
|
|
published: false
|
|
tags: #{get_tags(item[:categories])}
|
|
canonical_url: #{user.feed_mark_canonical ? feed_source_url : ''}
|
|
---
|
|
|
|
#{finalize_reverse_markdown(item, feed)}
|
|
|
|
HEREDOC
|
|
body.strip
|
|
end
|
|
|
|
def get_tags(categories)
|
|
categories.first(4).map { |tag| tag[0..19] }.join(",") if categories
|
|
end
|
|
|
|
def get_content(item)
|
|
item.content || item.summary || item.description
|
|
end
|
|
|
|
def finalize_reverse_markdown(item, feed)
|
|
cleaned_item_content = HtmlCleaner.new.clean_html(get_content(item))
|
|
cleaned_item_content = thorough_parsing(cleaned_item_content, feed.url)
|
|
ReverseMarkdown.convert(cleaned_item_content, github_flavored: true).
|
|
gsub("```\n\n```", "").gsub(/ |\u00A0/, " ")
|
|
end
|
|
|
|
def thorough_parsing(content, feed_url)
|
|
html_doc = Nokogiri::HTML(content)
|
|
find_and_replace_possible_links!(html_doc)
|
|
if feed_url.include?("medium.com")
|
|
parse_and_translate_gist_iframe!(html_doc)
|
|
parse_and_translate_youtube_iframe!(html_doc)
|
|
parse_and_translate_tweet!(html_doc)
|
|
else
|
|
clean_relative_path!(html_doc, feed_url)
|
|
end
|
|
html_doc.to_html
|
|
end
|
|
|
|
def parse_and_translate_gist_iframe!(html_doc)
|
|
html_doc.css("iframe").each do |iframe|
|
|
a_tag = iframe.css("a")
|
|
next if a_tag.empty?
|
|
possible_link = a_tag[0].inner_html
|
|
if /medium\.com\/media\/.+\/href/.match?(possible_link)
|
|
real_link = ""
|
|
open(possible_link) do |h|
|
|
real_link = h.base_uri.to_s
|
|
end
|
|
return unless real_link.include?("gist.github.com")
|
|
iframe.name = "p"
|
|
iframe.keys.each { |attr| iframe.remove_attribute(attr) }
|
|
iframe.inner_html = "{% gist #{real_link} %}"
|
|
end
|
|
end
|
|
html_doc
|
|
end
|
|
|
|
def parse_and_translate_tweet!(html_doc)
|
|
html_doc.search("style").remove
|
|
html_doc.search("script").remove
|
|
html_doc.css("blockquote").each do |bq|
|
|
bq_with_p = bq.css("p")
|
|
next if bq_with_p.empty?
|
|
second_content = bq_with_p.css("p")[1].css("a")[0].attributes["href"].value
|
|
if bq_with_p.length == 2 && second_content.include?("twitter.com")
|
|
bq.name = "p"
|
|
tweet_id = second_content.scan(/\/status\/(\d{10,})/).flatten.first
|
|
bq.inner_html = "{% tweet #{tweet_id} %}"
|
|
end
|
|
end
|
|
end
|
|
|
|
def parse_and_translate_youtube_iframe!(html_doc)
|
|
html_doc.css("iframe").each do |iframe|
|
|
if /youtube\.com/.match?(iframe.attributes["src"].value)
|
|
iframe.name = "p"
|
|
youtube_id = iframe.attributes["src"].value.scan(/embed%2F(.{4,12})%3F/).flatten.first
|
|
iframe.keys.each { |attr| iframe.remove_attribute(attr) }
|
|
iframe.inner_html = "{% youtube #{youtube_id} %}"
|
|
end
|
|
end
|
|
end
|
|
|
|
def clean_relative_path!(html_doc, url)
|
|
html_doc.css("img").each do |img_tag|
|
|
path = img_tag.attributes["src"].value
|
|
img_tag.attributes["src"].value = URI.join(url, path).to_s if path.start_with? "/"
|
|
end
|
|
end
|
|
|
|
def find_and_replace_possible_links!(html_doc)
|
|
html_doc.css("a").each do |a_tag|
|
|
link = a_tag.attributes["href"]&.value
|
|
next unless link
|
|
found_article = Article.find_by(feed_source_url: link)&.decorate
|
|
if found_article
|
|
a_tag.attributes["href"].value = found_article.url
|
|
end
|
|
end
|
|
end
|
|
|
|
def get_host_without_www(url)
|
|
url = "http://#{url}" if URI.parse(url).scheme.nil?
|
|
host = URI.parse(url).host.downcase
|
|
host.start_with?("www.") ? host[4..-1] : host
|
|
end
|
|
|
|
def medium_reply?(item)
|
|
get_host_without_www(item.url.strip) == "medium.com" &&
|
|
!item[:categories] &&
|
|
content_is_not_the_title?(item)
|
|
end
|
|
|
|
def content_is_not_the_title?(item)
|
|
# [[:space:]] removes all whitespace, including unicode ones.
|
|
content = item.content.gsub(/[[:space:]]/, " ")
|
|
title = item.title.gsub("…", "")
|
|
content.include?(title)
|
|
end
|
|
|
|
def article_exist?(user, item)
|
|
user.articles.find_by_title(item.title.strip.gsub('"', '\"'))
|
|
end
|
|
|
|
def log_error(error_output)
|
|
logger = Logger.new(STDOUT)
|
|
logger.info(error_output)
|
|
end
|
|
end
|