More specs and refactoring podcasts fetching (#3245)

* More specs and refactoring podcasts fetching

* Fixed PodcastFeed references in tests

* Pass podcast_id instead of podcast to CreateEpisode
This commit is contained in:
Anna Buianova 2019-06-22 04:31:15 +03:00 committed by Ben Halpern
parent 91de529705
commit bfa1bdc37f
12 changed files with 394 additions and 95 deletions

View file

@ -2,7 +2,7 @@ module Podcasts
class GetEpisodesJob < ApplicationJob
queue_as :podcasts_get_episodes
def perform(podcast_id, limit = 1000, feed = PodcastFeed.new)
def perform(podcast_id, limit = 1000, feed = Podcasts::Feed.new)
podcast = Podcast.find_by(id: podcast_id)
return unless podcast

View file

@ -1,88 +0,0 @@
require "rss"
require "rss/itunes"
class PodcastFeed
def get_episodes(podcast, num = 1000)
rss = HTTParty.get(podcast.feed_url).body
feed = RSS::Parser.parse(rss, false)
feed.items.first(num).each do |item|
ep = existing_episode(item, podcast).first
if ep
update_existing_episode(ep, item, podcast)
else
create_new_episode(item, podcast)
end
end
feed.items.size
rescue StandardError => e
Rails.logger.error(e)
end
private
# returns empty array if an episode doesn't exist
def existing_episode(item, podcast)
# presence returns nil if the query is an empty array, otherwise returns the array
podcasts = PodcastEpisode.where(media_url: item.enclosure.url).presence ||
PodcastEpisode.where(title: item.title).presence ||
PodcastEpisode.where(guid: item.guid.to_s).presence ||
(podcast.unique_website_url? && PodcastEpisode.where(website_url: item.link).presence)
podcasts.to_a
end
def create_new_episode(item, podcast)
ep = PodcastEpisode.new
ep.title = item.title
ep.podcast_id = podcast.id
ep.slug = item.title.parameterize
ep.subtitle = item.itunes_subtitle
ep.summary = item.itunes_summary
ep.website_url = item.link
ep.guid = item.guid
get_media_url(ep, item, podcast)
begin
ep.published_at = item.pubDate.to_date
rescue StandardError => e
Rails.logger.error("not a valid date: #{e}")
end
ep.body = item.content_encoded || item.itunes_summary || item.description
ep.save!
end
def update_existing_episode(episode, item, _podcast)
if episode.published_at.nil?
begin
episode.published_at = item.pubDate.to_date
episode.save
rescue StandardError => e
Rails.logger.error("not a valid date: #{e}")
end
end
update_media_url(episode, item)
end
def get_media_url(episode, item, podcast)
episode.media_url = if Rails.env.test? ||
HTTParty.head(item.enclosure.url.gsub(/http:/, "https:")).code == 200
item.enclosure.url.gsub(/http:/, "https:")
else
item.enclosure.url
end
rescue StandardError
# podcast episode must have a media_url
episode.media_url = item.enclosure.url
podcast.update(status_notice: "This podcast may not be playable in the browser") if podcast.status_notice.empty?
end
def update_media_url(episode, item)
if episode.media_url.include?("https")
nil
elsif !episode.media_url.include?("https") &&
item.enclosure.url.include?("https")
episode.update!(media_url: item.enclosure.url)
end
rescue StandardError
message = "something went wrong with #{podcast.title}, #{episode.title} -- #{episode.media_url}"
Rails.logger.error(message)
end
end

View file

@ -0,0 +1,49 @@
module Podcasts
class CreateEpisode
def initialize(podcast_id, item)
@podcast_id = podcast_id
@item = item
end
def self.call(*args)
new(*args).call
end
def call
ep = PodcastEpisode.new
ep.title = item.title
ep.podcast_id = podcast_id
ep.slug = item.title.parameterize
ep.subtitle = item.itunes_subtitle
ep.summary = item.itunes_summary
ep.website_url = item.link
ep.guid = item.guid
get_media_url(ep)
begin
ep.published_at = item.pubDate.to_date
rescue StandardError => e
Rails.logger.error("not a valid date: #{e}")
end
ep.body = item.content_encoded || item.itunes_summary || item.description
ep.save!
ep
end
private
attr_reader :podcast_id, :item
# checking url when it is https is useless, the url is set to the enclosure url anyway
def get_media_url(episode)
episode.media_url = if HTTParty.head(item.enclosure.url.gsub(/http:/, "https:")).code == 200
item.enclosure.url.gsub(/http:/, "https:")
else
item.enclosure.url
end
rescue StandardError
# podcast episode must have a media_url
episode.media_url = item.enclosure.url
episode.podcast.update(status_notice: "This podcast may not be playable in the browser") if episode.podcast.status_notice.empty?
end
end
end

View file

@ -0,0 +1,18 @@
require "rss"
require "rss/itunes"
module Podcasts
class Feed
def get_episodes(podcast, num = 1000)
rss = HTTParty.get(podcast.feed_url).body
feed = RSS::Parser.parse(rss, false)
get_episode = Podcasts::GetEpisode.new(podcast)
feed.items.first(num).each do |item|
get_episode.call(item)
end
feed.items.size
rescue StandardError => e
Rails.logger.error(e)
end
end
end

View file

@ -0,0 +1,30 @@
module Podcasts
class GetEpisode
def initialize(podcast)
@podcast = podcast
end
def call(item)
ep = existing_episode(item, podcast).first
if ep
Podcasts::UpdateEpisode.call(ep, item)
else
Podcasts::CreateEpisode.call(podcast.id, item)
end
end
private
attr_reader :podcast
# returns empty array if an episode doesn't exist
def existing_episode(item, podcast)
# presence returns nil if the query is an empty array, otherwise returns the array
podcasts = PodcastEpisode.where(media_url: item.enclosure.url).presence ||
PodcastEpisode.where(title: item.title).presence ||
PodcastEpisode.where(guid: item.guid.to_s).presence ||
(podcast.unique_website_url? && PodcastEpisode.where(website_url: item.link).presence)
podcasts.to_a
end
end
end

View file

@ -0,0 +1,35 @@
module Podcasts
class UpdateEpisode
def initialize(episode, item)
@episode = episode
@item = item
end
def self.call(*args)
new(*args).call
end
def call
update_published_at unless episode.published_at?
update_media_url if !episode.media_url.include?("https") && item.enclosure.url.include?("https")
end
private
attr_reader :episode, :item
def update_published_at
episode.published_at = item.pubDate.to_date
episode.save
rescue StandardError => e
Rails.logger.error("not a valid date: #{e}")
end
def update_media_url
episode.update!(media_url: item.enclosure.url)
rescue StandardError
message = "something went wrong with #{episode.podcast_title}, #{episode.title} -- #{episode.media_url}"
Rails.logger.error(message)
end
end
end

View file

@ -9,7 +9,7 @@ RSpec.describe "ArticlesApi", type: :request, vcr: vcr_option do
let(:podcast) { create(:podcast, feed_url: "http://softwareengineeringdaily.com/feed/podcast/") }
before do
PodcastFeed.new.get_episodes(podcast, 2)
Podcasts::Feed.new.get_episodes(podcast, 2)
end
describe "GET /api/articles" do

View file

@ -0,0 +1,61 @@
require "rails_helper"
require "rss"
require "rss/itunes"
RSpec.describe Podcasts::CreateEpisode, type: :service do
let!(:podcast) { create(:podcast) }
context "when item has an https media_url" do
let!(:item) { RSS::Parser.parse("spec/support/fixtures/developertea.rss", false).items.first }
before do
stub_request(:head, item.enclosure.url).to_return(status: 200)
end
it "creates an episode" do
expect do
described_class.call(podcast.id, item)
end.to change(PodcastEpisode, :count).by(1)
end
it "creates an episode with correct data" do
episode = described_class.call(podcast.id, item)
expect(episode.title).to eq("Individual Contributor Career Growth w/ Matt Klein (part 1)")
expect(episode.podcast_id).to eq(podcast.id)
expect(episode.website_url).to eq("http://developertea.simplecast.fm/50464d4b")
expect(episode.guid).to include("53b17a1e-271b-40e3-a084-a67b4fcba562")
end
it "rescues an exception when pubDate is invalid" do
allow(item).to receive(:pubDate).and_return("not a date, haha")
episode = described_class.call(podcast.id, item)
expect(episode).to be_persisted
expect(episode.published_at).to eq(nil)
end
end
context "when item has an http media url" do
let!(:item) { RSS::Parser.parse("spec/support/fixtures/awayfromthekeyboard.rss", false).items.first }
let(:https_url) { "https://awayfromthekeyboard.com/wp-content/uploads/2018/02/Episode_075_Lara_Hogan_Demystifies_Public_Speaking.mp3" }
it "sets media_url to https version when it is available" do
stub_request(:head, https_url).to_return(status: 200)
episode = described_class.call(podcast.id, item)
expect(episode.media_url).to eq(https_url)
end
it "keeps an http media url when https version is not available" do
stub_request(:head, https_url).to_return(status: 404)
episode = described_class.call(podcast.id, item)
expect(episode.media_url).to eq(item.enclosure.url)
end
# enable when the logic will not rely solely on exception
xit "sets status notice when https version is not available" do
stub_request(:head, https_url).to_return(status: 404)
described_class.call(podcast.id, item)
podcast.reload
expect(podcast.status_notice).to include("may not be playable")
end
end
end

View file

@ -5,7 +5,7 @@ vcr_option = {
allow_playback_repeats: "true"
}
RSpec.describe PodcastFeed, vcr: vcr_option do
RSpec.describe Podcasts::Feed, vcr: vcr_option do
let(:feed_url) { "http://softwareengineeringdaily.com/feed/podcast/" }
let(:podcast) { create(:podcast, feed_url: feed_url) }
@ -14,14 +14,19 @@ RSpec.describe PodcastFeed, vcr: vcr_option do
end
context "when creating" do
before do
stub_request(:head, "https://traffic.libsyn.com/sedaily/AnalyseAsia.mp3").to_return(status: 200)
stub_request(:head, "https://traffic.libsyn.com/sedaily/IFTTT.mp3").to_return(status: 200)
end
it "fetches podcast episodes" do
expect do
PodcastFeed.new.get_episodes(podcast, 2)
described_class.new.get_episodes(podcast, 2)
end.to change(PodcastEpisode, :count).by(2)
end
it "fetches correct podcasts" do
PodcastFeed.new.get_episodes(podcast, 2)
described_class.new.get_episodes(podcast, 2)
episodes = podcast.podcast_episodes
expect(episodes.pluck(:title).sort).to eq(["Analyse Asia with Bernard Leong", "IFTTT Architecture with Nicky Leach"])
expect(episodes.pluck(:media_url).sort).to eq(%w[https://traffic.libsyn.com/sedaily/AnalyseAsia.mp3 https://traffic.libsyn.com/sedaily/IFTTT.mp3])
@ -34,12 +39,12 @@ RSpec.describe PodcastFeed, vcr: vcr_option do
it "does not refetch already fetched episodes" do
expect do
PodcastFeed.new.get_episodes(podcast, 2)
described_class.new.get_episodes(podcast, 2)
end.not_to change(PodcastEpisode, :count)
end
it "updates published_at for existing episodes" do
PodcastFeed.new.get_episodes(podcast, 2)
described_class.new.get_episodes(podcast, 2)
episode.reload
episode2.reload
expect(episode.published_at).to be_truthy

View file

@ -0,0 +1,38 @@
require "rails_helper"
RSpec.describe Podcasts::UpdateEpisode, type: :service do
let(:podcast) { create(:podcast) }
let(:enclosure) { instance_double("RSS::Rss::Channel::Item::Enclosure", url: "https://audio.simplecast.com/2330f132.mp3") }
let(:item) { instance_double("RSS::Rss::Channel::Item", pubDate: "2019-06-19", enclosure: enclosure) }
it "updates published_at if it's nil" do
episode = create(:podcast_episode, podcast: podcast, published_at: nil)
described_class.call(episode, item)
episode.reload
expect(episode.published_at).to be_truthy
end
it "updates media_url if the item url contains https" do
episode = create(:podcast_episode, podcast: podcast, media_url: "http://audio.simplecast.com/2330f132.mp3")
described_class.call(episode, item)
episode.reload
expect(episode.media_url).to eq("https://audio.simplecast.com/2330f132.mp3")
end
it "catches expception when pubDate is invalid" do
invalid_item = instance_double("RSS::Rss::Channel::Item", pubDate: "i'm not a date", enclosure: enclosure)
episode = create(:podcast_episode, podcast: podcast, published_at: nil)
described_class.call(episode, invalid_item)
expect(episode.published_at).to be_nil
end
it "does fine when there's nothing to update" do
published_at = Time.current - 1.day
episode = create(:podcast_episode, podcast: podcast, media_url: "https://audio.simplecast.com/100.mp3", published_at: published_at)
described_class.call(episode, item)
episode.reload
expect(episode.media_url).to eq("https://audio.simplecast.com/100.mp3")
expect(episode.published_at.strftime("%Y-%m-%d")).to eq(published_at.strftime("%Y-%m-%d"))
end
end

View file

@ -0,0 +1,51 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:wfw="http://wellformedweb.org/CommentAPI/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:georss="http://www.georss.org/georss" xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#">
<channel>
<title>Away From The Keyboard</title>
<atom:link href="http://awayfromthekeyboard.com/?feed=awayfromthekeyboard" rel="self" type="application/rss+xml"/>
<link>http://awayfromthekeyboard.com</link>
<description>Away From The Keyboard is where technologists tell their stories of how they started, how they grew, how they learned, and how they unwind. Hosted by software developers Cecil Phillip and Richie Rump, Away From The Keyboard looks at the human side of technology in a fun and informative way. New episodes are released every Tuesday.</description>
<lastBuildDate>Sun, 30 Sep 2018 10:00:52 +0000</lastBuildDate>
<language>en-us</language>
<copyright>&amp; © 2019 Away From The Keyboard</copyright>
<itunes:subtitle>A podcast exploring the other side of the technical community</itunes:subtitle>
<itunes:author>Away From The Keyboard</itunes:author>
<itunes:owner>
<itunes:name>Away From The Keyboard</itunes:name>
<itunes:email>jorriss@gmail.com</itunes:email>
</itunes:owner>
<itunes:image href="http://awayfromthekeyboard.com/wp-content/uploads/2015/04/Away_from_the_Keyboard_iTunes.jpg"/>
<image>
<url>http://awayfromthekeyboard.com/wp-content/uploads/2015/04/Away_from_the_Keyboard_iTunes.jpg</url>
<title>Away From The Keyboard</title>
<link>http://awayfromthekeyboard.com</link>
</image>
<itunes:explicit>no</itunes:explicit>
<itunes:category text="Technology">
<itunes:category text="Software How-To"/>
</itunes:category>
<sy:updatePeriod>hourly</sy:updatePeriod>
<sy:updateFrequency>1</sy:updateFrequency>
<generator>https://wordpress.org/?v=5.1.1</generator>
<site xmlns="com-wordpress:feed-additions:1">89028513</site>
<item>
<title>Episode 75: Lara Hogan Demystifies Public Speaking</title>
<itunes:summary>The conversation starts with Lara discussing her start in technology which believe it or not started with Lord of the Rings, international studies, a semester in Prague, and photography. Lara then goes in and discusses how she started speaking publicly, how she prepares to for a presentation, and why she wrote the book &quot;Demystifying Public Speaking&quot;. The panel then talks travel and their tips for traveling. Lara then shares about her trip to New Zealand and what she does when she&#039;s away from the keyboard.
</itunes:summary>
<itunes:image href="http://awayfromthekeyboard.com/wp-content/uploads/2018/02/Lara_Hogan_Front-400x400.png"/>
<link>http://awayfromthekeyboard.com/2018/02/28/episode-75-lara-hogan-demystifies-public-speaking/</link>
<pubDate>Wed, 28 Feb 2018 11:00:41 +0000</pubDate>
<dc:creator>Richie Rump</dc:creator>
<guid isPermaLink="false">http://awayfromthekeyboard.com/?p=1092</guid>
<description>The conversation starts with Lara discussing her start in technology which believe it or not started with Lord of the Rings, international studies, a semester in Prague, and photography. Lara then goes in and discusses how she started speaking publicly, how she prepares to for a presentation, and why she wrote the book "Demystifying Public Speaking". The panel then talks travel and their tips for traveling. Lara then shares about her trip to New Zealand and what she does when she's away from the keyboard.
</description>
<content:encoded>The conversation starts with Lara discussing her start in technology which believe it or not started with Lord of the Rings, international studies, a semester in Prague, and photography. Lara then goes in and discusses how she started speaking publicly, how she prepares to for a presentation, and why she wrote the book "Demystifying Public Speaking". The panel then talks travel and their tips for traveling. Lara then shares about her trip to New Zealand and what she does when she's away from the keyboard.
</content:encoded>
<enclosure url="http://awayfromthekeyboard.com/wp-content/uploads/2018/02/Episode_075_Lara_Hogan_Demystifies_Public_Speaking.mp3" length="46704556" type="audio/mpeg"/>
<itunes:duration>43:09</itunes:duration>
<itunes:explicit>No</itunes:explicit>
<itunes:author>Richie Rump</itunes:author>
<post-id>1092</post-id>
</item>
</channel>
</rss>

View file

@ -0,0 +1,100 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel>
<atom:link rel="self" type="application/atom+xml" href="https://rss.simplecast.com/podcasts/363/rss" title="MP3 Audio"/>
<title>Developer Tea</title>
<generator>https://simplecast.com</generator>
<description>Developer Tea exists to help driven developers connect to their ultimate purpose and excel at their work so that they can positively impact the people they influence.
With over 13 million downloads to date, Developer Tea is a short podcast hosted by Jonathan Cutrell (@jcutrell), co-founder of Spec and developer at @Clearbit. We hope you'll take the topics from this podcast and continue the conversation, either online or in person with your peers. Twitter: @developertea :: Email: developertea@gmail.com</description>
<copyright>© 2018 Spec Network, Inc.</copyright>
<language>en-us</language>
<pubDate>Wed, 19 Jun 2019 05:00:00 -0400</pubDate>
<lastBuildDate>Wed, 19 Jun 2019 19:24:51 -0400</lastBuildDate>
<link>http://www.developertea.com</link>
<image>
<url>https://media.simplecast.com/podcast/image/363/1471485029-artwork.jpg</url>
<title>Developer Tea</title>
<link>http://www.developertea.com</link>
</image>
<itunes:new-feed-url>https://rss.simplecast.com/podcasts/363/rss</itunes:new-feed-url>
<itunes:author>Spec</itunes:author>
<itunes:image href="https://media.simplecast.com/podcast/image/363/1471485029-artwork.jpg"/>
<itunes:summary>Developer Tea exists to help driven developers connect to their ultimate purpose and excel at their work so that they can positively impact the people they influence.
With over 13 million downloads to date, Developer Tea is a short podcast hosted by Jonathan Cutrell (@jcutrell), co-founder of Spec and developer at @Clearbit. We hope you'll take the topics from this podcast and continue the conversation, either online or in person with your peers. Twitter: @developertea :: Email: developertea@gmail.com</itunes:summary>
<itunes:subtitle>Developer Tea exists to help driven developers connect to their ultimate purpose and excel at their work so that they can positively impact the people they influence.</itunes:subtitle>
<itunes:explicit>no</itunes:explicit>
<itunes:keywords>short podcasts for developers, development, programming, web development</itunes:keywords>
<itunes:type>episodic</itunes:type>
<itunes:owner>
<itunes:name>Spec Network, Inc.</itunes:name>
<itunes:email>shows@spec.fm</itunes:email>
</itunes:owner>
<itunes:category text="Technology"/>
<itunes:category text="Business">
<itunes:category text="Careers"/>
</itunes:category>
<itunes:category text="Society &amp; Culture"/>
<item>
<title>Individual Contributor Career Growth w/ Matt Klein (part 1)</title>
<guid isPermaLink="false">53b17a1e-271b-40e3-a084-a67b4fcba562</guid>
<link>http://developertea.simplecast.fm/50464d4b</link>
<description>What does a long career as an individual contributor look like? The answer isn't always clear cut, especially if you're give the option of becoming a manager. Today, we'll talk to Matt Klein about how he approaches that.
</description>
<content:encoded>
<![CDATA[<p>What does a long career as an individual contributor look like? The answer isn't always clear cut, especially if you're given the option of becoming a manager. Today, we'll talk to <a href="https://twitter.com/mattklein123?lang=en">Matt Klein</a> about how he approaches this in part 1 of our two part interview.</p>
<a name="Matt.on.the.Web"></a>
<h2>Matt on the Web</h2>
<p>Matt is on the engineering team at <a href="https://twitter.com/lyft">Lyft</a> and one of the main contributors to the open source project, <a href="https://twitter.com/envoyproxy">Envoy</a>.</p>
<ul>
<li><a href="https://twitter.com/mattklein123?lang=en">Twitter</a></li>
<li><a href="https://blog.envoyproxy.io/@mattklein123">Medium</a></li>
</ul>
<a name="Get.in.touch"></a>
<h2>Get in touch</h2>
<p>If you have questions about today's episode, want to start a conversation about today's topic or just want to let us know if you found this episode valuable I encourage you to join the conversation or start your own on our community platform <a href="https://spectrum.chat/specfm/developer-tea">Spectrum.chat/specfm/developer-tea</a></p>
<a name="L.....Leave.a.Review"></a>
<h2>🧡 Leave a Review</h2>
<p>If you're enjoying the show and want to support the content head over to iTunes and <a href="https://itunes.apple.com/us/artist/spec/1019380766?mt=2">leave a review</a>! It helps other developers discover the show and keep us focused on what matters to you.</p>
<a name="L.....Subscribe.to.the.Tea.Break.Challenge"></a>
<h2>🍵 Subscribe to the Tea Break Challenge</h2>
<p> This is a daily challenge designed help you become more self-aware and be a better developer so you can have a positive impact on the people around you. Check it out and give it a try at <a href="https://www.teabreakchallenge.com/.">https://www.teabreakchallenge.com/.</a></p>
<a name="L.....Thanks.to.today.s.sponsor:.Sentry"></a>
<h2>🙏 Thanks to today's sponsor: <a href="https://sentry.io/welcome/">Sentry</a></h2>
<p>Sentry tells you about errors in your code before your customers have a chance to encounter them.</p>
<p>Not only do we tell you about them, we also give you all the details youll need to be able to fix them. Youll see exactly how many users have been impacted by a bug, the stack trace, the commit that the error was released as part of, the engineer who wrote the line of code that is currently busted, and a lot more.</p>
<p>Give it a try for yourself at <a href="https://www.sentry.io">Sentry.io</a></p>
]]>
</content:encoded>
<pubDate>Wed, 19 Jun 2019 05:00:00 -0400</pubDate>
<author>shows@spec.fm (Spec)</author>
<enclosure url="https://audio.simplecast.com/4071c0b8.mp3" length="30373893" type="audio/mpeg"/>
<itunes:author>Spec</itunes:author>
<itunes:image href="https://media.simplecast.com/episode/image/301708/1560835715-artwork.jpg"/>
<itunes:duration>00:31:35</itunes:duration>
<itunes:summary>What does a long career as an individual contributor look like? The answer isn't always clear cut, especially if you're give the option of becoming a manager. Today, we'll talk to Matt Klein about how he approaches that.
</itunes:summary>
<itunes:subtitle>What does a long career as an individual contributor look like? The answer isn't always clear cut, especially if you're give the option of becoming a manager. Today, we'll talk to Matt Klein about how he approaches that.
</itunes:subtitle>
<itunes:keywords></itunes:keywords>
<itunes:explicit>no</itunes:explicit>
<itunes:episodeType>full</itunes:episodeType>
<itunes:episode>696</itunes:episode>
</item>
</channel>
</rss>