Index Articles and Podcast Episodes to Elasticsearch (#6488) [deploy]

* Index Articles and PodcastEpisodes to Elasticsearch

* add callbacks to podcast episodes and specs

* refactor and clean up serializers

app/models/article.rb

@@ -4,6 +4,10 @@ class Article < ApplicationRecord
   include AlgoliaSearch
   include Storext.model
   include Reactable
+  include Searchable
+
+  SEARCH_SERIALIZER = Search::ArticleSerializer
+  SEARCH_CLASS = Search::FeedContent
 
   acts_as_taggable_on :tags
   resourcify
@@ -71,6 +75,8 @@ class Article < ApplicationRecord
 
   after_update_commit :update_notifications, if: proc { |article| article.notifications.any? && !article.saved_changes.empty? }
   after_commit :async_score_calc, :update_main_image_background_hex, :touch_collection
+  after_commit :index_to_elasticsearch, on: %i[create update]
+  after_commit :remove_from_elasticsearch, on: [:destroy]
 
   before_destroy :before_destroy_actions, prepend: true
 
@@ -382,7 +388,7 @@ class Article < ApplicationRecord
   end
 
   def video_duration_in_minutes
-    minutes = (video_duration_in_seconds.to_i / 60) % 60
+    minutes = video_duration_in_minutes_integer
     seconds = video_duration_in_seconds.to_i % 60
     seconds = "0#{seconds}" if seconds.to_s.size == 1
 
@@ -391,6 +397,10 @@ class Article < ApplicationRecord
     hours < 1 ? "#{minutes}:#{seconds}" : "#{hours}:#{minutes}:#{seconds}"
   end
 
+  def video_duration_in_minutes_integer
+    (video_duration_in_seconds.to_i / 60) % 60
+  end
+
   # keep public because it's used in algolia jobs
   def index_id
     "articles-#{id}"

app/models/podcast_episode.rb

@@ -1,5 +1,9 @@
 class PodcastEpisode < ApplicationRecord
   include AlgoliaSearch
+  include Searchable
+
+  SEARCH_SERIALIZER = Search::PodcastEpisodeSerializer
+  SEARCH_CLASS = Search::FeedContent
 
   acts_as_taggable
 
@@ -23,6 +27,9 @@ class PodcastEpisode < ApplicationRecord
   after_destroy :purge, :purge_all
   after_save    :bust_cache
 
+  after_commit :index_to_elasticsearch, on: %i[create update]
+  after_commit :remove_from_elasticsearch, on: [:destroy]
+
   before_validation :process_html_and_prefix_all_images
 
   scope :reachable, -> { where(reachable: true) }

app/serializers/search/article_serializer.rb

@@ -0,0 +1,38 @@
+module Search
+  class ArticleSerializer
+    include FastJsonapi::ObjectSerializer
+
+    attributes :id, :approved, :body_text, :class_name, :cloudinary_video_url,
+               :comments_count, :experience_level_rating, :experience_level_rating_distribution,
+               :featured, :featured_number, :flare_tag, :hotness_score, :language,
+               :main_image, :path, :positive_reactions_count, :published,
+               :published_at, :reactions_count, :reading_time, :score, :title
+
+    # video_duration_in_minutes in Elasticsearch is mapped as an integer
+    # however, it really is a string in the format 00:00 which is why we
+    # added an extra field to handle that string
+    attribute :video_duration_string, &:video_duration_in_minutes
+    attribute :video_duration_in_minutes, &:video_duration_in_minutes_integer
+
+    attribute :tags do |article|
+      article.tags.map do |tag|
+        { name: tag.name, keywords_for_search: tag.keywords_for_search }
+      end
+    end
+
+    attribute :user do |article|
+      NestedUserSerializer.new(article.user).serializable_hash.dig(
+        :data, :attributes
+      )
+    end
+
+    attribute :organization, if: proc { |a| a.organization.present? } do |article|
+      {
+        slug: article.organization.slug,
+        name: article.organization.name,
+        id: article.organization.id,
+        profile_image_90: article.organization.profile_image_90
+      }
+    end
+  end
+end

app/serializers/search/nested_user_serializer.rb

@@ -0,0 +1,7 @@
+module Search
+  class NestedUserSerializer
+    include FastJsonapi::ObjectSerializer
+
+    attributes :id, :name, :pro, :profile_image_90, :username
+  end
+end

app/serializers/search/podcast_episode_serializer.rb

@@ -0,0 +1,23 @@
+module Search
+  class PodcastEpisodeSerializer
+    include FastJsonapi::ObjectSerializer
+
+    attributes :id, :body_text, :class_name, :comments_count,
+               :featured, :featured_number, :hotness_score, :main_image, :path,
+               :positive_reactions_count, :published, :published_at, :quote,
+               :reactions_count, :search_score, :subtitle, :summary, :title,
+               :website_url
+
+    attribute :tags do |pde|
+      pde.tags.map do |tag|
+        { name: tag.name, keywords_for_search: tag.keywords_for_search }
+      end
+    end
+
+    attribute :user do |pde|
+      NestedUserSerializer.new(pde.podcast.creator).serializable_hash.dig(
+        :data, :attributes
+      )
+    end
+  end
+end

config/elasticsearch/mappings/feed_content.json

@@ -158,6 +158,9 @@
     "video_duration_in_minutes": {
       "type": "integer"
     },
+    "video_duration_string": {
+      "type": "keyword"
+    },
     "website_url": {
       "type": "keyword"
     }

lib/data_update_scripts/20200305201642_index_feed_content_to_elasticsearch.rb

@@ -0,0 +1,17 @@
+module DataUpdateScripts
+  class IndexFeedContentToElasticsearch
+    def run
+      Article.select(:id).find_each do |article|
+        Search::IndexToElasticsearchWorker.set(queue: :low_priority).perform_async(
+          "Article", article.id
+        )
+      end
+
+      PodcastEpisode.select(:id).find_each do |pde|
+        Search::IndexToElasticsearchWorker.set(queue: :low_priority).perform_async(
+          "PodcastEpisode", pde.id
+        )
+      end
+    end
+  end
+end

spec/lib/data_update_scripts/index_feed_content_to_elasticsearch_spec.rb

@@ -0,0 +1,15 @@
+require "rails_helper"
+require Rails.root.join("lib/data_update_scripts/20200305201642_index_feed_content_to_elasticsearch.rb")
+
+describe DataUpdateScripts::IndexFeedContentToElasticsearch, elasticsearch: true do
+  it "indexes feed content(articles and podcast episodes) to Elasticsearch" do
+    article = create(:article)
+    podcast_episode = create(:podcast_episode)
+    expect { article.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
+    expect { podcast_episode.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
+
+    sidekiq_perform_enqueued_jobs { described_class.new.run }
+    expect(article.elasticsearch_doc).not_to be_nil
+    expect(podcast_episode.elasticsearch_doc).not_to be_nil
+  end
+end

spec/models/article_spec.rb

@@ -29,6 +29,22 @@ RSpec.describe Article, type: :model do
     it { is_expected.to validate_presence_of(:user_id) }
     it { is_expected.not_to allow_value("foo").for(:main_image_background_hex_color) }
 
+    describe "#after_commit" do
+      it "on update enqueues job to index article to elasticsearch" do
+        article.save
+        sidekiq_assert_enqueued_with(job: Search::IndexToElasticsearchWorker, args: [described_class.to_s, article.id]) do
+          article.save
+        end
+      end
+
+      it "on destroy enqueues job to delete article from elasticsearch" do
+        article.save
+        sidekiq_assert_enqueued_with(job: Search::RemoveFromElasticsearchIndexWorker, args: [described_class::SEARCH_CLASS.to_s, article.id]) do
+          article.destroy
+        end
+      end
+    end
+
     context "when published" do
       before do
         # rubocop:disable RSpec/NamedSubject

spec/models/podcast_episode_spec.rb

@@ -29,6 +29,22 @@ RSpec.describe PodcastEpisode, type: :model do
     end
   end
 
+  describe "#after_commit" do
+    it "on update enqueues job to index podcast_episode to elasticsearch" do
+      podcast_episode.save
+      sidekiq_assert_enqueued_with(job: Search::IndexToElasticsearchWorker, args: [described_class.to_s, podcast_episode.id]) do
+        podcast_episode.save
+      end
+    end
+
+    it "on destroy enqueues job to delete podcast_episode from elasticsearch" do
+      podcast_episode.save
+      sidekiq_assert_enqueued_with(job: Search::RemoveFromElasticsearchIndexWorker, args: [described_class::SEARCH_CLASS.to_s, podcast_episode.id]) do
+        podcast_episode.destroy
+      end
+    end
+  end
+
   describe "#description" do
     it "strips tags from the body" do
       ep2 = build(:podcast_episode, guid: podcast_episode.guid)

spec/serializers/search/article_serializer_spec.rb

@@ -0,0 +1,15 @@
+require "rails_helper"
+
+RSpec.describe Search::ArticleSerializer do
+  let(:user) { create(:user) }
+  let(:organization) { create(:organization) }
+  let(:article) { create(:article, user: user, organization: organization) }
+
+  it "serializes an article" do
+    data_hash = described_class.new(article).serializable_hash.dig(:data, :attributes)
+    user_data = Search::NestedUserSerializer.new(user).serializable_hash.dig(:data, :attributes)
+    expect(data_hash[:user]).to eq(user_data)
+    expect(data_hash.dig(:organization, :id)).to eq(organization.id)
+    expect(data_hash.keys).to include(:id, :body_text, :hotness_score)
+  end
+end

spec/serializers/search/podcast_episode_serializer_spec.rb

@@ -0,0 +1,14 @@
+require "rails_helper"
+
+RSpec.describe Search::PodcastEpisodeSerializer do
+  let(:user) { create(:user) }
+  let(:podcast) { create(:podcast, creator_id: user.id) }
+  let(:podcast_ep) { create(:podcast_episode, podcast: podcast) }
+
+  it "serializes a podcast episode" do
+    data_hash = described_class.new(podcast_ep).serializable_hash.dig(:data, :attributes)
+    user_data = Search::NestedUserSerializer.new(user).serializable_hash.dig(:data, :attributes)
+    expect(data_hash[:user]).to eq(user_data)
+    expect(data_hash.keys).to include(:id, :body_text)
+  end
+end