[deploy] Add public reactions count fields to Elasticsearch (#7956)

* Add public_reactions_count field to elasticsearch

* Add new data update scripts and update old ones

* Use BulkIndexWorker to batch indexing

* Use proper search class for handling feed content

* Remove redundant script oops

* Rename data update script and remove old specs

* Combine specs into one file

app/models/podcast_episode.rb

@@ -42,6 +42,7 @@ class PodcastEpisode < ApplicationRecord
   scope :for_user, lambda { |user|
     joins(:podcast).where(podcasts: { creator_id: user.id })
   }
+  scope :eager_load_serialized_data, -> {}
 
   def search_id
     "podcast_episode_#{id}"
@@ -75,6 +76,7 @@ class PodcastEpisode < ApplicationRecord
   alias hotness_score zero_method
   alias search_score zero_method
   alias positive_reactions_count zero_method
+  alias public_reactions_count zero_method
 
   def class_name
     self.class.name

app/models/user.rb

@@ -144,6 +144,7 @@ class User < ApplicationRecord
   validate :update_rate_limit
 
   alias_attribute :positive_reactions_count, :reactions_count
+  alias_attribute :public_reactions_count, :reactions_count
   alias_attribute :subscribed_to_welcome_notifications?, :welcome_notifications
 
   scope :with_this_week_comments, lambda { |number|

app/serializers/search/article_serializer.rb

@@ -7,7 +7,7 @@ module Search
     attributes :approved, :body_text, :class_name, :cloudinary_video_url,
                :comments_count, :experience_level_rating, :experience_level_rating_distribution,
                :featured, :featured_number, :hotness_score, :language,
-               :main_image, :path, :positive_reactions_count, :published,
+               :main_image, :path, :positive_reactions_count, :public_reactions_count, :published,
                :published_at, :reactions_count, :reading_time, :score, :title
 
     # video_duration_in_minutes in Elasticsearch is mapped as an integer

app/serializers/search/comment_serializer.rb

@@ -4,7 +4,7 @@ module Search
 
     attribute :id, &:search_id
 
-    attributes :path, :positive_reactions_count
+    attributes :path, :positive_reactions_count, :public_reactions_count
 
     attribute :body_text, &:body_markdown
     attribute :class_name do |comment|

app/serializers/search/podcast_episode_serializer.rb

@@ -5,7 +5,7 @@ module Search
     attribute :id, &:search_id
 
     attributes :body_text, :class_name, :comments_count, :hotness_score, :path,
-               :positive_reactions_count, :published, :published_at, :quote,
+               :positive_reactions_count, :public_reactions_count, :published, :published_at, :quote,
                :reactions_count, :search_score, :subtitle, :summary, :title,
                :website_url
 

app/serializers/search/user_serializer.rb

@@ -13,6 +13,7 @@ module Search
                :name,
                :path,
                :positive_reactions_count,
+               :public_reactions_count,
                :profile_image_90,
                :reactions_count,
                :username

app/services/search/query_builders/feed_content.rb

@@ -52,6 +52,7 @@ module Search
         main_image
         path
         positive_reactions_count
+        public_reactions_count
         published_at
         readable_publish_date_string
         reading_time

app/workers/search/bulk_index_worker.rb

@@ -10,7 +10,9 @@ module Search
         where(id: ids).
         find_each.map(&:serialized_search_hash)
 
-      "Search::#{object_class}".constantize.bulk_index(data_hashes)
+      search_class = object_class.constantize::SEARCH_CLASS
+
+      search_class.bulk_index(data_hashes)
     end
   end
 end

config/elasticsearch/mappings/feed_content.json

@@ -87,6 +87,9 @@
     "positive_reactions_count": {
       "type": "integer"
     },
+    "public_reactions_count": {
+      "type": "integer"
+    },
     "published": {
       "type": "boolean"
     },

config/elasticsearch/mappings/users.json

@@ -41,6 +41,9 @@
     "positive_reactions_count": {
       "type": "integer"
     },
+    "public_reactions_count": {
+      "type": "integer"
+    },
     "profile_image_90": {
       "type": "keyword"
     },

lib/data_update_scripts/20200326145114_re_index_feed_content_to_elasticsearch.rb

@@ -1,11 +1,11 @@
 module DataUpdateScripts
   class ReIndexFeedContentToElasticsearch
     def run
-      clear_existing_feed_documents
+      # clear_existing_feed_documents
 
-      index_docs(Article.pluck(:id), "Article")
-      index_docs(PodcastEpisode.pluck(:id), "PodcastEpisode")
-      index_docs(Comment.pluck(:id), "Comment")
+      # index_docs(Article.pluck(:id), "Article")
+      # index_docs(PodcastEpisode.pluck(:id), "PodcastEpisode")
+      # index_docs(Comment.pluck(:id), "Comment")
     end
 
     private

lib/data_update_scripts/20200406213152_re_index_users_to_elasticsearch.rb

@@ -1,11 +1,11 @@
 module DataUpdateScripts
   class ReIndexUsersToElasticsearch
     def run
-      User.select(:id).find_each do |user|
-        Search::IndexWorker.set(queue: :low_priority).perform_async(
-          "User", user.id
-        )
-      end
+      # User.select(:id).find_each do |user|
+      #   Search::IndexWorker.set(queue: :low_priority).perform_async(
+      #     "User", user.id
+      #   )
+      # end
     end
   end
 end

lib/data_update_scripts/20200519142908_re_index_feed_content_and_users_to_elasticsearch.rb

@@ -0,0 +1,27 @@
+module DataUpdateScripts
+  class ReIndexFeedContentAndUsersToElasticsearch
+    def run
+      Article.select(:id).in_batches(of: 100) do |batch|
+        Search::BulkIndexWorker.set(queue: :default).perform_async(
+          "Article", batch.pluck(:id)
+        )
+      end
+      Comment.select(:id).in_batches(of: 100) do |batch|
+        Search::BulkIndexWorker.set(queue: :default).perform_async(
+          "Comment", batch.pluck(:id)
+        )
+      end
+      PodcastEpisode.select(:id).in_batches(of: 100) do |batch|
+        Search::BulkIndexWorker.set(queue: :default).perform_async(
+          "PodcastEpisode", batch.pluck(:id)
+        )
+      end
+
+      User.select(:id).in_batches(of: 200) do |batch|
+        Search::BulkIndexWorker.set(queue: :default).perform_async(
+          "User", batch.pluck(:id)
+        )
+      end
+    end
+  end
+end

spec/lib/data_update_scripts/re_index_feed_content_and_users_to_elasticsearch_spec.rb

@@ -0,0 +1,36 @@
+# rubocop:disable RSpec/ExampleLength
+# rubocop:disable RSpec/MultipleExpectations
+require "rails_helper"
+require Rails.root.join("lib/data_update_scripts/20200519142908_re_index_feed_content_and_users_to_elasticsearch.rb")
+
+describe DataUpdateScripts::ReIndexFeedContentAndUsersToElasticsearch do
+  after do
+    Search::FeedContent.refresh_index
+    Search::User.refresh_index
+  end
+
+  it "indexes feed content(articles, comments, podcast episodes) and users to Elasticsearch" do
+    article = create(:article)
+    podcast_episode = create(:podcast_episode)
+    comment = create(:comment)
+    user = create(:user)
+
+    expect { article.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
+    expect { podcast_episode.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
+    expect { comment.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
+    expect { user.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
+
+    sidekiq_perform_enqueued_jobs { described_class.new.run }
+    expect(article.elasticsearch_doc).not_to be_nil
+    expect(podcast_episode.elasticsearch_doc).not_to be_nil
+    expect(comment.elasticsearch_doc).not_to be_nil
+    expect(user.elasticsearch_doc).not_to be_nil
+
+    expect(article.elasticsearch_doc["_source"].keys).to include "public_reactions_count"
+    expect(podcast_episode.elasticsearch_doc["_source"].keys).to include "public_reactions_count"
+    expect(comment.elasticsearch_doc["_source"].keys).to include "public_reactions_count"
+    expect(user.elasticsearch_doc["_source"].keys).to include "public_reactions_count"
+  end
+end
+# rubocop:enable RSpec/ExampleLength
+# rubocop:enable RSpec/MultipleExpectations

spec/lib/data_update_scripts/re_index_feed_content_to_elasticsearch_spec.rb

@@ -1,20 +0,0 @@
-require "rails_helper"
-require Rails.root.join("lib/data_update_scripts/20200326145114_re_index_feed_content_to_elasticsearch.rb")
-
-describe DataUpdateScripts::ReIndexFeedContentToElasticsearch, elasticsearch: "FeedContent" do
-  after { Search::FeedContent.refresh_index }
-
-  it "indexes feed content(articles, comments, podcast episodes) to Elasticsearch" do
-    article = create(:article)
-    podcast_episode = create(:podcast_episode)
-    comment = create(:comment)
-    expect { article.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
-    expect { podcast_episode.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
-    expect { comment.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
-
-    sidekiq_perform_enqueued_jobs { described_class.new.run }
-    expect(article.elasticsearch_doc).not_to be_nil
-    expect(podcast_episode.elasticsearch_doc).not_to be_nil
-    expect(comment.elasticsearch_doc).not_to be_nil
-  end
-end

spec/lib/data_update_scripts/re_index_users_to_elasticsearch_spec.rb

@@ -1,11 +0,0 @@
-require "rails_helper"
-require Rails.root.join("lib/data_update_scripts/20200406213152_re_index_users_to_elasticsearch.rb")
-
-describe DataUpdateScripts::ReIndexUsersToElasticsearch, elasticsearch: "User" do
-  it "indexes users to Elasticsearch" do
-    user = create(:user)
-    expect { user.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)
-    sidekiq_perform_enqueued_jobs { described_class.new.run }
-    expect(user.elasticsearch_doc).not_to be_nil
-  end
-end