Clear Elasticsearch Data Instead of Resetting Entire Index for Specs (#7602)

* Clear Elasticsearch Data Instead of Resetting Entire Index for Specs

* only reset for specs that are messing with index creation and deletion, recreate indexes before and after those

* add back elasticsearch meta info to specs and update data update specs

* update serializer specs

* update search and rest of specs with new meta data tag

spec/lib/data_update_scripts/index_chat_channel_memberships_to_elasticsearch_spec.rb

@@ -1,7 +1,7 @@
 require "rails_helper"
 require Rails.root.join("lib/data_update_scripts/20200218195023_index_chat_channel_memberships_to_elasticsearch.rb")
 
-describe DataUpdateScripts::IndexChatChannelMembershipsToElasticsearch, elasticsearch: true do
+describe DataUpdateScripts::IndexChatChannelMembershipsToElasticsearch, elasticsearch: "ChatChannelMembership" do
   it "indexes chat channel memberships to Elasticsearch" do
     chat_channel_membership = FactoryBot.create(:chat_channel_membership)
     expect { chat_channel_membership.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)

spec/lib/data_update_scripts/index_classified_listings_to_elasticsearch_spec.rb

@@ -1,7 +1,7 @@
 require "rails_helper"
 require Rails.root.join("lib/data_update_scripts/20200217215802_index_classified_listings_to_elasticsearch.rb")
 
-describe DataUpdateScripts::IndexClassifiedListingsToElasticsearch, elasticsearch: true do
+describe DataUpdateScripts::IndexClassifiedListingsToElasticsearch, elasticsearch: "ClassifiedListing" do
   it "indexes classified_listings to Elasticsearch" do
     classified_listing = create(:classified_listing)
     expect { classified_listing.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)

spec/lib/data_update_scripts/index_reading_list_reactions_spec.rb

@@ -1,7 +1,7 @@
 require "rails_helper"
 require Rails.root.join("lib/data_update_scripts/20200415200651_index_reading_list_reactions.rb")
 
-describe DataUpdateScripts::IndexReadingListReactions, elasticsearch: true do
+describe DataUpdateScripts::IndexReadingListReactions, elasticsearch: "Reaction" do
   it "indexes feed content(articles, comments, podcast episodes) to Elasticsearch" do
     reactions = create_list(:reaction, 3, category: "readinglist")
     Sidekiq::Worker.clear_all

spec/lib/data_update_scripts/re_index_feed_content_to_elasticsearch_spec.rb

@@ -1,7 +1,9 @@
 require "rails_helper"
 require Rails.root.join("lib/data_update_scripts/20200326145114_re_index_feed_content_to_elasticsearch.rb")
 
-describe DataUpdateScripts::ReIndexFeedContentToElasticsearch, elasticsearch: true do
+describe DataUpdateScripts::ReIndexFeedContentToElasticsearch, elasticsearch: "FeedContent" do
+  after { Search::FeedContent.refresh_index }
+
   it "indexes feed content(articles, comments, podcast episodes) to Elasticsearch" do
     article = create(:article)
     podcast_episode = create(:podcast_episode)

spec/lib/data_update_scripts/re_index_users_to_elasticsearch_spec.rb

@@ -1,7 +1,7 @@
 require "rails_helper"
 require Rails.root.join("lib/data_update_scripts/20200406213152_re_index_users_to_elasticsearch.rb")
 
-describe DataUpdateScripts::ReIndexUsersToElasticsearch, elasticsearch: true do
+describe DataUpdateScripts::ReIndexUsersToElasticsearch, elasticsearch: "User" do
   it "indexes users to Elasticsearch" do
     user = create(:user)
     expect { user.elasticsearch_doc }.to raise_error(Search::Errors::Transport::NotFound)

spec/lib/data_update_scripts/resync_elasticsearch_documents_spec.rb

@@ -1,7 +1,12 @@
 require "rails_helper"
 require Rails.root.join("lib/data_update_scripts/20200410152018_resync_elasticsearch_documents.rb")
 
-describe DataUpdateScripts::ResyncElasticsearchDocuments, elasticsearch: true do
+describe DataUpdateScripts::ResyncElasticsearchDocuments, elasticsearch: %w[FeedContent User Tag] do
+  after do
+    Article::SEARCH_CLASS.refresh_index
+    User::SEARCH_CLASS.refresh_index
+  end
+
   it "indexes podcast episodes and tags to Elasticsearch" do
     tag = create(:tag)
     podcast_episode = create(:podcast_episode)

spec/rails_helper.rb

@@ -78,9 +78,18 @@ RSpec.configure do |config|
     Sidekiq::Worker.clear_all # worker jobs shouldn't linger around between tests
   end
 
-  config.around(:each, elasticsearch: true) do |example|
+  config.around(:each, elasticsearch_reset: true) do |example|
     Search::Cluster.recreate_indexes
     example.run
+    Search::Cluster.recreate_indexes
+  end
+
+  config.around(:each, :elasticsearch) do |ex|
+    klasses = Array.wrap(ex.metadata[:elasticsearch]).map do |search_class|
+      Search.const_get(search_class)
+    end
+    klasses.each { |klass| clear_elasticsearch_data(klass) }
+    ex.run
   end
 
   config.around(:each, throttle: true) do |example|

spec/serializers/search/article_serializer_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Search::ArticleSerializer do
     expect(data_hash.keys).to include(:id, :body_text, :hotness_score)
   end
 
-  it "creates valid json for Elasticsearch", elasticsearch: true do
+  it "creates valid json for Elasticsearch", elasticsearch: "FeedContent" do
     data_hash = described_class.new(article).serializable_hash.dig(:data, :attributes)
     result = Article::SEARCH_CLASS.index(article.id, data_hash)
     expect(result["result"]).to eq("created")

spec/serializers/search/comment_serializer_spec.rb

@@ -12,7 +12,7 @@ RSpec.describe Search::CommentSerializer do
     expect(data_hash.keys).to include(:id, :body_text, :hotness_score, :title)
   end
 
-  it "creates valid json for Elasticsearch", elasticsearch: true do
+  it "creates valid json for Elasticsearch", elasticsearch: "FeedContent" do
     data_hash = described_class.new(comment).serializable_hash.dig(:data, :attributes)
     result = Comment::SEARCH_CLASS.index(comment.id, data_hash)
     expect(result["result"]).to eq("created")

spec/serializers/search/user_serializer_spec.rb

@@ -8,7 +8,7 @@ RSpec.describe Search::UserSerializer do
     expect(data_hash.keys).to include(:id, :name, :path, :username, :roles)
   end
 
-  it "creates valid json for Elasticsearch", elasticsearch: true do
+  it "creates valid json for Elasticsearch", elasticsearch: "User" do
     data_hash = described_class.new(user).serializable_hash.dig(:data, :attributes)
     result = User::SEARCH_CLASS.index(user.id, data_hash)
     expect(result["result"]).to eq("created")

spec/services/search/base_spec.rb

@@ -1,6 +1,6 @@
 require "rails_helper"
 
-RSpec.describe Search::Base, type: :service, elasticsearch: true do
+RSpec.describe Search::Base, type: :service do
   let(:document_id) { 123 }
 
   before do
@@ -8,6 +8,8 @@ RSpec.describe Search::Base, type: :service, elasticsearch: true do
     stub_const("#{described_class}::INDEX_NAME", "tags_#{Rails.env}")
     stub_const("#{described_class}::INDEX_ALIAS", "tags_#{Rails.env}_alias")
     stub_const("#{described_class}::MAPPINGS", Search::Tag::MAPPINGS)
+    Search::Tag.refresh_index
+    clear_elasticsearch_data(Search::Tag)
     allow(described_class).to receive(:index_settings).and_return({})
   end
 
@@ -67,7 +69,7 @@ RSpec.describe Search::Base, type: :service, elasticsearch: true do
     end
   end
 
-  describe "::create_index" do
+  describe "::create_index", elasticsearch_reset: true do
     it "creates an elasticsearch index with INDEX_NAME" do
       described_class.delete_index
       expect(Search::Client.indices.exists(index: described_class::INDEX_NAME)).to eq(false)
@@ -86,7 +88,7 @@ RSpec.describe Search::Base, type: :service, elasticsearch: true do
     end
   end
 
-  describe "::delete_index" do
+  describe "::delete_index", elasticsearch_reset: true do
     it "deletes an elasticsearch index with INDEX_NAME" do
       expect(Search::Client.indices.exists(index: described_class::INDEX_NAME)).to eq(true)
       described_class.delete_index

spec/services/search/chat_channel_membership_spec.rb

@@ -1,7 +1,7 @@
 require "rails_helper"
 
-RSpec.describe Search::ChatChannelMembership, type: :service, elasticsearch: true do
-  describe "::search_documents" do
+RSpec.describe Search::ChatChannelMembership, type: :service do
+  describe "::search_documents", elasticsearch: "ChatChannelMembership" do
     let(:user) { create(:user) }
     let(:chat_channel_membership1) { create(:chat_channel_membership, user_id: user.id) }
     let(:chat_channel_membership2) { create(:chat_channel_membership, user_id: user.id) }

spec/services/search/classified_listing_spec.rb

@@ -1,7 +1,7 @@
 require "rails_helper"
 
-RSpec.describe Search::ClassifiedListing, type: :service, elasticsearch: true do
-  describe "::search_documents" do
+RSpec.describe Search::ClassifiedListing, type: :service do
+  describe "::search_documents", elasticsearch: "ClassifiedListing" do
     let(:classified_listing) { create(:classified_listing) }
 
     it "parses classified_listing document hits from search response" do

spec/services/search/feed_content_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe Search::FeedContent, type: :service do
     expect(described_class::MAPPINGS).not_to be_nil
   end
 
-  describe "::search_documents", elasticsearch: true do
+  describe "::search_documents", elasticsearch: "FeedContent" do
     let(:article1) { create(:article) }
     let(:article2) { create(:article) }
 
@@ -140,8 +140,8 @@ RSpec.describe Search::FeedContent, type: :service do
     end
   end
 
-  describe "document counts" do
-    it "returns counts for each document class", elasticsearch: true do
+  describe "document counts", elasticsearch: "FeedContent" do
+    it "returns counts for each document class" do
       article = create(:article)
       comment = create(:comment)
       pde = create(:podcast_episode)

spec/services/search/reaction_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe Search::Reaction, type: :service do
     expect(described_class::MAPPINGS).not_to be_nil
   end
 
-  describe "::search_documents", elasticsearch: true do
+  describe "::search_documents", elasticsearch: "Reaction" do
     let(:article1) { create(:article) }
     let(:article2) { create(:article) }
     let(:reaction1) { create(:reaction, category: "readinglist", reactable: article1) }

spec/services/search/tag_spec.rb

@@ -1,7 +1,7 @@
 require "rails_helper"
 
-RSpec.describe Search::Tag, type: :service, elasticsearch: true do
-  describe "::search_documents" do
+RSpec.describe Search::Tag, type: :service do
+  describe "::search_documents", elasticsearch: "Tag" do
     let(:tag_doc_1) { { "name" => "tag1" } }
     let(:tag_doc_2) { { "name" => "tag2" } }
     let(:mock_search_response) do

spec/services/search/user_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe Search::User, type: :service do
     expect(described_class::MAPPINGS).not_to be_nil
   end
 
-  describe "::search_documents", elasticsearch: true do
+  describe "::search_documents", elasticsearch: "User" do
     let(:user1) { create(:user) }
     let(:user2) { create(:user) }
 

spec/system/articles/user_visits_articles_by_timeframe_spec.rb

@@ -116,11 +116,10 @@ RSpec.describe "User visits articles by timeframe", type: :system do
     end
   end
 
-  context "when user has logged in", js: true do
+  context "when user has logged in", js: true, elasticsearch: "FeedContent" do
     let(:user) { create(:user) }
 
     before do
-      clear_elasticsearch_data(Search::FeedContent)
       sign_in user
       visit "/top/week"
     end

spec/workers/search/bulk_index_worker_spec.rb

@@ -1,12 +1,12 @@
 require "rails_helper"
 
-RSpec.describe Search::BulkIndexWorker, type: :worker, elasticsearch: true do
+RSpec.describe Search::BulkIndexWorker, type: :worker do
   let(:worker) { subject }
   let(:article) { create(:article) }
 
   include_examples "#enqueues_on_correct_queue", "high_priority", ["Reaction", 1]
 
-  it "indexes documents for a set of given ids and object class" do
+  it "indexes documents for a set of given ids and object class", elasticsearch: "Reaction" do
     reactions = [create(:reaction, reactable: article), create(:reaction), create(:reaction)]
     Sidekiq::Worker.clear_all
 

spec/workers/search/index_worker_spec.rb

@@ -1,6 +1,6 @@
 require "rails_helper"
 
-RSpec.describe Search::IndexWorker, type: :worker, elasticsearch: true do
+RSpec.describe Search::IndexWorker, type: :worker, elasticsearch: "Tag" do
   let(:worker) { subject }
 
   include_examples "#enqueues_on_correct_queue", "high_priority", ["Tag", 1]