Add Code To Setup the Elasticsearch Tag Index (#5919) [deploy]

.travis.yml

@@ -15,7 +15,6 @@ addons:
       - $(ls tmp/screenshots/*.png | tr "\n" ":")
     debug: true
 services:
-  - elasticsearch
   - redis-server
 env:
   global:
@@ -37,6 +36,14 @@ branches:
 before_install:
   - date --rfc-3339=seconds
   - nvm install
+  - pkill -9 -f elasticsearch || true
+  - curl -s -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.5.2-amd64.deb
+  - sudo dpkg -i --force-confnew elasticsearch-7.5.2-amd64.deb
+  - sudo sed -i.old 's/-Xms1g/-Xms128m/' /etc/elasticsearch/jvm.options
+  - sudo sed -i.old 's/-Xmx1g/-Xmx128m/' /etc/elasticsearch/jvm.options
+  - echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlog4j.skipJansi=true\n-server\n' | sudo tee -a /etc/elasticsearch/jvm.options
+  - sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
+  - sudo systemctl start elasticsearch
 install:
   - bundle install --path vendor/bundle
   - yarn install

app/services/search/cluster.rb

@@ -0,0 +1,44 @@
+module Search
+  class Cluster
+    SEARCH_CLASSES = [Search::Tag].freeze
+
+    class << self
+      def recreate_indexes
+        delete_indexes
+        setup_indexes
+      end
+
+      def setup_indexes
+        create_indexes
+        add_aliases
+        update_mappings
+      end
+
+      def create_indexes
+        SEARCH_CLASSES.each do |search_class|
+          next if SearchClient.indices.exists(index: search_class::INDEX_NAME)
+
+          search_class.create_index
+        end
+      end
+
+      def add_aliases
+        SEARCH_CLASSES.each(&:add_alias)
+      end
+
+      def update_mappings
+        SEARCH_CLASSES.each(&:update_mappings)
+      end
+
+      def delete_indexes
+        return if Rails.env.production?
+
+        SEARCH_CLASSES.each do |search_class|
+          next unless SearchClient.indices.exists(index: search_class::INDEX_NAME)
+
+          search_class.delete_index
+        end
+      end
+    end
+  end
+end

app/services/search/tag.rb

@@ -0,0 +1,84 @@
+module Search
+  class Tag
+    INDEX_NAME = "tags_#{Rails.env}".freeze
+    INDEX_ALIAS = "tags_#{Rails.env}_alias".freeze
+
+    class << self
+      def index(tag_id, serialized_data)
+        SearchClient.index(
+          id: tag_id,
+          index: INDEX_ALIAS,
+          body: serialized_data,
+        )
+      end
+
+      def get(tag_id)
+        SearchClient.get(id: tag_id, index: INDEX_ALIAS)
+      end
+
+      def create_index(index_name: INDEX_NAME)
+        SearchClient.indices.create(index: index_name, body: settings)
+      end
+
+      def delete_index(index_name: INDEX_NAME)
+        SearchClient.indices.delete(index: index_name)
+      end
+
+      def add_alias(index_name: INDEX_NAME, index_alias: INDEX_ALIAS)
+        SearchClient.indices.put_alias(index: index_name, name: index_alias)
+      end
+
+      def update_mappings(index_alias: INDEX_ALIAS)
+        SearchClient.indices.put_mapping(index: index_alias, body: mappings)
+      end
+
+      private
+
+      def settings
+        { settings: { index: index_settings } }
+      end
+
+      def index_settings
+        if Rails.env.production?
+          {
+            number_of_shards: 1,
+            number_of_replicas: 1
+          }
+        else
+          {
+            number_of_shards: 1,
+            number_of_replicas: 0
+          }
+        end
+      end
+
+      def mappings
+        {
+          dynamic: "strict",
+          properties: {
+            id: {
+              type: "keyword"
+            },
+            name: {
+              type: "text",
+              fields: {
+                raw: {
+                  type: "keyword"
+                }
+              }
+            },
+            hotness_score: {
+              type: "integer"
+            },
+            supported: {
+              type: "boolean"
+            },
+            short_summary: {
+              type: "text"
+            }
+          }
+        }
+      end
+    end
+  end
+end

lib/tasks/search.rake

@@ -0,0 +1,21 @@
+namespace :search do
+  desc "set up Elasticsearch indexes"
+  task setup: :environment do
+    Search::Cluster.setup_indexes
+  end
+
+  desc "update Elasticsearch index mappings"
+  task update_mappings: :environment do
+     Search::Cluster.update_mappings
+  end
+
+  desc "tear down Elasticsearch indexes"
+  task destroy: :environment do
+    if Rails.env.production?
+      puts "Will NOT destroy indexes in production"
+      exit
+    end
+
+    Search::Cluster.destroy_indexes
+  end
+end

spec/rails_helper.rb

@@ -84,6 +84,11 @@ RSpec.configure do |config|
     Sidekiq::Worker.clear_all # worker jobs shouldn't linger around between tests
   end
 
+  config.around(:each, elasticsearch: true) do |example|
+    Search::Cluster.recreate_indexes
+    example.run
+  end
+
   config.after do
     SiteConfig.clear_cache
   end

spec/services/search/cluster_spec.rb

@@ -0,0 +1,69 @@
+require "rails_helper"
+
+RSpec.describe Search::Cluster, type: :service do
+  describe "::recreate_indexes" do
+    it "destroys and sets up indexes" do
+      allow(described_class).to receive(:delete_indexes)
+      allow(described_class).to receive(:setup_indexes)
+
+      described_class.recreate_indexes
+      expect(described_class).to have_received(:delete_indexes)
+      expect(described_class).to have_received(:setup_indexes)
+    end
+  end
+
+  describe "::setup_indexes" do
+    it "creates, adds aliases and updates mappings for indexes" do
+      allow(described_class).to receive(:create_indexes)
+      allow(described_class).to receive(:add_aliases)
+      allow(described_class).to receive(:update_mappings)
+
+      described_class.setup_indexes
+      expect(described_class).to have_received(:create_indexes)
+      expect(described_class).to have_received(:add_aliases)
+      expect(described_class).to have_received(:update_mappings)
+    end
+  end
+
+  describe "::create_indexes" do
+    it "calls create_index for each search class" do
+      allow(SearchClient.indices).to receive(:exists).and_return(false)
+      described_class::SEARCH_CLASSES.each do |search_class|
+        allow(search_class).to receive(:create_index)
+      end
+      described_class.create_indexes
+      expect(described_class::SEARCH_CLASSES).to all(have_received(:create_index))
+    end
+  end
+
+  describe "::delete_indexes" do
+    it "calls delete_index for each search class" do
+      allow(SearchClient.indices).to receive(:exists).and_return(true)
+      described_class::SEARCH_CLASSES.each do |search_class|
+        allow(search_class).to receive(:delete_index)
+      end
+      described_class.delete_indexes
+      expect(described_class::SEARCH_CLASSES).to all(have_received(:delete_index))
+    end
+  end
+
+  describe "::add_aliases" do
+    it "calls add_alias for each search class" do
+      described_class::SEARCH_CLASSES.each do |search_class|
+        allow(search_class).to receive(:add_alias)
+      end
+      described_class.add_aliases
+      expect(described_class::SEARCH_CLASSES).to all(have_received(:add_alias))
+    end
+  end
+
+  describe "::update_mappings" do
+    it "calls update_mappings for each search class" do
+      described_class::SEARCH_CLASSES.each do |search_class|
+        allow(search_class).to receive(:update_mappings)
+      end
+      described_class.update_mappings
+      expect(described_class::SEARCH_CLASSES).to all(have_received(:update_mappings))
+    end
+  end
+end

spec/services/search/tag_spec.rb

@@ -0,0 +1,88 @@
+require "rails_helper"
+
+RSpec.describe Search::Tag, type: :service, elasticsearch: true do
+  describe "::index" do
+    it "indexes a tag to elasticsearch" do
+      tag = FactoryBot.create(:tag)
+      expect { described_class.get(tag.id) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      described_class.index(tag.id, id: tag.id)
+      expect(described_class.get(tag.id)).not_to be_nil
+    end
+  end
+
+  describe "::get" do
+    it "fetches a document for a given ID from elasticsearch" do
+      tag = FactoryBot.create(:tag)
+      described_class.index(tag.id, id: tag.id)
+      expect { described_class.get(tag.id) }.not_to raise_error
+    end
+  end
+
+  describe "::create_index" do
+    it "creates an elasticsearch index with INDEX_NAME" do
+      described_class.delete_index
+      expect(SearchClient.indices.exists(index: described_class::INDEX_NAME)).to eq(false)
+      described_class.create_index
+      expect(SearchClient.indices.exists(index: described_class::INDEX_NAME)).to eq(true)
+    end
+
+    it "creates an elasticsearch index with name argument" do
+      other_name = "random"
+      expect(SearchClient.indices.exists(index: other_name)).to eq(false)
+      described_class.create_index(index_name: other_name)
+      expect(SearchClient.indices.exists(index: other_name)).to eq(true)
+
+      # Have to cleanup index since it wont automatically be handled by our cluster class bc of the unexpected name
+      described_class.delete_index(index_name: other_name)
+    end
+  end
+
+  describe "::delete_index" do
+    it "deletes an elasticsearch index with INDEX_NAME" do
+      expect(SearchClient.indices.exists(index: described_class::INDEX_NAME)).to eq(true)
+      described_class.delete_index
+      expect(SearchClient.indices.exists(index: described_class::INDEX_NAME)).to eq(false)
+    end
+
+    it "deletes an elasticsearch index with name argument" do
+      other_name = "random"
+      described_class.create_index(index_name: other_name)
+      expect(SearchClient.indices.exists(index: other_name)).to eq(true)
+
+      described_class.delete_index(index_name: other_name)
+      expect(SearchClient.indices.exists(index: other_name)).to eq(false)
+    end
+  end
+
+  describe "::add_alias" do
+    it "adds alias INDEX_ALIAS to elasticsearch index with INDEX_NAME" do
+      SearchClient.indices.delete_alias(index: described_class::INDEX_NAME, name: described_class::INDEX_ALIAS)
+      expect(SearchClient.indices.exists(index: described_class::INDEX_ALIAS)).to eq(false)
+      described_class.add_alias
+      expect(SearchClient.indices.exists(index: described_class::INDEX_ALIAS)).to eq(true)
+    end
+
+    it "adds custom alias to elasticsearch index with INDEX_NAME" do
+      other_alias = "random"
+      expect(SearchClient.indices.exists(index: other_alias)).to eq(false)
+      described_class.add_alias(index_name: described_class::INDEX_NAME, index_alias: other_alias)
+      expect(SearchClient.indices.exists(index: other_alias)).to eq(true)
+    end
+  end
+
+  describe "::update_mappings" do
+    it "updates index mappings for tag index", :aggregate_failures do
+      other_name = "random"
+      described_class.create_index(index_name: other_name)
+      initial_mapping = SearchClient.indices.get_mapping(index: other_name).dig(other_name, "mappings")
+      expect(initial_mapping).to be_empty
+
+      described_class.update_mappings(index_alias: other_name)
+      mapping = SearchClient.indices.get_mapping(index: other_name).dig(other_name, "mappings")
+      expect(mapping.deep_stringify_keys).to include(described_class.send("mappings").deep_stringify_keys)
+
+      # Have to cleanup index since it wont automatically be handled by our cluster class bc of the unexpected name
+      described_class.delete_index(index_name: other_name)
+    end
+  end
+end