diff --git a/.travis.yml b/.travis.yml index 5856cf2be..7c714dd37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ addons: - $(ls tmp/screenshots/*.png | tr "\n" ":") debug: true services: - - elasticsearch - redis-server env: global: @@ -37,6 +36,14 @@ branches: before_install: - date --rfc-3339=seconds - nvm install + - pkill -9 -f elasticsearch || true + - curl -s -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.5.2-amd64.deb + - sudo dpkg -i --force-confnew elasticsearch-7.5.2-amd64.deb + - sudo sed -i.old 's/-Xms1g/-Xms128m/' /etc/elasticsearch/jvm.options + - sudo sed -i.old 's/-Xmx1g/-Xmx128m/' /etc/elasticsearch/jvm.options + - echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlog4j.skipJansi=true\n-server\n' | sudo tee -a /etc/elasticsearch/jvm.options + - sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch + - sudo systemctl start elasticsearch install: - bundle install --path vendor/bundle - yarn install diff --git a/app/services/search/cluster.rb b/app/services/search/cluster.rb new file mode 100644 index 000000000..290202a11 --- /dev/null +++ b/app/services/search/cluster.rb @@ -0,0 +1,44 @@ +module Search + class Cluster + SEARCH_CLASSES = [Search::Tag].freeze + + class << self + def recreate_indexes + delete_indexes + setup_indexes + end + + def setup_indexes + create_indexes + add_aliases + update_mappings + end + + def create_indexes + SEARCH_CLASSES.each do |search_class| + next if SearchClient.indices.exists(index: search_class::INDEX_NAME) + + search_class.create_index + end + end + + def add_aliases + SEARCH_CLASSES.each(&:add_alias) + end + + def update_mappings + SEARCH_CLASSES.each(&:update_mappings) + end + + def delete_indexes + return if Rails.env.production? + + SEARCH_CLASSES.each do |search_class| + next unless SearchClient.indices.exists(index: search_class::INDEX_NAME) + + search_class.delete_index + end + end + end + end +end diff --git a/app/services/search/tag.rb b/app/services/search/tag.rb new file mode 100644 index 000000000..fd3b73e6d --- /dev/null +++ b/app/services/search/tag.rb @@ -0,0 +1,84 @@ +module Search + class Tag + INDEX_NAME = "tags_#{Rails.env}".freeze + INDEX_ALIAS = "tags_#{Rails.env}_alias".freeze + + class << self + def index(tag_id, serialized_data) + SearchClient.index( + id: tag_id, + index: INDEX_ALIAS, + body: serialized_data, + ) + end + + def get(tag_id) + SearchClient.get(id: tag_id, index: INDEX_ALIAS) + end + + def create_index(index_name: INDEX_NAME) + SearchClient.indices.create(index: index_name, body: settings) + end + + def delete_index(index_name: INDEX_NAME) + SearchClient.indices.delete(index: index_name) + end + + def add_alias(index_name: INDEX_NAME, index_alias: INDEX_ALIAS) + SearchClient.indices.put_alias(index: index_name, name: index_alias) + end + + def update_mappings(index_alias: INDEX_ALIAS) + SearchClient.indices.put_mapping(index: index_alias, body: mappings) + end + + private + + def settings + { settings: { index: index_settings } } + end + + def index_settings + if Rails.env.production? + { + number_of_shards: 1, + number_of_replicas: 1 + } + else + { + number_of_shards: 1, + number_of_replicas: 0 + } + end + end + + def mappings + { + dynamic: "strict", + properties: { + id: { + type: "keyword" + }, + name: { + type: "text", + fields: { + raw: { + type: "keyword" + } + } + }, + hotness_score: { + type: "integer" + }, + supported: { + type: "boolean" + }, + short_summary: { + type: "text" + } + } + } + end + end + end +end diff --git a/lib/tasks/search.rake b/lib/tasks/search.rake new file mode 100644 index 000000000..f99efbd9d --- /dev/null +++ b/lib/tasks/search.rake @@ -0,0 +1,21 @@ +namespace :search do + desc "set up Elasticsearch indexes" + task setup: :environment do + Search::Cluster.setup_indexes + end + + desc "update Elasticsearch index mappings" + task update_mappings: :environment do + Search::Cluster.update_mappings + end + + desc "tear down Elasticsearch indexes" + task destroy: :environment do + if Rails.env.production? + puts "Will NOT destroy indexes in production" + exit + end + + Search::Cluster.destroy_indexes + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index cdac5084f..3a580da96 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -84,6 +84,11 @@ RSpec.configure do |config| Sidekiq::Worker.clear_all # worker jobs shouldn't linger around between tests end + config.around(:each, elasticsearch: true) do |example| + Search::Cluster.recreate_indexes + example.run + end + config.after do SiteConfig.clear_cache end diff --git a/spec/services/search/cluster_spec.rb b/spec/services/search/cluster_spec.rb new file mode 100644 index 000000000..123c614b9 --- /dev/null +++ b/spec/services/search/cluster_spec.rb @@ -0,0 +1,69 @@ +require "rails_helper" + +RSpec.describe Search::Cluster, type: :service do + describe "::recreate_indexes" do + it "destroys and sets up indexes" do + allow(described_class).to receive(:delete_indexes) + allow(described_class).to receive(:setup_indexes) + + described_class.recreate_indexes + expect(described_class).to have_received(:delete_indexes) + expect(described_class).to have_received(:setup_indexes) + end + end + + describe "::setup_indexes" do + it "creates, adds aliases and updates mappings for indexes" do + allow(described_class).to receive(:create_indexes) + allow(described_class).to receive(:add_aliases) + allow(described_class).to receive(:update_mappings) + + described_class.setup_indexes + expect(described_class).to have_received(:create_indexes) + expect(described_class).to have_received(:add_aliases) + expect(described_class).to have_received(:update_mappings) + end + end + + describe "::create_indexes" do + it "calls create_index for each search class" do + allow(SearchClient.indices).to receive(:exists).and_return(false) + described_class::SEARCH_CLASSES.each do |search_class| + allow(search_class).to receive(:create_index) + end + described_class.create_indexes + expect(described_class::SEARCH_CLASSES).to all(have_received(:create_index)) + end + end + + describe "::delete_indexes" do + it "calls delete_index for each search class" do + allow(SearchClient.indices).to receive(:exists).and_return(true) + described_class::SEARCH_CLASSES.each do |search_class| + allow(search_class).to receive(:delete_index) + end + described_class.delete_indexes + expect(described_class::SEARCH_CLASSES).to all(have_received(:delete_index)) + end + end + + describe "::add_aliases" do + it "calls add_alias for each search class" do + described_class::SEARCH_CLASSES.each do |search_class| + allow(search_class).to receive(:add_alias) + end + described_class.add_aliases + expect(described_class::SEARCH_CLASSES).to all(have_received(:add_alias)) + end + end + + describe "::update_mappings" do + it "calls update_mappings for each search class" do + described_class::SEARCH_CLASSES.each do |search_class| + allow(search_class).to receive(:update_mappings) + end + described_class.update_mappings + expect(described_class::SEARCH_CLASSES).to all(have_received(:update_mappings)) + end + end +end diff --git a/spec/services/search/tag_spec.rb b/spec/services/search/tag_spec.rb new file mode 100644 index 000000000..c1fc2032c --- /dev/null +++ b/spec/services/search/tag_spec.rb @@ -0,0 +1,88 @@ +require "rails_helper" + +RSpec.describe Search::Tag, type: :service, elasticsearch: true do + describe "::index" do + it "indexes a tag to elasticsearch" do + tag = FactoryBot.create(:tag) + expect { described_class.get(tag.id) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + described_class.index(tag.id, id: tag.id) + expect(described_class.get(tag.id)).not_to be_nil + end + end + + describe "::get" do + it "fetches a document for a given ID from elasticsearch" do + tag = FactoryBot.create(:tag) + described_class.index(tag.id, id: tag.id) + expect { described_class.get(tag.id) }.not_to raise_error + end + end + + describe "::create_index" do + it "creates an elasticsearch index with INDEX_NAME" do + described_class.delete_index + expect(SearchClient.indices.exists(index: described_class::INDEX_NAME)).to eq(false) + described_class.create_index + expect(SearchClient.indices.exists(index: described_class::INDEX_NAME)).to eq(true) + end + + it "creates an elasticsearch index with name argument" do + other_name = "random" + expect(SearchClient.indices.exists(index: other_name)).to eq(false) + described_class.create_index(index_name: other_name) + expect(SearchClient.indices.exists(index: other_name)).to eq(true) + + # Have to cleanup index since it wont automatically be handled by our cluster class bc of the unexpected name + described_class.delete_index(index_name: other_name) + end + end + + describe "::delete_index" do + it "deletes an elasticsearch index with INDEX_NAME" do + expect(SearchClient.indices.exists(index: described_class::INDEX_NAME)).to eq(true) + described_class.delete_index + expect(SearchClient.indices.exists(index: described_class::INDEX_NAME)).to eq(false) + end + + it "deletes an elasticsearch index with name argument" do + other_name = "random" + described_class.create_index(index_name: other_name) + expect(SearchClient.indices.exists(index: other_name)).to eq(true) + + described_class.delete_index(index_name: other_name) + expect(SearchClient.indices.exists(index: other_name)).to eq(false) + end + end + + describe "::add_alias" do + it "adds alias INDEX_ALIAS to elasticsearch index with INDEX_NAME" do + SearchClient.indices.delete_alias(index: described_class::INDEX_NAME, name: described_class::INDEX_ALIAS) + expect(SearchClient.indices.exists(index: described_class::INDEX_ALIAS)).to eq(false) + described_class.add_alias + expect(SearchClient.indices.exists(index: described_class::INDEX_ALIAS)).to eq(true) + end + + it "adds custom alias to elasticsearch index with INDEX_NAME" do + other_alias = "random" + expect(SearchClient.indices.exists(index: other_alias)).to eq(false) + described_class.add_alias(index_name: described_class::INDEX_NAME, index_alias: other_alias) + expect(SearchClient.indices.exists(index: other_alias)).to eq(true) + end + end + + describe "::update_mappings" do + it "updates index mappings for tag index", :aggregate_failures do + other_name = "random" + described_class.create_index(index_name: other_name) + initial_mapping = SearchClient.indices.get_mapping(index: other_name).dig(other_name, "mappings") + expect(initial_mapping).to be_empty + + described_class.update_mappings(index_alias: other_name) + mapping = SearchClient.indices.get_mapping(index: other_name).dig(other_name, "mappings") + expect(mapping.deep_stringify_keys).to include(described_class.send("mappings").deep_stringify_keys) + + # Have to cleanup index since it wont automatically be handled by our cluster class bc of the unexpected name + described_class.delete_index(index_name: other_name) + end + end +end