Add Additional Search Fields and Search Highlighting (#6734) [deploy]

app/controllers/search_controller.rb

@@ -25,6 +25,7 @@ class SearchController < ApplicationController
     sort_by
     tag_names
     user_id
+    class_name
   ].freeze
 
   def tags

app/serializers/search/article_serializer.rb

@@ -16,6 +16,8 @@ module Search
     attribute :video_duration_string, &:video_duration_in_minutes
     attribute :video_duration_in_minutes, &:video_duration_in_minutes_integer
 
+    attribute :readable_publish_date_string, &:readable_publish_date
+
     attribute :flare_tag_hash, if: proc { |a| a.flare_tag.present? }, &:flare_tag
 
     attribute :tags do |article|

app/serializers/search/podcast_episode_serializer.rb

@@ -5,11 +5,16 @@ module Search
     attribute :id, &:search_id
 
     attributes :body_text, :class_name, :comments_count,
-               :featured, :featured_number, :hotness_score, :main_image, :path,
+               :featured, :featured_number, :hotness_score, :path,
                :positive_reactions_count, :published, :published_at, :quote,
                :reactions_count, :search_score, :subtitle, :summary, :title,
                :website_url
 
+    attribute :main_image do |pde|
+      ProfileImage.new(pde.podcast).get(width: 90)
+    end
+    attribute :slug, &:podcast_slug
+
     attribute :tags do |pde|
       pde.tags.map do |tag|
         { name: tag.name, keywords_for_search: tag.keywords_for_search }

app/services/search/feed_content.rb

@@ -13,13 +13,21 @@ module Search
 
         results = search(body: query_hash)
         hits = results.dig("hits", "hits").map do |feed_doc|
-          feed_doc.dig("_source")
+          prepare_doc(feed_doc)
         end
         paginate_hits(hits, params)
       end
 
       private
 
+      def prepare_doc(hit)
+        source = hit.dig("_source")
+        source["tag_list"] = hit["tags"]&.map { |t| t["name"] } || []
+        source["user_id"] = hit.dig("_source", "user", "id")
+        source["highlight"] = hit["highlight"]
+        source
+      end
+
       def index_settings
         if Rails.env.production?
           {

app/services/search/query_builders/feed_content.rb

@@ -1,10 +1,20 @@
 module Search
   module QueryBuilders
     class FeedContent < QueryBase
-      QUERY_KEYS = %i[
-        search_fields
-      ].freeze
+      # In order for highlighting to work properly we have to search the fields we want to highlight
+      QUERY_KEYS = {
+        search_fields: [
+          "tags.*",
+          "body_text",
+          "title",
+          "user.name",
+          "user.username",
+          "organization.name",
+        ]
+      }.freeze
 
+      # Search keys from our controllers may not match what we have stored in Elasticsearch so we map them here,
+      # this allows us to change our Elasticsearch docs without worrying about the frontend
       TERM_KEYS = {
         tag_names: "tags.name",
         approved: "approved",
@@ -22,6 +32,10 @@ module Search
         size: 0
       }.freeze
 
+      HIGHLIGHT_FIELDS = %w[
+        body_text
+      ].freeze
+
       attr_accessor :params, :body
 
       def initialize(params)
@@ -31,6 +45,16 @@ module Search
 
       private
 
+      def add_highlight_fields
+        highlight_fields = { fields: {} }
+        HIGHLIGHT_FIELDS.each do |field_name|
+          # This hash can be filled with options to further customize our highlighting
+          # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#request-body-search-highlighting
+          highlight_fields[:fields][field_name] = { order: :score }
+        end
+        @body[:highlight] = highlight_fields
+      end
+
       def build_queries
         @body[:query] = { bool: {} }
         @body[:query][:bool][:filter] = filter_conditions if filter_keys_present?
@@ -62,7 +86,7 @@ module Search
         TERM_KEYS.map do |term_key, search_key|
           next unless @params.key? term_key
 
-          { term: { search_key => @params[term_key] } }
+          { terms: { search_key => Array.wrap(@params[term_key]) } }
         end.compact
       end
 

app/services/search/query_builders/query_base.rb

@@ -14,8 +14,11 @@ module Search
         build_queries
         add_sort
         set_size
+        add_highlight_fields
       end
 
+      def add_highlight_fields; end
+
       def add_sort
         sort_key = @params[:sort_by] || self.class::DEFAULT_PARAMS[:sort_by]
         sort_direction = @params[:sort_direction] || self.class::DEFAULT_PARAMS[:sort_direction]
@@ -30,17 +33,19 @@ module Search
       end
 
       def query_keys_present?
-        self.class::QUERY_KEYS.detect { |key| @params[key].present? }
+        self.class::QUERY_KEYS.detect { |key, _| @params[key].present? }
       end
 
       def query_conditions
-        self.class::QUERY_KEYS.map do |query_key|
+        self.class::QUERY_KEYS.map do |query_key, query_fields|
           next if @params[query_key].blank?
 
+          fields = query_fields.presence || [query_key]
+
           {
             simple_query_string: {
               query: "#{@params[query_key]}*",
-              fields: [query_key],
+              fields: fields,
               lenient: true,
               analyze_wildcard: true
             }

config/elasticsearch/mappings/feed_content.json

@@ -99,6 +99,9 @@
     "readable_publish_date": {
       "type": "date"
     },
+    "readable_publish_date_string": {
+      "type": "keyword"
+    },
     "reading_time": {
       "type": "integer"
     },
@@ -111,6 +114,9 @@
     "search_score": {
       "type": "integer"
     },
+    "slug": {
+      "type": "keyword"
+    },
     "subtitle": {
       "type": "text"
     },

spec/services/search/feed_content_spec.rb

@@ -18,6 +18,18 @@ RSpec.describe Search::FeedContent, type: :service do
       expect(described_class).to have_received(:search).with(body: a_kind_of(Hash))
     end
 
+    it "returns highlighted fields" do
+      allow(article1).to receive(:body_text).and_return("I love ruby")
+      allow(article2).to receive(:body_text).and_return("Ruby Tuesday is yummy")
+      index_documents([article1, article2])
+      query_params = { size: 5, search_fields: "love ruby" }
+
+      feed_docs = described_class.search_documents(params: query_params)
+      expect(feed_docs.count).to eq(2)
+      doc_highlights = feed_docs.map { |t| t.dig("highlight", "body_text") }.flatten
+      expect(doc_highlights).to include("I <em>love</em> <em>ruby</em>", "<em>Ruby</em> Tuesday is yummy")
+    end
+
     context "with a query" do
       it "searches by search_fields" do
         allow(article1).to receive(:title).and_return("ruby")

spec/services/search/query_builders/feed_content_spec.rb

@@ -15,12 +15,14 @@ RSpec.describe Search::QueryBuilders::FeedContent, type: :service do
   end
 
   describe "#as_hash" do
+    let(:query_fields) { described_class::QUERY_KEYS[:search_fields] }
+
     it "applies QUERY_KEYS from params" do
       params = { search_fields: "test" }
       filter = described_class.new(params)
       exepcted_query = [{
         "simple_query_string" => {
-          "query" => "test*", "fields" => [:search_fields], "lenient" => true, "analyze_wildcard" => true
+          "query" => "test*", "fields" => query_fields, "lenient" => true, "analyze_wildcard" => true
         }
       }]
       expect(filter.as_hash.dig("query", "bool", "must")).to match_array(exepcted_query)
@@ -30,10 +32,10 @@ RSpec.describe Search::QueryBuilders::FeedContent, type: :service do
       params = { approved: true, tag_names: "beginner", user_id: 777, class_name: "Article" }
       filter = described_class.new(params)
       exepcted_filters = [
-        { "term" => { "approved" => true } },
-        { "term" => { "tags.name" => "beginner" } },
-        { "term" => { "user.id" => 777 } },
-        { "term" => { "class_name" => "Article" } },
+        { "terms" => { "approved" => [true] } },
+        { "terms" => { "tags.name" => ["beginner"] } },
+        { "terms" => { "user.id" => [777] } },
+        { "terms" => { "class_name" => ["Article"] } },
       ]
       expect(filter.as_hash.dig("query", "bool", "filter")).to match_array(exepcted_filters)
     end
@@ -54,11 +56,11 @@ RSpec.describe Search::QueryBuilders::FeedContent, type: :service do
         params = { search_fields: "ruby", published_at: { lte: Time.current }, tag_names: "cfp" }
         filter = described_class.new(params)
         exepcted_query = [{
-          "simple_query_string" => { "query" => "ruby*", "fields" => [:search_fields], "lenient" => true, "analyze_wildcard" => true }
+          "simple_query_string" => { "query" => "ruby*", "fields" => query_fields, "lenient" => true, "analyze_wildcard" => true }
         }]
         exepcted_filters = [
           { "range" => { "published_at" => { lte: Time.current } } },
-          { "term" => { "tags.name" => "cfp" } },
+          { "terms" => { "tags.name" => ["cfp"] } },
         ]
         expect(filter.as_hash.dig("query", "bool", "must")).to match_array(exepcted_query)
         expect(filter.as_hash.dig("query", "bool", "filter")).to match_array(exepcted_filters)
@@ -70,7 +72,7 @@ RSpec.describe Search::QueryBuilders::FeedContent, type: :service do
       filter = described_class.new(params)
       exepcted_query = [{
         "simple_query_string" => {
-          "query" => "cfp*", "fields" => [:search_fields], "lenient" => true, "analyze_wildcard" => true
+          "query" => "cfp*", "fields" => query_fields, "lenient" => true, "analyze_wildcard" => true
         }
       }]
       expect(filter.as_hash.dig("query", "bool", "must")).to match_array(exepcted_query)