Log Elasticsearch index counts to Datadog (#6845) [deploy]

app/services/search/base.rb

@@ -37,6 +37,10 @@ module Search
         Search::Client.indices.put_mapping(index: index_alias, body: self::MAPPINGS)
       end
 
+      def document_count
+        Search::Client.count(index: self::INDEX_ALIAS).dig("count")
+      end
+
       private
 
       def search(body:)

app/workers/metrics/record_data_counts_worker.rb

@@ -1,5 +1,5 @@
 module Metrics
-  class RecordDbTableCountsWorker
+  class RecordDataCountsWorker
     include Sidekiq::Worker
     sidekiq_options queue: :low_priority, retry: 10
 
@@ -9,6 +9,11 @@ module Metrics
         estimate = model.estimated_count
         Rails.logger.info("db_table_size", table_info: { table_name: model.table_name, table_size: estimate })
         DatadogStatsClient.gauge("postgres.db_table_size", estimate, tags: { table_name: model.table_name })
+
+        next unless model.const_defined?(:SEARCH_CLASS)
+
+        document_count = model::SEARCH_CLASS.document_count
+        DatadogStatsClient.gauge("elasticsearch.document_count", document_count, tags: { table_name: model.table_name })
       end
     end
   end

lib/tasks/fetch.rake

@@ -117,8 +117,8 @@ task fix_credits_count_cache: :environment do
   Credit.counter_culture_fix_counts only: %i[user organization]
 end
 
-task record_db_table_counts: :environment do
-  Metrics::RecordDbTableCountsWorker.perform_async
+task record_data_counts: :environment do
+  Metrics::RecordDataCountsWorker.perform_async
 end
 
 task log_worker_queue_stats: :environment do

spec/workers/metrics/record_data_counts_worker_spec.rb

@@ -1,6 +1,6 @@
 require "rails_helper"
 
-RSpec.describe Metrics::RecordDbTableCountsWorker, type: :worker do
+RSpec.describe Metrics::RecordDataCountsWorker, type: :worker do
   default_logger = Rails.logger
 
   include_examples "#enqueues_on_correct_queue", "low_priority", 1
@@ -22,5 +22,14 @@ RSpec.describe Metrics::RecordDbTableCountsWorker, type: :worker do
         DatadogStatsClient,
       ).to have_received(:gauge).with("postgres.db_table_size", 0, Hash).at_least(1)
     end
+
+    it "logs index counts in Datadog" do
+      allow(DatadogStatsClient).to receive(:gauge)
+      described_class.new.perform
+
+      expect(
+        DatadogStatsClient,
+      ).to have_received(:gauge).with("elasticsearch.document_count", 0, Hash).at_least(1)
+    end
   end
 end