[deploy] Sample Noisy Honeycomb Spans (#7891)

app/lib/honeycomb/noise_cancelling_sampler.rb

@@ -0,0 +1,27 @@
+module Honeycomb
+  class NoiseCancellingSampler
+    extend Honeycomb::DeterministicSampler
+
+    NOISY_COMMANDS = [
+      "GET rails-settings-cached/v1",
+      "TIME",
+      "BEGIN",
+      "COMMIT",
+    ].freeze
+
+    def self.sample(fields)
+      if NOISY_COMMANDS.include?(fields["redis.command"]) || NOISY_COMMANDS.include?(fields["sql.active_record.sql"])
+        rate = 100
+        [should_sample(rate, fields["trace.trace_id"]), rate]
+      elsif fields["redis.command"]&.start_with?("BRPOP")
+        rate = 1000
+        [should_sample(rate, fields["trace.trace_id"]), rate]
+      elsif fields["redis.command"]&.start_with?("INCRBY") || fields["redis.command"]&.start_with?("TTL")
+        rate = 100
+        [should_sample(rate, fields["trace.trace_id"]), rate]
+      else
+        [true, 1]
+      end
+    end
+  end
+end

app/services/search/client.rb

@@ -33,7 +33,6 @@ module Search
       ].freeze
 
       def request
-        Honeycomb.add_field("name", "elasticsearch")
         yield
       rescue *TRANSPORT_EXCEPTIONS => e
         class_name = e.class.name.demodulize

config/initializers/honeycomb.rb

@@ -32,6 +32,10 @@ else
           fields.delete("sql.active_record.datadog_span")
         end
       end
+      # Sample away highly redundant events
+      config.sample_hook do |fields|
+        Honeycomb::NoiseCancellingSampler.sample(fields)
+      end
     end
   end
 end

spec/lib/honeycomb/noise_cancelling_sampler_spec.rb

@@ -0,0 +1,39 @@
+require "rails_helper"
+
+RSpec.describe Honeycomb::NoiseCancellingSampler do
+  let(:trace_id) { "TRACE_ID" }
+
+  before { allow(described_class).to receive(:should_sample) }
+
+  context "without a noisy command" do
+    it "does not sample" do
+      result = described_class.sample({ "request.path" => "/me", "trace.trace_id" => trace_id })
+      expect(described_class).not_to have_received(:should_sample)
+      expect(result).to eq([true, 1])
+    end
+  end
+
+  context "with a redis command" do
+    it "samples if its a NOISY_COMMANDS" do
+      described_class.sample({ "redis.command" => "TIME", "trace.trace_id" => trace_id })
+      expect(described_class).to have_received(:should_sample).with(100, trace_id)
+    end
+
+    it "samples if the command is BRPOP" do
+      described_class.sample({ "redis.command" => "BRPOP", "trace.trace_id" => trace_id })
+      expect(described_class).to have_received(:should_sample).with(1000, trace_id)
+    end
+
+    it "samples if the command starts with TTL" do
+      described_class.sample({ "redis.command" => "TTL this and that", "trace.trace_id" => trace_id })
+      expect(described_class).to have_received(:should_sample).with(100, trace_id)
+    end
+  end
+
+  context "with a active_record SQL command" do
+    it "samples if its a NOISY_COMMANDS" do
+      described_class.sample({ "sql.active_record.sql" => "COMMIT", "trace.trace_id" => trace_id })
+      expect(described_class).to have_received(:should_sample).with(100, trace_id)
+    end
+  end
+end