diff --git a/app/lib/honeycomb/noise_cancelling_sampler.rb b/app/lib/honeycomb/noise_cancelling_sampler.rb new file mode 100644 index 000000000..3d5d0db90 --- /dev/null +++ b/app/lib/honeycomb/noise_cancelling_sampler.rb @@ -0,0 +1,27 @@ +module Honeycomb + class NoiseCancellingSampler + extend Honeycomb::DeterministicSampler + + NOISY_COMMANDS = [ + "GET rails-settings-cached/v1", + "TIME", + "BEGIN", + "COMMIT", + ].freeze + + def self.sample(fields) + if NOISY_COMMANDS.include?(fields["redis.command"]) || NOISY_COMMANDS.include?(fields["sql.active_record.sql"]) + rate = 100 + [should_sample(rate, fields["trace.trace_id"]), rate] + elsif fields["redis.command"]&.start_with?("BRPOP") + rate = 1000 + [should_sample(rate, fields["trace.trace_id"]), rate] + elsif fields["redis.command"]&.start_with?("INCRBY") || fields["redis.command"]&.start_with?("TTL") + rate = 100 + [should_sample(rate, fields["trace.trace_id"]), rate] + else + [true, 1] + end + end + end +end diff --git a/app/services/search/client.rb b/app/services/search/client.rb index dffe07027..a73852b7d 100644 --- a/app/services/search/client.rb +++ b/app/services/search/client.rb @@ -33,7 +33,6 @@ module Search ].freeze def request - Honeycomb.add_field("name", "elasticsearch") yield rescue *TRANSPORT_EXCEPTIONS => e class_name = e.class.name.demodulize diff --git a/config/initializers/honeycomb.rb b/config/initializers/honeycomb.rb index 6f7365929..98e6636ea 100644 --- a/config/initializers/honeycomb.rb +++ b/config/initializers/honeycomb.rb @@ -32,6 +32,10 @@ else fields.delete("sql.active_record.datadog_span") end end + # Sample away highly redundant events + config.sample_hook do |fields| + Honeycomb::NoiseCancellingSampler.sample(fields) + end end end end diff --git a/spec/lib/honeycomb/noise_cancelling_sampler_spec.rb b/spec/lib/honeycomb/noise_cancelling_sampler_spec.rb new file mode 100644 index 000000000..463739ed5 --- /dev/null +++ b/spec/lib/honeycomb/noise_cancelling_sampler_spec.rb @@ -0,0 +1,39 @@ +require "rails_helper" + +RSpec.describe Honeycomb::NoiseCancellingSampler do + let(:trace_id) { "TRACE_ID" } + + before { allow(described_class).to receive(:should_sample) } + + context "without a noisy command" do + it "does not sample" do + result = described_class.sample({ "request.path" => "/me", "trace.trace_id" => trace_id }) + expect(described_class).not_to have_received(:should_sample) + expect(result).to eq([true, 1]) + end + end + + context "with a redis command" do + it "samples if its a NOISY_COMMANDS" do + described_class.sample({ "redis.command" => "TIME", "trace.trace_id" => trace_id }) + expect(described_class).to have_received(:should_sample).with(100, trace_id) + end + + it "samples if the command is BRPOP" do + described_class.sample({ "redis.command" => "BRPOP", "trace.trace_id" => trace_id }) + expect(described_class).to have_received(:should_sample).with(1000, trace_id) + end + + it "samples if the command starts with TTL" do + described_class.sample({ "redis.command" => "TTL this and that", "trace.trace_id" => trace_id }) + expect(described_class).to have_received(:should_sample).with(100, trace_id) + end + end + + context "with a active_record SQL command" do + it "samples if its a NOISY_COMMANDS" do + described_class.sample({ "sql.active_record.sql" => "COMMIT", "trace.trace_id" => trace_id }) + expect(described_class).to have_received(:should_sample).with(100, trace_id) + end + end +end