Feature:Send Alerts for Failed DataUpdateScripts to Datadog (#11381)

app/workers/metrics/check_data_update_script_statuses.rb

@@ -0,0 +1,17 @@
+module Metrics
+  class CheckDataUpdateScriptStatuses
+    include Sidekiq::Worker
+    sidekiq_options queue: :low_priority, retry: 10
+
+    def perform
+      failed_scripts = DataUpdateScript.failed.where(created_at: 1.week.ago..Time.current)
+      failed_scripts.find_each do |script|
+        DatadogStatsClient.count(
+          "data_update_scripts.failures",
+          1,
+          tags: ["file_name:#{script.file_name}"],
+        )
+      end
+    end
+  end
+end

config/schedule.yml

@@ -16,6 +16,9 @@ record_daily_notifications:
 record_data_counts:
   cron: "10 * * * *" # every hour, 10 min after the hour
   class: "Metrics::RecordDataCountsWorker"
+check_data_update_script_statuses:
+  cron: "30 * * * *" # 30 min after every hour
+  class: "Metrics::CheckDataUpdateScriptStatuses"
 award_yearly_club_badges:
   cron: "0 0 * * *" # daily at 12 am UTC
   class: "BadgeAchievements::BadgeAwardWorker"

spec/factories/data_update_script.rb

@@ -1,5 +1,5 @@
 FactoryBot.define do
   factory :data_update_script do
-    file_name { "20200214151804_data_update_test_script" }
+    file_name { |n| "20200214151804_data_update_test_script#{n}" }
   end
 end

spec/workers/metrics/check_data_update_script_statuses_spec.rb

@@ -0,0 +1,22 @@
+require "rails_helper"
+
+RSpec.describe Metrics::CheckDataUpdateScriptStatuses, type: :worker do
+  include_examples "#enqueues_on_correct_queue", "low_priority", 1
+
+  describe "#perform" do
+    it "logs recently failed script" do
+      create(:data_update_script)
+      create(:data_update_script, status: :failed, created_at: 1.month.ago)
+      failed_script = create(:data_update_script, status: :failed)
+      allow(DatadogStatsClient).to receive(:count)
+      described_class.new.perform
+
+      expect(DatadogStatsClient).to have_received(:count).once
+      expect(
+        DatadogStatsClient,
+      ).to have_received(:count).with(
+        "data_update_scripts.failures", 1, { tags: ["file_name:#{failed_script.file_name}"] }
+      ).at_least(1)
+    end
+  end
+end