From 60b7572f5c6ec56835a4f6ef9cc45d0c1e2d59b5 Mon Sep 17 00:00:00 2001
From: Molly Struve <mollylbs@gmail.com>
Date: Thu, 12 Nov 2020 11:59:54 -0500
Subject: [PATCH] Feature:Send Alerts for Failed DataUpdateScripts to Datadog
 (#11381)

---
 .../check_data_update_script_statuses.rb      | 17 ++++++++++++++
 config/schedule.yml                           |  3 +++
 spec/factories/data_update_script.rb          |  2 +-
 .../check_data_update_script_statuses_spec.rb | 22 +++++++++++++++++++
 4 files changed, 43 insertions(+), 1 deletion(-)
 create mode 100644 app/workers/metrics/check_data_update_script_statuses.rb
 create mode 100644 spec/workers/metrics/check_data_update_script_statuses_spec.rb

diff --git a/app/workers/metrics/check_data_update_script_statuses.rb b/app/workers/metrics/check_data_update_script_statuses.rb
new file mode 100644
index 000000000..0f88a7cbd
--- /dev/null
+++ b/app/workers/metrics/check_data_update_script_statuses.rb
@@ -0,0 +1,17 @@
+module Metrics
+  class CheckDataUpdateScriptStatuses
+    include Sidekiq::Worker
+    sidekiq_options queue: :low_priority, retry: 10
+
+    def perform
+      failed_scripts = DataUpdateScript.failed.where(created_at: 1.week.ago..Time.current)
+      failed_scripts.find_each do |script|
+        DatadogStatsClient.count(
+          "data_update_scripts.failures",
+          1,
+          tags: ["file_name:#{script.file_name}"],
+        )
+      end
+    end
+  end
+end
diff --git a/config/schedule.yml b/config/schedule.yml
index 7a13ea24a..3ef74b361 100644
--- a/config/schedule.yml
+++ b/config/schedule.yml
@@ -16,6 +16,9 @@ record_daily_notifications:
 record_data_counts:
   cron: "10 * * * *" # every hour, 10 min after the hour
   class: "Metrics::RecordDataCountsWorker"
+check_data_update_script_statuses:
+  cron: "30 * * * *" # 30 min after every hour
+  class: "Metrics::CheckDataUpdateScriptStatuses"
 award_yearly_club_badges:
   cron: "0 0 * * *" # daily at 12 am UTC
   class: "BadgeAchievements::BadgeAwardWorker"
diff --git a/spec/factories/data_update_script.rb b/spec/factories/data_update_script.rb
index 3ea50cb3b..10ac611a7 100644
--- a/spec/factories/data_update_script.rb
+++ b/spec/factories/data_update_script.rb
@@ -1,5 +1,5 @@
 FactoryBot.define do
   factory :data_update_script do
-    file_name { "20200214151804_data_update_test_script" }
+    file_name { |n| "20200214151804_data_update_test_script#{n}" }
   end
 end
diff --git a/spec/workers/metrics/check_data_update_script_statuses_spec.rb b/spec/workers/metrics/check_data_update_script_statuses_spec.rb
new file mode 100644
index 000000000..a299193c0
--- /dev/null
+++ b/spec/workers/metrics/check_data_update_script_statuses_spec.rb
@@ -0,0 +1,22 @@
+require "rails_helper"
+
+RSpec.describe Metrics::CheckDataUpdateScriptStatuses, type: :worker do
+  include_examples "#enqueues_on_correct_queue", "low_priority", 1
+
+  describe "#perform" do
+    it "logs recently failed script" do
+      create(:data_update_script)
+      create(:data_update_script, status: :failed, created_at: 1.month.ago)
+      failed_script = create(:data_update_script, status: :failed)
+      allow(DatadogStatsClient).to receive(:count)
+      described_class.new.perform
+
+      expect(DatadogStatsClient).to have_received(:count).once
+      expect(
+        DatadogStatsClient,
+      ).to have_received(:count).with(
+        "data_update_scripts.failures", 1, { tags: ["file_name:#{failed_script.file_name}"] }
+      ).at_least(1)
+    end
+  end
+end