Notify slack when release phase fails (#6300) [deploy]

Envfile

@@ -139,6 +139,7 @@ variable :RECAPTCHA_SITE, :String, default: "Optional"
 # Slack for customer alerts
 # (https://api.slack.com/)
 variable :SLACK_CHANNEL, :String, default: ""
+variable :SLACK_DEPLOY_CHANNEL, :String, default: ""
 variable :SLACK_WEBHOOK_URL, :String, default: ""
 
 # Stripe for payment system

app/lib/release_phase_notifier.rb

@@ -0,0 +1,15 @@
+class ReleasePhaseNotifier
+  def self.ping_slack
+    return unless ApplicationConfig["SLACK_WEBHOOK_URL"].present? && ApplicationConfig["SLACK_DEPLOY_CHANNEL"].present?
+
+    client = Slack::Notifier.new(
+      ApplicationConfig["SLACK_WEBHOOK_URL"],
+      channel: ApplicationConfig["SLACK_DEPLOY_CHANNEL"],
+      username: "Heroku",
+    )
+
+    client.ping("Release Phase Failed: #{ENV['FAILED_COMMAND']}")
+  rescue Slack::Notifier::APIError => e
+    Rails.logger.info("Slack API error occured: #{e.message}")
+  end
+end

release-tasks.sh

@@ -1,14 +1,22 @@
 #!/bin/bash
 
-# enable echo mode
-set -x
+notify () {
+  FAILED_COMMAND="$(caller): ${BASH_COMMAND}" \
+    bundle exec rails runner "ReleasePhaseNotifier.ping_slack"
+}
+
+trap notify ERR
+
+# enable echo mode (-x) and exit on error (-e)
+# -E ensures that ERR traps get inherited by functions, command substitutions, and subshell environments.
+set -Eex
 
 # abort release if deploy status equals "blocked"
 [[ $DEPLOY_STATUS = "blocked" ]] && echo "Deploy blocked" && exit 1
 
 # runs migration for Postgres, setups/updates Elasticsearch
 # and boots the app to check there are no errors
-STATEMENT_TIMEOUT=180000 bundle exec rails db:migrate && \
-  bundle exec rake search:setup && \
-  bundle exec rake data_updates:enqueue_data_update_worker && \
-  bundle exec rails runner "puts 'app load success'"
+STATEMENT_TIMEOUT=180000 bundle exec rails db:migrate
+bundle exec rake search:setup
+bundle exec rake data_updates:enqueue_data_update_worker
+bundle exec rails runner "puts 'app load success'"

spec/lib/release_phase_notifier_spec.rb

@@ -0,0 +1,34 @@
+require "rails_helper"
+
+RSpec.describe ReleasePhaseNotifier, type: :lib do
+  describe ".ping_slack" do
+    before do
+      allow(ApplicationConfig).to receive(:[]).with("SLACK_WEBHOOK_URL").and_return("url")
+      allow(ApplicationConfig).to receive(:[]).with("SLACK_DEPLOY_CHANNEL").and_return("channel")
+    end
+
+    it "sends a failure message to slack" do
+      mock_slack = Slack::Notifier.new("url")
+      ENV["FAILED_COMMAND"] = "rake db:migrate"
+      failure_message = "Release Phase Failed: #{ENV['FAILED_COMMAND']}"
+      allow(Slack::Notifier).to receive(:new) { mock_slack }
+      allow(mock_slack).to receive(:ping)
+
+      described_class.ping_slack
+      expect(mock_slack).to have_received(:ping).with(failure_message)
+    end
+
+    it "bails when config variables are missing" do
+      allow(ApplicationConfig).to receive(:[]).with("SLACK_WEBHOOK_URL").and_return("")
+      allow(ApplicationConfig).to receive(:[]).with("SLACK_DEPLOY_CHANNEL").and_return("")
+
+      expect { described_class.ping_slack }.not_to raise_error
+    end
+
+    it "rescues any Slack API Errors" do
+      allow(Slack::Notifier).to receive(:new).and_raise(Slack::Notifier::APIError.new)
+
+      expect { described_class.ping_slack }.not_to raise_error
+    end
+  end
+end