Set Up Framework For Running Data Update Scripts (#6025) [deploy]

.github/CODEOWNERS

@@ -25,6 +25,8 @@
 /config/                                            @thepracticaldev/sre
 /db/                                                @thepracticaldev/sre
 /docs/                                              @thepracticaldev/oss    @jacobherrington
+/lib/data_update_scripts/                           @thepracticaldev/sre
+/lib/sidekiq/                                       @thepracticaldev/sre
 /spec/requests/internal/                            @thepracticaldev/snackcase
 /spec/system/internal/                              @thepracticaldev/snackcase
 Gemfile                                             @thepracticaldev/sre    @thepracticaldev/oss

app/models/data_update_script.rb

@@ -0,0 +1,35 @@
+class DataUpdateScript < ApplicationRecord
+  DIRECTORY = Rails.root.join("lib/data_update_scripts").freeze
+  NAMESPACE = "DataUpdateScripts".freeze
+  STATUSES = { enqueued: 0, working: 1, succeeded: 2, failed: 3 }.freeze
+
+  enum status: STATUSES
+  validates :file_name, uniqueness: true
+
+  def self.filenames
+    Dir.glob("*.rb", base: DIRECTORY).map { |f| Pathname.new(f).basename(".rb").to_s }
+  end
+
+  def self.load_script_ids
+    filenames.
+      map { |file_name| find_or_create_by(file_name: file_name) }.
+      select(&:enqueued?).
+      map(&:id)
+  end
+
+  def mark_as_run!
+    update!(run_at: Time.now.utc, status: :working)
+  end
+
+  def mark_as_finished!
+    update!(finished_at: Time.now.utc, status: :succeeded)
+  end
+
+  def mark_as_failed!
+    update!(finished_at: Time.now.utc, status: :failed)
+  end
+
+  def file_class
+    "#{self.class::NAMESPACE}::#{file_name.camelcase}".constantize
+  end
+end

app/workers/data_update_worker.rb

@@ -0,0 +1,22 @@
+class DataUpdateWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: :high_priority, retry: 5
+
+  def perform
+    script_ids = DataUpdateScript.load_script_ids
+    scripts_to_run = DataUpdateScript.where(id: script_ids).select(&:enqueued?)
+
+    scripts_to_run.each do |script|
+      script.mark_as_run!
+      run_script(script)
+    end
+  end
+
+  def run_script(script)
+    script.file_class.new.run
+    script.mark_as_finished!
+  rescue StandardError => e
+    script.mark_as_failed!
+    Honeybadger.notify(e, context: { script_id: script.id })
+  end
+end

bin/setup

@@ -33,6 +33,9 @@ chdir APP_ROOT do
   puts "\n== Preparing Elasticsearch =="
   system! 'bin/rails search:setup'
 
+  puts "\n== Update Data =="
+  system! 'bin/rails data_updates:run'
+
   puts "\n== Removing old logs and tempfiles =="
   system! 'bin/rails log:clear tmp:clear'
 

bin/update

@@ -23,6 +23,9 @@ chdir APP_ROOT do
   puts "\n== Updating database =="
   system! 'bin/rails db:migrate'
 
+  puts "\n== Update Data =="
+  system! 'bin/rails data_updates:run'
+
   puts "\n== Update Elasticsearch =="
   system! 'bin/rails search:setup'
 

db/migrate/20200211192415_create_data_update_scripts.rb

@@ -0,0 +1,11 @@
+class CreateDataUpdateScripts < ActiveRecord::Migration[5.2]
+  def change
+    create_table :data_update_scripts do |t|
+      t.string :file_name, unique: true
+      t.integer :status, default: 0, null: false
+      t.timestamp :run_at
+      t.timestamp :finished_at
+      t.timestamps null: false
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_02_05_225813) do
+ActiveRecord::Schema.define(version: 2020_02_11_192415) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -350,6 +350,15 @@ ActiveRecord::Schema.define(version: 2020_02_05_225813) do
     t.index ["spent"], name: "index_credits_on_spent"
   end
 
+  create_table "data_update_scripts", force: :cascade do |t|
+    t.datetime "created_at", null: false
+    t.string "file_name"
+    t.datetime "finished_at"
+    t.datetime "run_at"
+    t.integer "status", default: 0, null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "delayed_jobs", id: :serial, force: :cascade do |t|
     t.integer "attempts", default: 0, null: false
     t.datetime "created_at"

lib/generators/data_update/USAGE

@@ -0,0 +1,17 @@
+Description:
+  Generates a data update script.
+
+Examples:
+  `rails generate data_update ExampleScript`
+
+    # lib/data_update_scripts/example_script.rb
+    module DataUpdateScripts
+      class ExampleScript
+        def run
+          # Place your data update logic here
+          # Make sure your code is idempotent and can be run safely
+          # multiple times at any time
+        end
+      end
+    end
+

lib/generators/data_update/data_update_generator.rb

@@ -0,0 +1,10 @@
+class DataUpdateGenerator < Rails::Generators::NamedBase
+  source_root File.expand_path("templates", __dir__)
+
+  def create_data_update_file
+    template(
+      "data_update.rb.tt",
+      File.join("lib", "data_update_scripts", class_path, "#{file_name}.rb"),
+    )
+  end
+end

lib/generators/data_update/templates/data_update.rb.tt

@@ -0,0 +1,11 @@
+<% module_namespacing do -%>
+module DataUpdateScripts
+  class <%= class_name %>
+    def run
+      # Place your data update logic here
+      # Make sure your code is idempotent and can be run safely
+      # multiple times at any time
+    end
+  end
+end
+<% end -%>

lib/tasks/data_updates.rake

@@ -0,0 +1,12 @@
+namespace :data_updates do
+  desc "Enqueue Sidekiq worker to handle data updates"
+  task enqueue_data_update_worker: :environment do
+    # Ensure new code has been deployed before we run our update scripts
+    DataUpdateWorker.perform_in(10.minutes)
+  end
+
+  desc "Run data updates"
+  task run: :environment do
+    DataUpdateWorker.new.perform
+  end
+end

spec/factories/data_update_script.rb

@@ -0,0 +1,5 @@
+FactoryBot.define do
+  factory :data_update_script do
+    file_name { "data_update_test_script" }
+  end
+end

spec/models/data_update_script_spec.rb

@@ -0,0 +1,68 @@
+require "rails_helper"
+
+RSpec.describe DataUpdateScript do
+  it { is_expected.to validate_uniqueness_of(:file_name) }
+
+  it "can constantize all script names" do
+    described_class.filenames.each do |filename|
+      expect { "#{described_class::NAMESPACE}::#{filename.camelcase}".constantize }.not_to raise_error
+    end
+  end
+
+  describe "::load_script_ids" do
+    let(:test_directory) { Rails.root.join("spec/support/fixtures/data_update_scripts") }
+
+    before { stub_const "#{described_class}::DIRECTORY", test_directory }
+
+    it "creates new DataUpdateScripts from files" do
+      expect do
+        described_class.load_script_ids
+      end.to change(described_class, :count).by(1)
+    end
+
+    it "returns script ids that need to be run" do
+      script = FactoryBot.create(:data_update_script)
+      need_running_ids = described_class.load_script_ids
+      expect(need_running_ids).to include(script.id)
+    end
+
+    it "does not return script ids that are running" do
+      script = FactoryBot.create(:data_update_script, run_at: Time.now.utc, status: :working)
+      need_running_ids = described_class.load_script_ids
+      expect(need_running_ids).not_to include(script.id)
+    end
+  end
+
+  describe "#mark_as_finished!" do
+    it "marks data update script as finished" do
+      test_script = FactoryBot.create(:data_update_script)
+      expect(test_script.finished_at).to be_nil
+      expect(test_script).to be_enqueued
+      test_script.mark_as_finished!
+      expect(test_script).to be_succeeded
+      expect(test_script.finished_at).not_to be_nil
+    end
+  end
+
+  describe "#mark_as_run!" do
+    it "marks data update script as working" do
+      test_script = FactoryBot.create(:data_update_script)
+      expect(test_script.run_at).to be_nil
+      expect(test_script).to be_enqueued
+      test_script.mark_as_run!
+      expect(test_script).to be_working
+      expect(test_script.run_at).not_to be_nil
+    end
+  end
+
+  describe "#mark_as_failed!" do
+    it "marks data update script as failed" do
+      test_script = FactoryBot.create(:data_update_script)
+      expect(test_script.finished_at).to be_nil
+      expect(test_script).to be_enqueued
+      test_script.mark_as_failed!
+      expect(test_script).to be_failed
+      expect(test_script.finished_at).not_to be_nil
+    end
+  end
+end

spec/support/fixtures/data_update_scripts/data_update_test_script.rb

@@ -0,0 +1,5 @@
+module DataUpdateScripts
+  class DataUpdateTestScript
+    def run; end
+  end
+end

spec/workers/data_update_worker_spec.rb

@@ -0,0 +1,34 @@
+require "rails_helper"
+require Rails.root.join("app/models/data_update_script.rb")
+
+RSpec.describe DataUpdateWorker, type: :worker do
+  let(:test_directory) { Rails.root.join("spec/support/fixtures/data_update_scripts") }
+  let(:worker) { described_class.new }
+
+  before do
+    stub_const "DataUpdateScript::DIRECTORY", test_directory
+  end
+
+  it "runs scripts that need running" do
+    expect do
+      worker.perform
+    end.to change(DataUpdateScript, :count).by(1)
+  end
+
+  it "will not run a script that has already been run" do
+    worker.perform
+    expect do
+      worker.perform
+    end.to change(DataUpdateScript, :count).by(0)
+  end
+
+  it "updates DataUpdateScript model" do
+    expect do
+      worker.perform
+    end.to change(DataUpdateScript, :count).by(1)
+    dus = DataUpdateScript.find_by(file_name: "data_update_test_script")
+    expect(dus.finished_at).not_to be_nil
+    expect(dus).to be_succeeded
+    expect(dus.run_at).not_to be_nil
+  end
+end