diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7e3e30687..4285195b6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -25,6 +25,8 @@ /config/ @thepracticaldev/sre /db/ @thepracticaldev/sre /docs/ @thepracticaldev/oss @jacobherrington +/lib/data_update_scripts/ @thepracticaldev/sre +/lib/sidekiq/ @thepracticaldev/sre /spec/requests/internal/ @thepracticaldev/snackcase /spec/system/internal/ @thepracticaldev/snackcase Gemfile @thepracticaldev/sre @thepracticaldev/oss diff --git a/app/models/data_update_script.rb b/app/models/data_update_script.rb new file mode 100644 index 000000000..96759d575 --- /dev/null +++ b/app/models/data_update_script.rb @@ -0,0 +1,35 @@ +class DataUpdateScript < ApplicationRecord + DIRECTORY = Rails.root.join("lib/data_update_scripts").freeze + NAMESPACE = "DataUpdateScripts".freeze + STATUSES = { enqueued: 0, working: 1, succeeded: 2, failed: 3 }.freeze + + enum status: STATUSES + validates :file_name, uniqueness: true + + def self.filenames + Dir.glob("*.rb", base: DIRECTORY).map { |f| Pathname.new(f).basename(".rb").to_s } + end + + def self.load_script_ids + filenames. + map { |file_name| find_or_create_by(file_name: file_name) }. + select(&:enqueued?). + map(&:id) + end + + def mark_as_run! + update!(run_at: Time.now.utc, status: :working) + end + + def mark_as_finished! + update!(finished_at: Time.now.utc, status: :succeeded) + end + + def mark_as_failed! + update!(finished_at: Time.now.utc, status: :failed) + end + + def file_class + "#{self.class::NAMESPACE}::#{file_name.camelcase}".constantize + end +end diff --git a/app/workers/data_update_worker.rb b/app/workers/data_update_worker.rb new file mode 100644 index 000000000..f252730e4 --- /dev/null +++ b/app/workers/data_update_worker.rb @@ -0,0 +1,22 @@ +class DataUpdateWorker + include Sidekiq::Worker + sidekiq_options queue: :high_priority, retry: 5 + + def perform + script_ids = DataUpdateScript.load_script_ids + scripts_to_run = DataUpdateScript.where(id: script_ids).select(&:enqueued?) + + scripts_to_run.each do |script| + script.mark_as_run! + run_script(script) + end + end + + def run_script(script) + script.file_class.new.run + script.mark_as_finished! + rescue StandardError => e + script.mark_as_failed! + Honeybadger.notify(e, context: { script_id: script.id }) + end +end diff --git a/bin/setup b/bin/setup index f43feea71..b3e34c143 100755 --- a/bin/setup +++ b/bin/setup @@ -33,6 +33,9 @@ chdir APP_ROOT do puts "\n== Preparing Elasticsearch ==" system! 'bin/rails search:setup' + puts "\n== Update Data ==" + system! 'bin/rails data_updates:run' + puts "\n== Removing old logs and tempfiles ==" system! 'bin/rails log:clear tmp:clear' diff --git a/bin/update b/bin/update index b5dc3d52d..92008104e 100755 --- a/bin/update +++ b/bin/update @@ -23,6 +23,9 @@ chdir APP_ROOT do puts "\n== Updating database ==" system! 'bin/rails db:migrate' + puts "\n== Update Data ==" + system! 'bin/rails data_updates:run' + puts "\n== Update Elasticsearch ==" system! 'bin/rails search:setup' diff --git a/db/migrate/20200211192415_create_data_update_scripts.rb b/db/migrate/20200211192415_create_data_update_scripts.rb new file mode 100644 index 000000000..709425ab0 --- /dev/null +++ b/db/migrate/20200211192415_create_data_update_scripts.rb @@ -0,0 +1,11 @@ +class CreateDataUpdateScripts < ActiveRecord::Migration[5.2] + def change + create_table :data_update_scripts do |t| + t.string :file_name, unique: true + t.integer :status, default: 0, null: false + t.timestamp :run_at + t.timestamp :finished_at + t.timestamps null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 828b4f6ca..9996819be 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_02_05_225813) do +ActiveRecord::Schema.define(version: 2020_02_11_192415) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -350,6 +350,15 @@ ActiveRecord::Schema.define(version: 2020_02_05_225813) do t.index ["spent"], name: "index_credits_on_spent" end + create_table "data_update_scripts", force: :cascade do |t| + t.datetime "created_at", null: false + t.string "file_name" + t.datetime "finished_at" + t.datetime "run_at" + t.integer "status", default: 0, null: false + t.datetime "updated_at", null: false + end + create_table "delayed_jobs", id: :serial, force: :cascade do |t| t.integer "attempts", default: 0, null: false t.datetime "created_at" diff --git a/lib/generators/data_update/USAGE b/lib/generators/data_update/USAGE new file mode 100644 index 000000000..972f3ea04 --- /dev/null +++ b/lib/generators/data_update/USAGE @@ -0,0 +1,17 @@ +Description: + Generates a data update script. + +Examples: + `rails generate data_update ExampleScript` + + # lib/data_update_scripts/example_script.rb + module DataUpdateScripts + class ExampleScript + def run + # Place your data update logic here + # Make sure your code is idempotent and can be run safely + # multiple times at any time + end + end + end + diff --git a/lib/generators/data_update/data_update_generator.rb b/lib/generators/data_update/data_update_generator.rb new file mode 100644 index 000000000..88a5fa7dc --- /dev/null +++ b/lib/generators/data_update/data_update_generator.rb @@ -0,0 +1,10 @@ +class DataUpdateGenerator < Rails::Generators::NamedBase + source_root File.expand_path("templates", __dir__) + + def create_data_update_file + template( + "data_update.rb.tt", + File.join("lib", "data_update_scripts", class_path, "#{file_name}.rb"), + ) + end +end diff --git a/lib/generators/data_update/templates/data_update.rb.tt b/lib/generators/data_update/templates/data_update.rb.tt new file mode 100644 index 000000000..8cf4f651f --- /dev/null +++ b/lib/generators/data_update/templates/data_update.rb.tt @@ -0,0 +1,11 @@ +<% module_namespacing do -%> +module DataUpdateScripts + class <%= class_name %> + def run + # Place your data update logic here + # Make sure your code is idempotent and can be run safely + # multiple times at any time + end + end +end +<% end -%> diff --git a/lib/tasks/data_updates.rake b/lib/tasks/data_updates.rake new file mode 100644 index 000000000..3f0ff2b7e --- /dev/null +++ b/lib/tasks/data_updates.rake @@ -0,0 +1,12 @@ +namespace :data_updates do + desc "Enqueue Sidekiq worker to handle data updates" + task enqueue_data_update_worker: :environment do + # Ensure new code has been deployed before we run our update scripts + DataUpdateWorker.perform_in(10.minutes) + end + + desc "Run data updates" + task run: :environment do + DataUpdateWorker.new.perform + end +end diff --git a/spec/factories/data_update_script.rb b/spec/factories/data_update_script.rb new file mode 100644 index 000000000..4652ec681 --- /dev/null +++ b/spec/factories/data_update_script.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :data_update_script do + file_name { "data_update_test_script" } + end +end diff --git a/spec/models/data_update_script_spec.rb b/spec/models/data_update_script_spec.rb new file mode 100644 index 000000000..f4da27062 --- /dev/null +++ b/spec/models/data_update_script_spec.rb @@ -0,0 +1,68 @@ +require "rails_helper" + +RSpec.describe DataUpdateScript do + it { is_expected.to validate_uniqueness_of(:file_name) } + + it "can constantize all script names" do + described_class.filenames.each do |filename| + expect { "#{described_class::NAMESPACE}::#{filename.camelcase}".constantize }.not_to raise_error + end + end + + describe "::load_script_ids" do + let(:test_directory) { Rails.root.join("spec/support/fixtures/data_update_scripts") } + + before { stub_const "#{described_class}::DIRECTORY", test_directory } + + it "creates new DataUpdateScripts from files" do + expect do + described_class.load_script_ids + end.to change(described_class, :count).by(1) + end + + it "returns script ids that need to be run" do + script = FactoryBot.create(:data_update_script) + need_running_ids = described_class.load_script_ids + expect(need_running_ids).to include(script.id) + end + + it "does not return script ids that are running" do + script = FactoryBot.create(:data_update_script, run_at: Time.now.utc, status: :working) + need_running_ids = described_class.load_script_ids + expect(need_running_ids).not_to include(script.id) + end + end + + describe "#mark_as_finished!" do + it "marks data update script as finished" do + test_script = FactoryBot.create(:data_update_script) + expect(test_script.finished_at).to be_nil + expect(test_script).to be_enqueued + test_script.mark_as_finished! + expect(test_script).to be_succeeded + expect(test_script.finished_at).not_to be_nil + end + end + + describe "#mark_as_run!" do + it "marks data update script as working" do + test_script = FactoryBot.create(:data_update_script) + expect(test_script.run_at).to be_nil + expect(test_script).to be_enqueued + test_script.mark_as_run! + expect(test_script).to be_working + expect(test_script.run_at).not_to be_nil + end + end + + describe "#mark_as_failed!" do + it "marks data update script as failed" do + test_script = FactoryBot.create(:data_update_script) + expect(test_script.finished_at).to be_nil + expect(test_script).to be_enqueued + test_script.mark_as_failed! + expect(test_script).to be_failed + expect(test_script.finished_at).not_to be_nil + end + end +end diff --git a/spec/support/fixtures/data_update_scripts/data_update_test_script.rb b/spec/support/fixtures/data_update_scripts/data_update_test_script.rb new file mode 100644 index 000000000..c748cd140 --- /dev/null +++ b/spec/support/fixtures/data_update_scripts/data_update_test_script.rb @@ -0,0 +1,5 @@ +module DataUpdateScripts + class DataUpdateTestScript + def run; end + end +end diff --git a/spec/workers/data_update_worker_spec.rb b/spec/workers/data_update_worker_spec.rb new file mode 100644 index 000000000..0013ffe1a --- /dev/null +++ b/spec/workers/data_update_worker_spec.rb @@ -0,0 +1,34 @@ +require "rails_helper" +require Rails.root.join("app/models/data_update_script.rb") + +RSpec.describe DataUpdateWorker, type: :worker do + let(:test_directory) { Rails.root.join("spec/support/fixtures/data_update_scripts") } + let(:worker) { described_class.new } + + before do + stub_const "DataUpdateScript::DIRECTORY", test_directory + end + + it "runs scripts that need running" do + expect do + worker.perform + end.to change(DataUpdateScript, :count).by(1) + end + + it "will not run a script that has already been run" do + worker.perform + expect do + worker.perform + end.to change(DataUpdateScript, :count).by(0) + end + + it "updates DataUpdateScript model" do + expect do + worker.perform + end.to change(DataUpdateScript, :count).by(1) + dus = DataUpdateScript.find_by(file_name: "data_update_test_script") + expect(dus.finished_at).not_to be_nil + expect(dus).to be_succeeded + expect(dus.run_at).not_to be_nil + end +end