diff --git a/app/models/data_update_script.rb b/app/models/data_update_script.rb index 96759d575..05afbc39c 100644 --- a/app/models/data_update_script.rb +++ b/app/models/data_update_script.rb @@ -3,11 +3,15 @@ class DataUpdateScript < ApplicationRecord NAMESPACE = "DataUpdateScripts".freeze STATUSES = { enqueued: 0, working: 1, succeeded: 2, failed: 3 }.freeze + default_scope { order(file_name: :asc) } + enum status: STATUSES validates :file_name, uniqueness: true def self.filenames - Dir.glob("*.rb", base: DIRECTORY).map { |f| Pathname.new(f).basename(".rb").to_s } + Dir.glob("*.rb", base: DIRECTORY).map do |f| + Pathname.new(f).basename(".rb").to_s + end end def self.load_script_ids @@ -18,18 +22,28 @@ class DataUpdateScript < ApplicationRecord end def mark_as_run! - update!(run_at: Time.now.utc, status: :working) + update!(run_at: Time.current, status: :working) end def mark_as_finished! - update!(finished_at: Time.now.utc, status: :succeeded) + update!(finished_at: Time.current, status: :succeeded) end def mark_as_failed! - update!(finished_at: Time.now.utc, status: :failed) + update!(finished_at: Time.current, status: :failed) + end + + def file_path + "#{self.class::DIRECTORY}/#{file_name}.rb" end def file_class - "#{self.class::NAMESPACE}::#{file_name.camelcase}".constantize + "#{self.class::NAMESPACE}::#{parsed_file_name.camelcase}".safe_constantize + end + + private + + def parsed_file_name + file_name.match(/\d{14}_(.*)/)[1] end end diff --git a/app/workers/data_update_worker.rb b/app/workers/data_update_worker.rb index f252730e4..290ce1d49 100644 --- a/app/workers/data_update_worker.rb +++ b/app/workers/data_update_worker.rb @@ -12,7 +12,10 @@ class DataUpdateWorker end end + private + def run_script(script) + require script.file_path script.file_class.new.run script.mark_as_finished! rescue StandardError => e diff --git a/config/boot.rb b/config/boot.rb index 988a5ddc4..36d6361a0 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -1,4 +1,5 @@ ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) +ENV["COMMAND"] = ARGV[0] require "bundler/setup" # Set up gems listed in the Gemfile. require "bootsnap/setup" # Speed up boot time by caching expensive operations. diff --git a/config/environments/development.rb b/config/environments/development.rb index fb0e600d1..a75706d3c 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -109,6 +109,15 @@ Rails.application.configure do Bullet.add_whitelist(type: :unused_eager_loading, class_name: "ApiSecret", association: :user) # acts-as-taggable-on has super weird eager loading problems: Bullet.add_whitelist(type: :n_plus_one_query, class_name: "ActsAsTaggableOn::Tagging", association: :tag) + + DATA_UPDATE_CHECK_COMMANDS = %w[c console s server].freeze + if DATA_UPDATE_CHECK_COMMANDS.include?(ENV["COMMAND"]) + script_ids = DataUpdateScript.load_script_ids + scripts_to_run = DataUpdateScript.where(id: script_ids).select(&:enqueued?) + if scripts_to_run.any? + raise "Data update scripts need to be run before you can start the application. Please run rake data_updates:run" + end + end end end diff --git a/docs/backend/data-update-scripts.md b/docs/backend/data-update-scripts.md new file mode 100644 index 000000000..b9d54ca23 --- /dev/null +++ b/docs/backend/data-update-scripts.md @@ -0,0 +1,63 @@ +--- +title: Data Update Scripts +--- + +## What are Data Update Scripts? + +Data Update Scripts were introduced in +[this PR](https://github.com/thepracticaldev/dev.to/pull/6025) and allow us to +run any data updates we might need. For example, if we added a column to the +database and then wanted to backfill that column with data, rather than going +and manually doing it in a console, we would use a DataUpdateScript. Another +example might be adding a new attribute to Elasticsearch. We could then use a +DataUpdateScript to reindex all of our models. + +## How it works + +First off, we added a +[DataUpdateScript model](https://github.com/thepracticaldev/dev.to/blob/master/app/models/data_update_script.rb) +to Rails and a corresponding database table. This table is what we use to keep +track of what scripts have been run and which ones have not/still need to be. + +To create a script you can use our custom Rails generator: + +``` +rails generate data_update BackfillColumnForArticles +``` + +This will create a simple Ruby class like below and all you have to do is fill +in the code it will run. + +```ruby +module DataUpdateScripts + class BackfillColumnForArticles + def run + # Place your data update logic here + # Make sure your code is idempotent and can be run safely + # multiple times at any time + end + end +end +``` + +Once your script is in place then you can either run `rails data_updates:run` +manually or you can let our setup and update scripts handle it. In our local +[bin/setup](https://github.com/thepracticaldev/dev.to/blob/master/bin/setup#L36) +and +[bin/update](https://github.com/thepracticaldev/dev.to/blob/master/bin/update#L26) +scripts you will see we have added an additional task to "Update Data". This +kicks off the rake task `data_updates:run` for you. + +The rake task itself will check the `lib/data_update_scripts` folder to see if +there are any new scripts that need to be run. It does this by reading all of +the files and then checking to see if they have a corresponding database entry. +If they do not, then we create a new one and run the script. If a database entry +already exists and it indicates the script has been run, then we skip that +script. + +## In production + +DataUpdateScripts are also run automatically when a production deploy goes out. +However, to ensure the new code they need to use has been deployed we use a +[`DataUpdateWorker`](https://github.com/thepracticaldev/dev.to/blob/master/app/workers/data_update_worker.rb) +via Sidekiq and set it to run 10 minutes after the deploy script has completed. diff --git a/docs/backend/readme.md b/docs/backend/readme.md index 884bdc7bc..671e3066c 100644 --- a/docs/backend/readme.md +++ b/docs/backend/readme.md @@ -6,6 +6,7 @@ items: - auth-github.md - authorization.md - configuration.md + - data-update-scripts.md - roles.md - algolia.md - pusher.md diff --git a/docs/deployment.md b/docs/deployment.md index 1a1b7273b..0183c1c06 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -38,21 +38,30 @@ Upon deploy, the app installs dependencies, bundles assets, and gets the app ready for launch. However, before it launches and releases the app Heroku runs a release script on a one-off dyno. If that release script/step succeeds the new app is released on all of the dynos. If that release script/step fails then the -deploy is halted and we are notified. During this release step, we first check -the DEPLOY_STATUS environment variable. In the event that we want to prevent -deploys, for example after a rollback, we will set DEPLOY_STATUS to "blocked". -This will cause the release script to exit with a code of 1 which will halt the -deploy. This ensures that we don't accidentally push out code while we are -waiting for a fix or running other tasks. Next, we run any outstanding -migrations. This ensures that a migration finishes successfully before the code -that uses it goes live. After running migrations, we update Elasticsearch. -Elasticsearch contains indexes which have mappings. Mappings are similar to -database schema. The same way we run a migration to update our database we have -to run a setup task to update any Elasticsearch mappings. Following updating all -of our datastores we use the Rails runner to output a simple string. Executing a -Rails runner command ensures that we can boot up the entire app successfully -before it is deployed. We deploy asynchronously, so the website is running the -new code a few minutes after deploy. A new instance of Heroku Rails console will -immediately run a new code. +deploy is halted and we are notified. + +The name of the script we use is `release-tasks.sh` and its in our root +directory. During this release step we do a few checks. + +1. We first check the DEPLOY_STATUS environment variable. In the event that we + want to prevent deploys, for example after a rollback, we will set + DEPLOY_STATUS to "blocked". This will cause the release script to exit with a + code of 1 which will halt the deploy. This ensures that we don't accidentally + push out code while we are waiting for a fix or running other tasks. +2. We run any outstanding migrations. This ensures that a migration finishes + successfully before the code that uses it goes live. +3. We run any data update scripts that need to be run. A data update script is + one that allows us to update data in the background separate from a + migration. For example, if we add a new field to Elasticsearch and need to + reindex all of our documents we would use a data update script. +4. We update Elasticsearch. Elasticsearch contains indexes which have mappings. + Mappings are similar to database schema. The same way we run a migration to + update our database we have to run a setup task to update any Elasticsearch + mappings. +5. Following updating all of our datastores we use the Rails runner to output a + simple string. Executing a Rails runner command ensures that we can boot up + the entire app successfully before it is deployed. We deploy asynchronously, + so the website is running the new code a few minutes after deploy. A new + instance of Heroku Rails console will immediately run a new code. ![](https://devcenter0.assets.heroku.com/article-images/1494371187-release-phase-diagram-3.png) diff --git a/lib/generators/data_update/data_update_generator.rb b/lib/generators/data_update/data_update_generator.rb index 88a5fa7dc..c259b2ee4 100644 --- a/lib/generators/data_update/data_update_generator.rb +++ b/lib/generators/data_update/data_update_generator.rb @@ -4,7 +4,7 @@ class DataUpdateGenerator < Rails::Generators::NamedBase def create_data_update_file template( "data_update.rb.tt", - File.join("lib", "data_update_scripts", class_path, "#{file_name}.rb"), + File.join("lib", "data_update_scripts", class_path, "#{Time.current.utc.strftime('%Y%m%d%H%M%S')}_#{file_name}.rb"), ) end end diff --git a/release-tasks.sh b/release-tasks.sh index d18ad7256..948c35203 100755 --- a/release-tasks.sh +++ b/release-tasks.sh @@ -10,4 +10,5 @@ set -x # and boots the app to check there are no errors STATEMENT_TIMEOUT=180000 bundle exec rails db:migrate && \ bundle exec rake search:setup && \ + bundle exec rake data_updates:enqueue_data_update_worker && \ bundle exec rails runner "puts 'app load success'" diff --git a/spec/factories/data_update_script.rb b/spec/factories/data_update_script.rb index 4652ec681..3ea50cb3b 100644 --- a/spec/factories/data_update_script.rb +++ b/spec/factories/data_update_script.rb @@ -1,5 +1,5 @@ FactoryBot.define do factory :data_update_script do - file_name { "data_update_test_script" } + file_name { "20200214151804_data_update_test_script" } end end diff --git a/spec/models/data_update_script_spec.rb b/spec/models/data_update_script_spec.rb index f4da27062..7be44289e 100644 --- a/spec/models/data_update_script_spec.rb +++ b/spec/models/data_update_script_spec.rb @@ -4,11 +4,19 @@ RSpec.describe DataUpdateScript do it { is_expected.to validate_uniqueness_of(:file_name) } it "can constantize all script names" do + described_class.load_script_ids # Create DataUpdateScripts in db described_class.filenames.each do |filename| - expect { "#{described_class::NAMESPACE}::#{filename.camelcase}".constantize }.not_to raise_error + script = described_class.find_by(file_name: filename) + expect { script.file_class }.not_to raise_error end end + it "default orders scripts by name" do + script1 = FactoryBot.create(:data_update_script, file_name: "456_test_script") + script2 = FactoryBot.create(:data_update_script, file_name: "123_test_script") + expect(described_class.pluck(:id)).to eq([script2.id, script1.id]) + end + describe "::load_script_ids" do let(:test_directory) { Rails.root.join("spec/support/fixtures/data_update_scripts") } @@ -65,4 +73,16 @@ RSpec.describe DataUpdateScript do expect(test_script.finished_at).not_to be_nil end end + + describe "#file_path" do + it "returns correct loadable file_path" do + described_class.load_script_ids + described_class.filenames.each do |filename| + expect do + script = described_class.find_by(file_name: filename) + require script.file_path + end.not_to raise_error + end + end + end end diff --git a/spec/support/fixtures/data_update_scripts/data_update_test_script.rb b/spec/support/fixtures/data_update_scripts/20200214151804_data_update_test_script.rb similarity index 100% rename from spec/support/fixtures/data_update_scripts/data_update_test_script.rb rename to spec/support/fixtures/data_update_scripts/20200214151804_data_update_test_script.rb diff --git a/spec/workers/data_update_worker_spec.rb b/spec/workers/data_update_worker_spec.rb index 0013ffe1a..e88b54982 100644 --- a/spec/workers/data_update_worker_spec.rb +++ b/spec/workers/data_update_worker_spec.rb @@ -26,7 +26,7 @@ RSpec.describe DataUpdateWorker, type: :worker do expect do worker.perform end.to change(DataUpdateScript, :count).by(1) - dus = DataUpdateScript.find_by(file_name: "data_update_test_script") + dus = DataUpdateScript.find_by(file_name: "20200214151804_data_update_test_script") expect(dus.finished_at).not_to be_nil expect(dus).to be_succeeded expect(dus.run_at).not_to be_nil