Document DataUpdateScripts, Add to Deploy Process, Add Development Check (#6068) [deploy]

app/models/data_update_script.rb

@@ -3,11 +3,15 @@ class DataUpdateScript < ApplicationRecord
   NAMESPACE = "DataUpdateScripts".freeze
   STATUSES = { enqueued: 0, working: 1, succeeded: 2, failed: 3 }.freeze
 
+  default_scope { order(file_name: :asc) }
+
   enum status: STATUSES
   validates :file_name, uniqueness: true
 
   def self.filenames
-    Dir.glob("*.rb", base: DIRECTORY).map { |f| Pathname.new(f).basename(".rb").to_s }
+    Dir.glob("*.rb", base: DIRECTORY).map do |f|
+      Pathname.new(f).basename(".rb").to_s
+    end
   end
 
   def self.load_script_ids
@@ -18,18 +22,28 @@ class DataUpdateScript < ApplicationRecord
   end
 
   def mark_as_run!
-    update!(run_at: Time.now.utc, status: :working)
+    update!(run_at: Time.current, status: :working)
   end
 
   def mark_as_finished!
-    update!(finished_at: Time.now.utc, status: :succeeded)
+    update!(finished_at: Time.current, status: :succeeded)
   end
 
   def mark_as_failed!
-    update!(finished_at: Time.now.utc, status: :failed)
+    update!(finished_at: Time.current, status: :failed)
+  end
+
+  def file_path
+    "#{self.class::DIRECTORY}/#{file_name}.rb"
   end
 
   def file_class
-    "#{self.class::NAMESPACE}::#{file_name.camelcase}".constantize
+    "#{self.class::NAMESPACE}::#{parsed_file_name.camelcase}".safe_constantize
+  end
+
+  private
+
+  def parsed_file_name
+    file_name.match(/\d{14}_(.*)/)[1]
   end
 end

app/workers/data_update_worker.rb

@@ -12,7 +12,10 @@ class DataUpdateWorker
     end
   end
 
+  private
+
   def run_script(script)
+    require script.file_path
     script.file_class.new.run
     script.mark_as_finished!
   rescue StandardError => e

config/boot.rb

@@ -1,4 +1,5 @@
 ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+ENV["COMMAND"] = ARGV[0]
 
 require "bundler/setup" # Set up gems listed in the Gemfile.
 require "bootsnap/setup" # Speed up boot time by caching expensive operations.

config/environments/development.rb

@@ -109,6 +109,15 @@ Rails.application.configure do
     Bullet.add_whitelist(type: :unused_eager_loading, class_name: "ApiSecret", association: :user)
     # acts-as-taggable-on has super weird eager loading problems: <https://github.com/mbleigh/acts-as-taggable-on/issues/91>
     Bullet.add_whitelist(type: :n_plus_one_query, class_name: "ActsAsTaggableOn::Tagging", association: :tag)
+
+    DATA_UPDATE_CHECK_COMMANDS = %w[c console s server].freeze
+    if DATA_UPDATE_CHECK_COMMANDS.include?(ENV["COMMAND"])
+      script_ids = DataUpdateScript.load_script_ids
+      scripts_to_run = DataUpdateScript.where(id: script_ids).select(&:enqueued?)
+      if scripts_to_run.any?
+        raise "Data update scripts need to be run before you can start the application. Please run rake data_updates:run"
+      end
+    end
   end
 end
 

docs/backend/data-update-scripts.md

@@ -0,0 +1,63 @@
+---
+title: Data Update Scripts
+---
+
+## What are Data Update Scripts?
+
+Data Update Scripts were introduced in
+[this PR](https://github.com/thepracticaldev/dev.to/pull/6025) and allow us to
+run any data updates we might need. For example, if we added a column to the
+database and then wanted to backfill that column with data, rather than going
+and manually doing it in a console, we would use a DataUpdateScript. Another
+example might be adding a new attribute to Elasticsearch. We could then use a
+DataUpdateScript to reindex all of our models.
+
+## How it works
+
+First off, we added a
+[DataUpdateScript model](https://github.com/thepracticaldev/dev.to/blob/master/app/models/data_update_script.rb)
+to Rails and a corresponding database table. This table is what we use to keep
+track of what scripts have been run and which ones have not/still need to be.
+
+To create a script you can use our custom Rails generator:
+
+```
+rails generate data_update BackfillColumnForArticles
+```
+
+This will create a simple Ruby class like below and all you have to do is fill
+in the code it will run.
+
+```ruby
+module DataUpdateScripts
+  class BackfillColumnForArticles
+    def run
+      # Place your data update logic here
+      # Make sure your code is idempotent and can be run safely
+      # multiple times at any time
+    end
+  end
+end
+```
+
+Once your script is in place then you can either run `rails data_updates:run`
+manually or you can let our setup and update scripts handle it. In our local
+[bin/setup](https://github.com/thepracticaldev/dev.to/blob/master/bin/setup#L36)
+and
+[bin/update](https://github.com/thepracticaldev/dev.to/blob/master/bin/update#L26)
+scripts you will see we have added an additional task to "Update Data". This
+kicks off the rake task `data_updates:run` for you.
+
+The rake task itself will check the `lib/data_update_scripts` folder to see if
+there are any new scripts that need to be run. It does this by reading all of
+the files and then checking to see if they have a corresponding database entry.
+If they do not, then we create a new one and run the script. If a database entry
+already exists and it indicates the script has been run, then we skip that
+script.
+
+## In production
+
+DataUpdateScripts are also run automatically when a production deploy goes out.
+However, to ensure the new code they need to use has been deployed we use a
+[`DataUpdateWorker`](https://github.com/thepracticaldev/dev.to/blob/master/app/workers/data_update_worker.rb)
+via Sidekiq and set it to run 10 minutes after the deploy script has completed.

docs/backend/readme.md

@@ -6,6 +6,7 @@ items:
   - auth-github.md
   - authorization.md
   - configuration.md
+  - data-update-scripts.md
   - roles.md
   - algolia.md
   - pusher.md

docs/deployment.md

@@ -38,21 +38,30 @@ Upon deploy, the app installs dependencies, bundles assets, and gets the app
 ready for launch. However, before it launches and releases the app Heroku runs a
 release script on a one-off dyno. If that release script/step succeeds the new
 app is released on all of the dynos. If that release script/step fails then the
-deploy is halted and we are notified. During this release step, we first check
-the DEPLOY_STATUS environment variable. In the event that we want to prevent
-deploys, for example after a rollback, we will set DEPLOY_STATUS to "blocked".
-This will cause the release script to exit with a code of 1 which will halt the
-deploy. This ensures that we don't accidentally push out code while we are
-waiting for a fix or running other tasks. Next, we run any outstanding
-migrations. This ensures that a migration finishes successfully before the code
-that uses it goes live. After running migrations, we update Elasticsearch.
-Elasticsearch contains indexes which have mappings. Mappings are similar to
-database schema. The same way we run a migration to update our database we have
-to run a setup task to update any Elasticsearch mappings. Following updating all
-of our datastores we use the Rails runner to output a simple string. Executing a
-Rails runner command ensures that we can boot up the entire app successfully
-before it is deployed. We deploy asynchronously, so the website is running the
-new code a few minutes after deploy. A new instance of Heroku Rails console will
-immediately run a new code.
+deploy is halted and we are notified.
+
+The name of the script we use is `release-tasks.sh` and its in our root
+directory. During this release step we do a few checks.
+
+1. We first check the DEPLOY_STATUS environment variable. In the event that we
+   want to prevent deploys, for example after a rollback, we will set
+   DEPLOY_STATUS to "blocked". This will cause the release script to exit with a
+   code of 1 which will halt the deploy. This ensures that we don't accidentally
+   push out code while we are waiting for a fix or running other tasks.
+2. We run any outstanding migrations. This ensures that a migration finishes
+   successfully before the code that uses it goes live.
+3. We run any data update scripts that need to be run. A data update script is
+   one that allows us to update data in the background separate from a
+   migration. For example, if we add a new field to Elasticsearch and need to
+   reindex all of our documents we would use a data update script.
+4. We update Elasticsearch. Elasticsearch contains indexes which have mappings.
+   Mappings are similar to database schema. The same way we run a migration to
+   update our database we have to run a setup task to update any Elasticsearch
+   mappings.
+5. Following updating all of our datastores we use the Rails runner to output a
+   simple string. Executing a Rails runner command ensures that we can boot up
+   the entire app successfully before it is deployed. We deploy asynchronously,
+   so the website is running the new code a few minutes after deploy. A new
+   instance of Heroku Rails console will immediately run a new code.
 
 ![](https://devcenter0.assets.heroku.com/article-images/1494371187-release-phase-diagram-3.png)

lib/generators/data_update/data_update_generator.rb

@@ -4,7 +4,7 @@ class DataUpdateGenerator < Rails::Generators::NamedBase
   def create_data_update_file
     template(
       "data_update.rb.tt",
-      File.join("lib", "data_update_scripts", class_path, "#{file_name}.rb"),
+      File.join("lib", "data_update_scripts", class_path, "#{Time.current.utc.strftime('%Y%m%d%H%M%S')}_#{file_name}.rb"),
     )
   end
 end

release-tasks.sh

@@ -10,4 +10,5 @@ set -x
 # and boots the app to check there are no errors
 STATEMENT_TIMEOUT=180000 bundle exec rails db:migrate && \
   bundle exec rake search:setup && \
+  bundle exec rake data_updates:enqueue_data_update_worker && \
   bundle exec rails runner "puts 'app load success'"

spec/factories/data_update_script.rb

@@ -1,5 +1,5 @@
 FactoryBot.define do
   factory :data_update_script do
-    file_name { "data_update_test_script" }
+    file_name { "20200214151804_data_update_test_script" }
   end
 end

spec/models/data_update_script_spec.rb

@@ -4,11 +4,19 @@ RSpec.describe DataUpdateScript do
   it { is_expected.to validate_uniqueness_of(:file_name) }
 
   it "can constantize all script names" do
+    described_class.load_script_ids # Create DataUpdateScripts in db
     described_class.filenames.each do |filename|
-      expect { "#{described_class::NAMESPACE}::#{filename.camelcase}".constantize }.not_to raise_error
+      script = described_class.find_by(file_name: filename)
+      expect { script.file_class }.not_to raise_error
     end
   end
 
+  it "default orders scripts by name" do
+    script1 = FactoryBot.create(:data_update_script, file_name: "456_test_script")
+    script2 = FactoryBot.create(:data_update_script, file_name: "123_test_script")
+    expect(described_class.pluck(:id)).to eq([script2.id, script1.id])
+  end
+
   describe "::load_script_ids" do
     let(:test_directory) { Rails.root.join("spec/support/fixtures/data_update_scripts") }
 
@@ -65,4 +73,16 @@ RSpec.describe DataUpdateScript do
       expect(test_script.finished_at).not_to be_nil
     end
   end
+
+  describe "#file_path" do
+    it "returns correct loadable file_path" do
+      described_class.load_script_ids
+      described_class.filenames.each do |filename|
+        expect do
+          script = described_class.find_by(file_name: filename)
+          require script.file_path
+        end.not_to raise_error
+      end
+    end
+  end
 end

spec/workers/data_update_worker_spec.rb

@@ -26,7 +26,7 @@ RSpec.describe DataUpdateWorker, type: :worker do
     expect do
       worker.perform
     end.to change(DataUpdateScript, :count).by(1)
-    dus = DataUpdateScript.find_by(file_name: "data_update_test_script")
+    dus = DataUpdateScript.find_by(file_name: "20200214151804_data_update_test_script")
     expect(dus.finished_at).not_to be_nil
     expect(dus).to be_succeeded
     expect(dus.run_at).not_to be_nil