[deploy] Update whitelisted params on Fastly on deployments (#7279)

* Add Fastly whitelisted params

* Create FastlyVCL::WhitelistedParams.update service

* Add specs

* Create rake task to call new service

* Add rake task to setup script

* Add documentatoin

* Move sort inside params_to_array method

* Rename VCL_REGEX to VCL_DELIMITER

* Add PR URL to docs

* Move snippet name to SNIPPET_NAME constant

* Refactor setting content

* Rename params_to_array to params_to_sorted_array

* Change guard and log success on update

- Return early if params are equal
- Log success message to Rails.logger on update
- Log params diff to Datadog

* Return true explicitly

* Move rake task execution from to release-tasks

* Remove unnecessary environment guard in task

* Remove duplicate code param

* Refactor string from build_content

* Remove reliance on sort

* Reorder logging to implicitly return true

* Update docs to reference release-script.sh

* Update docs

* Update whitelisted params

* Fix params_updated? bug and add more specs

* Remove duplicate param...oopsie!

app/services/fastly_vcl/whitelisted_params.rb

@@ -0,0 +1,67 @@
+module FastlyVCL
+  # Handles updates to our VCL snippet on Fastly that whitelists params
+  class WhitelistedParams
+    VCL_DELIMITER_START = "^(".freeze
+    VCL_DELIMITER_END = ")$".freeze
+    SNIPPET_NAME = "Whitelist certain querystring parameters".freeze
+    FILE_PARAMS = YAML.load_file("config/fastly/whitelisted_params.yml").freeze
+
+    class << self
+      def update
+        fastly = Fastly.new(api_key: ApplicationConfig["FASTLY_API_KEY"])
+        service = fastly.get_service(ApplicationConfig["FASTLY_SERVICE_ID"])
+        latest_version = service.version
+
+        snippet = fastly.get_snippet(ApplicationConfig["FASTLY_SERVICE_ID"],
+                                     latest_version.number,
+                                     SNIPPET_NAME)
+
+        current_params = params_to_array(snippet.content)
+
+        return unless params_updated?(current_params)
+
+        new_version = latest_version.clone
+        new_snippet = fastly.get_snippet(ApplicationConfig["FASTLY_SERVICE_ID"],
+                                         new_version.number,
+                                         SNIPPET_NAME)
+        new_snippet.content = build_content(FILE_PARAMS, new_snippet.content)
+        new_snippet.save!
+
+        new_version.activate!
+        log_params_diff_to_datadaog(current_params, new_version)
+        Rails.logger.info("Fastly updated to version #{new_version.number}.")
+      end
+
+      private
+
+      def params_updated?(current_params)
+        (current_params - FILE_PARAMS).any? || (FILE_PARAMS - current_params).any?
+      end
+
+      def params_to_array(snippet_content)
+        snippet_suffix = snippet_content.split(VCL_DELIMITER_START).last
+        fastly_params = snippet_suffix.split(VCL_DELIMITER_END).first
+
+        fastly_params.split("|")
+      end
+
+      def build_content(new_params, snippet_content)
+        new_params = new_params.join("|")
+        snippet_prefix = snippet_content.split(VCL_DELIMITER_START).first
+        snippet_suffix = snippet_content.split(VCL_DELIMITER_END).last
+
+        "#{snippet_prefix}#{VCL_DELIMITER_START}#{new_params}#{VCL_DELIMITER_END}#{snippet_suffix}"
+      end
+
+      def log_params_diff_to_datadaog(current_params, new_version)
+        tags = [
+          "added_params:#{(FILE_PARAMS - current_params).join(', ')}",
+          "removed_params:#{(current_params - FILE_PARAMS).join(', ')}",
+          "new_version:#{new_version.number}",
+        ]
+
+        DatadogStatsClient.increment("fastly.whitelist", tags: tags)
+      end
+    end
+  end
+end

config/fastly/whitelisted_params.yml

@@ -0,0 +1,56 @@
+---
+- a_id
+- args
+- article_id
+- article_ids
+- articles
+- asc
+- callback_url
+- category
+- chat_channel_id
+- client_id
+- code
+- collection_id
+- commentable_id
+- commentable_type
+- confirmation_token
+- created_at
+- end
+- filter
+- followable_id
+- followable_type
+- fork_id
+- i
+- key
+- message_offset
+- name
+- oauth_token
+- oauth_verifier
+- offset
+- org_id
+- organization_id
+- p
+- page
+- per_page
+- prefill
+- preview
+- purchaser
+- reactable_ids
+- redirect_uri
+- reported_url
+- reporter_username
+- response_type
+- scope
+- search
+- signature
+- sort
+- start
+- state
+- status
+- tag
+- tag_list
+- top
+- url
+- username
+- ut
+- verb

docs/backend/fastly.md

@@ -0,0 +1,49 @@
+---
+title: Fastly
+---
+
+## What is Fastly?
+
+[Fastly](https://www.fastly.com/) is a third party service we use for caching on
+the edge. It allows us to scale up and serve our millions of visitors quickly
+and efficiently.
+
+If you want to learn more about we use Fastly, check out this
+[talk](https://www.youtube.com/watch?v=Afy7H04X9Us) that one of our founders,
+[@benhalpern](https://dev.to/ben), gave at RailsConf 2018 talking about how we
+made our app so fast it went viral.
+
+## Whitelisting query string parameters
+
+In the context of contributing, here's what you need to know about Fastly. In
+order for our servers to receive any sort of query string parameters in a
+request, they must first be whitelisted in Fastly. For example, if you're
+creating a new API endpoint or updating an existing one to accept new
+parameters, you'll need to update Fastly.
+
+The reason we whitelist parameters in Fastly this way is so we don't have to
+consider junk parameters when busting the caches. Check out our
+[`CacheBuster`](https://github.com/thepracticaldev/dev.to/blob/master/app/labor/cache_buster.rb)
+to see examples of this.
+
+Previously this was a manual process done by an internal team member. Now we do
+it programmatically using the Fastly
+[gem](https://github.com/fastly/fastly-ruby) as of
+[this pr](https://github.com/thepracticaldev/dev.to/pull/7279).
+
+## How it works
+
+We created a new file, `config/fastly/whitelisted_params.yml`, to house all of
+the whitelisted params in Fastly.
+
+If you need to update this list, simply update this file. It's as easy as that!
+
+_Fastly is not setup for development._
+
+## In production
+
+Whitelisted params on Fastly are updated automatically when a production deploy
+goes out.
+
+We do this by executing `bin/rails fastly:update_whitelisted_params` in our
+`release-tasks.sh` script.

docs/backend/readme.md

@@ -7,6 +7,7 @@ items:
   - authorization.md
   - configuration.md
   - data-update-scripts.md
+  - fastly.md
   - roles.md
   - algolia.md
   - pusher.md

lib/tasks/fastly.rake

@@ -0,0 +1,6 @@
+namespace :fastly do
+  desc "Update VCL for whitelisted params on Fastly"
+  task update_whitelisted_params: :environment do
+    FastlyVCL::WhitelistedParams.update
+  end
+end

release-tasks.sh

@@ -17,6 +17,7 @@ set -Eex
 # runs migration for Postgres, setups/updates Elasticsearch
 # and boots the app to check there are no errors
 STATEMENT_TIMEOUT=180000 bundle exec rails db:migrate
+bundle exec rake fastly:update_whitelisted_params
 bundle exec rake search:setup
 bundle exec rake data_updates:enqueue_data_update_worker
 bundle exec rails runner "puts 'app load success'"

spec/services/fastly_vcl/whitelisted_params_spec.rb

@@ -0,0 +1,73 @@
+require "rails_helper"
+
+RSpec.describe FastlyVCL::WhitelistedParams, type: :service do
+  let(:fastly) { instance_double(Fastly) }
+  let(:fastly_service) { instance_double(Fastly::Service) }
+  let(:fastly_version) { instance_double(Fastly::Version) }
+  let(:fastly_snippet) { instance_double(Fastly::Snippet) }
+  let(:file_params) { YAML.load_file("config/fastly/whitelisted_params.yml") }
+  let(:snippet_content) { "#{described_class::VCL_DELIMITER_START}#{file_params.join('|')}#{described_class::VCL_DELIMITER_END}" }
+
+  # Fastly isn't setup for test or development environments so we have to stub
+  # quite a bit here to simulate Fastly working ¯\_(ツ)_/¯
+  before do
+    allow(Fastly).to receive(:new).and_return(fastly)
+    allow(fastly).to receive(:get_service).and_return(fastly_service)
+    allow(fastly_service).to receive(:version).and_return(fastly_version)
+    allow(fastly_version).to receive(:number).and_return(1)
+    allow(fastly).to receive(:get_snippet).and_return(fastly_snippet)
+    allow(fastly_snippet).to receive(:content).and_return(snippet_content)
+    allow(fastly_version).to receive(:clone).and_return(fastly_version)
+    allow(fastly_version).to receive(:number).and_return(99)
+    allow(described_class).to receive(:build_content).and_return(snippet_content)
+    allow(fastly_snippet).to receive(:content).and_return(fastly_snippet)
+    allow(described_class).to receive(:params_to_array).and_return(file_params)
+    allow(fastly_snippet).to receive(:content=).and_return(snippet_content)
+    allow(fastly_snippet).to receive(:save!).and_return(true)
+    allow(fastly_version).to receive(:activate!).and_return(true)
+  end
+
+  describe "::update" do
+    it "doesn't update if the params haven't changed" do
+      described_class.update
+      expect(fastly_version).not_to have_received(:clone)
+    end
+
+    it "updates Fastly if new params are added" do
+      new_params = file_params + ["new_param"]
+      stub_const("#{described_class}::FILE_PARAMS", new_params)
+      described_class.update
+      expect(fastly_version).to have_received(:activate!)
+    end
+
+    it "updates Fastly if params are removed" do
+      new_params = file_params - [file_params.last]
+      stub_const("#{described_class}::FILE_PARAMS", new_params)
+      described_class.update
+      expect(fastly_version).to have_received(:activate!)
+    end
+
+    it "overwrites updates made directly in Fastly" do
+      new_fastly_params = file_params + ["new_fastly_param"]
+      allow(described_class).to receive(:params_to_array).and_return(new_fastly_params)
+      described_class.update
+      expect(fastly_version).to have_received(:activate!)
+    end
+
+    it "logs success messages" do
+      allow(Rails.logger).to receive(:info)
+      allow(DatadogStatsClient).to receive(:increment)
+
+      old_param = file_params.last
+      new_params = file_params + ["new_param"] - [old_param]
+      stub_const("#{described_class}::FILE_PARAMS", new_params)
+      described_class.update
+
+      expect(Rails.logger).to have_received(:info)
+
+      tags = hash_including(tags: array_including("added_params:new_param", "removed_params:#{old_param}", "new_version:#{fastly_version.number}"))
+
+      expect(DatadogStatsClient).to have_received(:increment).with("fastly.whitelist", tags)
+    end
+  end
+end