From cc6249beff9b4c8f567979f72dd964063ea9ef00 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 16 Apr 2020 15:11:57 -0400 Subject: [PATCH] [deploy] Update whitelisted params on Fastly on deployments (#7279) * Add Fastly whitelisted params * Create FastlyVCL::WhitelistedParams.update service * Add specs * Create rake task to call new service * Add rake task to setup script * Add documentatoin * Move sort inside params_to_array method * Rename VCL_REGEX to VCL_DELIMITER * Add PR URL to docs * Move snippet name to SNIPPET_NAME constant * Refactor setting content * Rename params_to_array to params_to_sorted_array * Change guard and log success on update - Return early if params are equal - Log success message to Rails.logger on update - Log params diff to Datadog * Return true explicitly * Move rake task execution from to release-tasks * Remove unnecessary environment guard in task * Remove duplicate code param * Refactor string from build_content * Remove reliance on sort * Reorder logging to implicitly return true * Update docs to reference release-script.sh * Update docs * Update whitelisted params * Fix params_updated? bug and add more specs * Remove duplicate param...oopsie! --- app/services/fastly_vcl/whitelisted_params.rb | 67 +++++++++++++++++ config/fastly/whitelisted_params.yml | 56 ++++++++++++++ docs/backend/fastly.md | 49 +++++++++++++ docs/backend/readme.md | 1 + lib/tasks/fastly.rake | 6 ++ release-tasks.sh | 1 + .../fastly_vcl/whitelisted_params_spec.rb | 73 +++++++++++++++++++ 7 files changed, 253 insertions(+) create mode 100644 app/services/fastly_vcl/whitelisted_params.rb create mode 100644 config/fastly/whitelisted_params.yml create mode 100644 docs/backend/fastly.md create mode 100644 lib/tasks/fastly.rake create mode 100644 spec/services/fastly_vcl/whitelisted_params_spec.rb diff --git a/app/services/fastly_vcl/whitelisted_params.rb b/app/services/fastly_vcl/whitelisted_params.rb new file mode 100644 index 000000000..a7045ef22 --- /dev/null +++ b/app/services/fastly_vcl/whitelisted_params.rb @@ -0,0 +1,67 @@ +module FastlyVCL + # Handles updates to our VCL snippet on Fastly that whitelists params + class WhitelistedParams + VCL_DELIMITER_START = "^(".freeze + VCL_DELIMITER_END = ")$".freeze + SNIPPET_NAME = "Whitelist certain querystring parameters".freeze + FILE_PARAMS = YAML.load_file("config/fastly/whitelisted_params.yml").freeze + + class << self + def update + fastly = Fastly.new(api_key: ApplicationConfig["FASTLY_API_KEY"]) + service = fastly.get_service(ApplicationConfig["FASTLY_SERVICE_ID"]) + latest_version = service.version + + snippet = fastly.get_snippet(ApplicationConfig["FASTLY_SERVICE_ID"], + latest_version.number, + SNIPPET_NAME) + + current_params = params_to_array(snippet.content) + + return unless params_updated?(current_params) + + new_version = latest_version.clone + new_snippet = fastly.get_snippet(ApplicationConfig["FASTLY_SERVICE_ID"], + new_version.number, + SNIPPET_NAME) + new_snippet.content = build_content(FILE_PARAMS, new_snippet.content) + new_snippet.save! + + new_version.activate! + log_params_diff_to_datadaog(current_params, new_version) + Rails.logger.info("Fastly updated to version #{new_version.number}.") + end + + private + + def params_updated?(current_params) + (current_params - FILE_PARAMS).any? || (FILE_PARAMS - current_params).any? + end + + def params_to_array(snippet_content) + snippet_suffix = snippet_content.split(VCL_DELIMITER_START).last + fastly_params = snippet_suffix.split(VCL_DELIMITER_END).first + + fastly_params.split("|") + end + + def build_content(new_params, snippet_content) + new_params = new_params.join("|") + snippet_prefix = snippet_content.split(VCL_DELIMITER_START).first + snippet_suffix = snippet_content.split(VCL_DELIMITER_END).last + + "#{snippet_prefix}#{VCL_DELIMITER_START}#{new_params}#{VCL_DELIMITER_END}#{snippet_suffix}" + end + + def log_params_diff_to_datadaog(current_params, new_version) + tags = [ + "added_params:#{(FILE_PARAMS - current_params).join(', ')}", + "removed_params:#{(current_params - FILE_PARAMS).join(', ')}", + "new_version:#{new_version.number}", + ] + + DatadogStatsClient.increment("fastly.whitelist", tags: tags) + end + end + end +end diff --git a/config/fastly/whitelisted_params.yml b/config/fastly/whitelisted_params.yml new file mode 100644 index 000000000..b88fd9c51 --- /dev/null +++ b/config/fastly/whitelisted_params.yml @@ -0,0 +1,56 @@ +--- +- a_id +- args +- article_id +- article_ids +- articles +- asc +- callback_url +- category +- chat_channel_id +- client_id +- code +- collection_id +- commentable_id +- commentable_type +- confirmation_token +- created_at +- end +- filter +- followable_id +- followable_type +- fork_id +- i +- key +- message_offset +- name +- oauth_token +- oauth_verifier +- offset +- org_id +- organization_id +- p +- page +- per_page +- prefill +- preview +- purchaser +- reactable_ids +- redirect_uri +- reported_url +- reporter_username +- response_type +- scope +- search +- signature +- sort +- start +- state +- status +- tag +- tag_list +- top +- url +- username +- ut +- verb diff --git a/docs/backend/fastly.md b/docs/backend/fastly.md new file mode 100644 index 000000000..498e95d73 --- /dev/null +++ b/docs/backend/fastly.md @@ -0,0 +1,49 @@ +--- +title: Fastly +--- + +## What is Fastly? + +[Fastly](https://www.fastly.com/) is a third party service we use for caching on +the edge. It allows us to scale up and serve our millions of visitors quickly +and efficiently. + +If you want to learn more about we use Fastly, check out this +[talk](https://www.youtube.com/watch?v=Afy7H04X9Us) that one of our founders, +[@benhalpern](https://dev.to/ben), gave at RailsConf 2018 talking about how we +made our app so fast it went viral. + +## Whitelisting query string parameters + +In the context of contributing, here's what you need to know about Fastly. In +order for our servers to receive any sort of query string parameters in a +request, they must first be whitelisted in Fastly. For example, if you're +creating a new API endpoint or updating an existing one to accept new +parameters, you'll need to update Fastly. + +The reason we whitelist parameters in Fastly this way is so we don't have to +consider junk parameters when busting the caches. Check out our +[`CacheBuster`](https://github.com/thepracticaldev/dev.to/blob/master/app/labor/cache_buster.rb) +to see examples of this. + +Previously this was a manual process done by an internal team member. Now we do +it programmatically using the Fastly +[gem](https://github.com/fastly/fastly-ruby) as of +[this pr](https://github.com/thepracticaldev/dev.to/pull/7279). + +## How it works + +We created a new file, `config/fastly/whitelisted_params.yml`, to house all of +the whitelisted params in Fastly. + +If you need to update this list, simply update this file. It's as easy as that! + +_Fastly is not setup for development._ + +## In production + +Whitelisted params on Fastly are updated automatically when a production deploy +goes out. + +We do this by executing `bin/rails fastly:update_whitelisted_params` in our +`release-tasks.sh` script. diff --git a/docs/backend/readme.md b/docs/backend/readme.md index 671e3066c..62fd222b3 100644 --- a/docs/backend/readme.md +++ b/docs/backend/readme.md @@ -7,6 +7,7 @@ items: - authorization.md - configuration.md - data-update-scripts.md + - fastly.md - roles.md - algolia.md - pusher.md diff --git a/lib/tasks/fastly.rake b/lib/tasks/fastly.rake new file mode 100644 index 000000000..97a3b0b76 --- /dev/null +++ b/lib/tasks/fastly.rake @@ -0,0 +1,6 @@ +namespace :fastly do + desc "Update VCL for whitelisted params on Fastly" + task update_whitelisted_params: :environment do + FastlyVCL::WhitelistedParams.update + end +end diff --git a/release-tasks.sh b/release-tasks.sh index b4a192d9c..64075daef 100755 --- a/release-tasks.sh +++ b/release-tasks.sh @@ -17,6 +17,7 @@ set -Eex # runs migration for Postgres, setups/updates Elasticsearch # and boots the app to check there are no errors STATEMENT_TIMEOUT=180000 bundle exec rails db:migrate +bundle exec rake fastly:update_whitelisted_params bundle exec rake search:setup bundle exec rake data_updates:enqueue_data_update_worker bundle exec rails runner "puts 'app load success'" diff --git a/spec/services/fastly_vcl/whitelisted_params_spec.rb b/spec/services/fastly_vcl/whitelisted_params_spec.rb new file mode 100644 index 000000000..3feb3daeb --- /dev/null +++ b/spec/services/fastly_vcl/whitelisted_params_spec.rb @@ -0,0 +1,73 @@ +require "rails_helper" + +RSpec.describe FastlyVCL::WhitelistedParams, type: :service do + let(:fastly) { instance_double(Fastly) } + let(:fastly_service) { instance_double(Fastly::Service) } + let(:fastly_version) { instance_double(Fastly::Version) } + let(:fastly_snippet) { instance_double(Fastly::Snippet) } + let(:file_params) { YAML.load_file("config/fastly/whitelisted_params.yml") } + let(:snippet_content) { "#{described_class::VCL_DELIMITER_START}#{file_params.join('|')}#{described_class::VCL_DELIMITER_END}" } + + # Fastly isn't setup for test or development environments so we have to stub + # quite a bit here to simulate Fastly working ¯\_(ツ)_/¯ + before do + allow(Fastly).to receive(:new).and_return(fastly) + allow(fastly).to receive(:get_service).and_return(fastly_service) + allow(fastly_service).to receive(:version).and_return(fastly_version) + allow(fastly_version).to receive(:number).and_return(1) + allow(fastly).to receive(:get_snippet).and_return(fastly_snippet) + allow(fastly_snippet).to receive(:content).and_return(snippet_content) + allow(fastly_version).to receive(:clone).and_return(fastly_version) + allow(fastly_version).to receive(:number).and_return(99) + allow(described_class).to receive(:build_content).and_return(snippet_content) + allow(fastly_snippet).to receive(:content).and_return(fastly_snippet) + allow(described_class).to receive(:params_to_array).and_return(file_params) + allow(fastly_snippet).to receive(:content=).and_return(snippet_content) + allow(fastly_snippet).to receive(:save!).and_return(true) + allow(fastly_version).to receive(:activate!).and_return(true) + end + + describe "::update" do + it "doesn't update if the params haven't changed" do + described_class.update + expect(fastly_version).not_to have_received(:clone) + end + + it "updates Fastly if new params are added" do + new_params = file_params + ["new_param"] + stub_const("#{described_class}::FILE_PARAMS", new_params) + described_class.update + expect(fastly_version).to have_received(:activate!) + end + + it "updates Fastly if params are removed" do + new_params = file_params - [file_params.last] + stub_const("#{described_class}::FILE_PARAMS", new_params) + described_class.update + expect(fastly_version).to have_received(:activate!) + end + + it "overwrites updates made directly in Fastly" do + new_fastly_params = file_params + ["new_fastly_param"] + allow(described_class).to receive(:params_to_array).and_return(new_fastly_params) + described_class.update + expect(fastly_version).to have_received(:activate!) + end + + it "logs success messages" do + allow(Rails.logger).to receive(:info) + allow(DatadogStatsClient).to receive(:increment) + + old_param = file_params.last + new_params = file_params + ["new_param"] - [old_param] + stub_const("#{described_class}::FILE_PARAMS", new_params) + described_class.update + + expect(Rails.logger).to have_received(:info) + + tags = hash_including(tags: array_including("added_params:new_param", "removed_params:#{old_param}", "new_version:#{fastly_version.number}")) + + expect(DatadogStatsClient).to have_received(:increment).with("fastly.whitelist", tags) + end + end +end