From b74b035e22a248e8141166b03dbe8b4d0ee3b7fe Mon Sep 17 00:00:00 2001
From: Alex <alexandersmith223@gmail.com>
Date: Mon, 4 May 2020 10:18:19 -0400
Subject: [PATCH] [deploy] Add new VCL snippets for Fastly (#7654)

* Add new VCL snippets

* Fix specs

* Partial match Fuzz Faster

* Remove PHP
---
 config/fastly/snippets/bad_bot_detection.vcl  | 139 ++++++++++++++++++
 ...ber_user_token_in_cookie_to_set_header.vcl |  19 +++
 .../stale_while_revalidate_controls.vcl       |   5 +
 spec/services/fastly_config/snippets_spec.rb  |   6 +-
 4 files changed, 166 insertions(+), 3 deletions(-)
 create mode 100644 config/fastly/snippets/bad_bot_detection.vcl
 create mode 100644 config/fastly/snippets/check_for_remember_user_token_in_cookie_to_set_header.vcl
 create mode 100644 config/fastly/snippets/stale_while_revalidate_controls.vcl

diff --git a/config/fastly/snippets/bad_bot_detection.vcl b/config/fastly/snippets/bad_bot_detection.vcl
new file mode 100644
index 000000000..d27248431
--- /dev/null
+++ b/config/fastly/snippets/bad_bot_detection.vcl
@@ -0,0 +1,139 @@
+sub vcl_recv {
+  # Disallow bot traffic based on user-agent
+  if (
+    req.http.user-agent ~ "^$"
+    || req.http.user-agent ~ "^Java"
+    || req.http.user-agent ~ "^Jakarta"
+    || req.http.user-agent ~ "IDBot"
+    || req.http.user-agent ~ "id-search"
+    || req.http.user-agent ~ "User-Agent"
+    || req.http.user-agent ~ "compatible ;"
+    || req.http.user-agent ~ "ConveraCrawler"
+    || req.http.user-agent ~ "^Mozilla$"
+    || req.http.user-agent ~ "libwww"
+    || req.http.user-agent ~ "lwp-trivial"
+    || req.http.user-agent ~ "urllib"
+    || req.http.user-agent ~ "GT:WWW"
+    || req.http.user-agent ~ "Snoopy"
+    || req.http.user-agent ~ "MFC_Tear_Sample"
+    || req.http.user-agent ~ "HTTP::Lite"
+    || req.http.user-agent ~ "PHPCrawl"
+    || req.http.user-agent ~ "URI::Fetch"
+    || req.http.user-agent ~ "Zend_Http_Client"
+    || req.http.user-agent ~ "http client"
+    || req.http.user-agent ~ "PECL::HTTP"
+    || req.http.user-agent ~ "panscient.com"
+    || req.http.user-agent ~ "IBM EVV"
+    || req.http.user-agent ~ "Bork-edition"
+    || req.http.user-agent ~ "Fetch API Request"
+    || req.http.user-agent ~ "PleaseCrawl"
+    || req.http.user-agent ~ "[A-Z][a-z]{3,} [a-z]{4,} [a-z]{4,}"
+    || req.http.user-agent ~ "layeredtech.com"
+    || req.http.user-agent ~ "WEP Search"
+    || req.http.user-agent ~ "Wells Search II"
+    || req.http.user-agent ~ "Missigua Locator"
+    || req.http.user-agent ~ "ISC Systems iRc Search 2.1"
+    || req.http.user-agent ~ "Microsoft URL Control"
+    || req.http.user-agent ~ "Indy Library"
+    || req.http.user-agent ~ "Fuzz Faster"
+    || req.http.user-agent == "8484 Boston Project v 1.0"
+    || req.http.user-agent == "Atomic_Email_Hunter/4.0"
+    || req.http.user-agent == "atSpider/1.0"
+    || req.http.user-agent == "autoemailspider"
+    || req.http.user-agent == "China Local Browse 2.6"
+    || req.http.user-agent == "ContactBot/0.2"
+    || req.http.user-agent == "ContentSmartz"
+    || req.http.user-agent == "DataCha0s/2.0"
+    || req.http.user-agent == "DataCha0s/2.0"
+    || req.http.user-agent == "DBrowse 1.4b"
+    || req.http.user-agent == "DBrowse 1.4d"
+    || req.http.user-agent == "Demo Bot DOT 16b"
+    || req.http.user-agent == "Demo Bot Z 16b"
+    || req.http.user-agent == "DSurf15a 01"
+    || req.http.user-agent == "DSurf15a 71"
+    || req.http.user-agent == "DSurf15a 81"
+    || req.http.user-agent == "DSurf15a VA"
+    || req.http.user-agent == "EBrowse 1.4b"
+    || req.http.user-agent == "Educate Search VxB"
+    || req.http.user-agent == "EmailSiphon"
+    || req.http.user-agent == "EmailWolf 1.00"
+    || req.http.user-agent == "ESurf15a 15"
+    || req.http.user-agent == "ExtractorPro"
+    || req.http.user-agent == "Franklin Locator 1.8"
+    || req.http.user-agent == "FSurf15a 01"
+    || req.http.user-agent == "Full Web Bot 0416B"
+    || req.http.user-agent == "Full Web Bot 0516B"
+    || req.http.user-agent == "Full Web Bot 2816B"
+    || req.http.user-agent == "Guestbook Auto Submitter"
+    || req.http.user-agent == "Industry Program 1.0.x"
+    || req.http.user-agent == "ISC Systems iRc Search 2.1"
+    || req.http.user-agent == "IUPUI Research Bot v 1.9a"
+    || req.http.user-agent == "LARBIN-EXPERIMENTAL (efp@gmx.net)"
+    || req.http.user-agent == "LetsCrawl.com/1.0 +http://letscrawl.com/"
+    || req.http.user-agent == "Lincoln State Web Browser"
+    || req.http.user-agent == "LMQueueBot/0.2"
+    || req.http.user-agent == "LWP::Simple/5.803"
+    || req.http.user-agent == "Mac Finder 1.0.xx"
+    || req.http.user-agent == "MFC Foundation Class Library 4.0"
+    || req.http.user-agent == "Microsoft URL Control - 6.00.8xxx"
+    || req.http.user-agent == "Missauga Locate 1.0.0"
+    || req.http.user-agent == "Missigua Locator 1.9"
+    || req.http.user-agent == "Missouri College Browse"
+    || req.http.user-agent == "Mizzu Labs 2.2"
+    || req.http.user-agent == "Mo College 1.9"
+    || req.http.user-agent == "Mozilla/2.0 (compatible; NEWT ActiveX; Win32)"
+    || req.http.user-agent == "Mozilla/3.0 (compatible; Indy Library)"
+    || req.http.user-agent == "Mozilla/4.0 (compatible; Advanced Email Extractor v2.xx)"
+    || req.http.user-agent == "Mozilla/4.0 (compatible; Iplexx Spider/1.0 http://www.iplexx.at)"
+    || req.http.user-agent == "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent"
+    || req.http.user-agent == "Mozilla/4.0 efp@gmx.net"
+    || req.http.user-agent == "Mozilla/5.0 (Version: xxxx Type:xx)"
+    || req.http.user-agent == "MVAClient"
+    || req.http.user-agent == "NameOfAgent (CMS Spider)"
+    || req.http.user-agent == "NASA Search 1.0"
+    || req.http.user-agent == "Nsauditor/1.x"
+    || req.http.user-agent == "PBrowse 1.4b"
+    || req.http.user-agent == "PEval 1.4b"
+    || req.http.user-agent == "Poirot"
+    || req.http.user-agent == "Port Huron Labs"
+    || req.http.user-agent == "Production Bot 0116B"
+    || req.http.user-agent == "Production Bot 2016B"
+    || req.http.user-agent == "Production Bot DOT 3016B"
+    || req.http.user-agent == "Program Shareware 1.0.2"
+    || req.http.user-agent == "PSurf15a 11"
+    || req.http.user-agent == "PSurf15a 51"
+    || req.http.user-agent == "PSurf15a VA"
+    || req.http.user-agent == "psycheclone"
+    || req.http.user-agent == "RSurf15a 41"
+    || req.http.user-agent == "RSurf15a 51"
+    || req.http.user-agent == "RSurf15a 81"
+    || req.http.user-agent == "searchbot admin@google.com"
+    || req.http.user-agent == "ShablastBot 1.0"
+    || req.http.user-agent == "snap.com beta crawler v0"
+    || req.http.user-agent == "Snapbot/1.0"
+    || req.http.user-agent == "sogou develop spider"
+    || req.http.user-agent == "Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
+    || req.http.user-agent == "sogou spider"
+    || req.http.user-agent == "Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
+    || req.http.user-agent == "sohu agent"
+    || req.http.user-agent == "SSurf15a 11"
+    || req.http.user-agent == "TSurf15a 11"
+    || req.http.user-agent == "Under the Rainbow 2.2"
+    || req.http.user-agent == "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"
+    || req.http.user-agent == "VadixBot"
+    || req.http.user-agent == "WebVulnCrawl.blogspot.com/1.0 libwww-perl/5.803"
+    || req.http.user-agent == "Wells Search II"
+    || req.http.user-agent == "WEP Search 00"
+  ) {
+    error 703 "Forbidden Bots";
+  }
+}
+
+sub vcl_error {
+  if (obj.status == 703) {
+    set obj.status = 403;
+    set obj.response = "Forbidden Bots";
+    synthetic {""};
+    return (deliver);
+  }
+}
diff --git a/config/fastly/snippets/check_for_remember_user_token_in_cookie_to_set_header.vcl b/config/fastly/snippets/check_for_remember_user_token_in_cookie_to_set_header.vcl
new file mode 100644
index 000000000..979415e21
--- /dev/null
+++ b/config/fastly/snippets/check_for_remember_user_token_in_cookie_to_set_header.vcl
@@ -0,0 +1,19 @@
+sub vcl_recv {
+  if (req.http.Cookie ~ "remember_user_token") {
+   set req.http.X-Loggedin = "logged-in";
+  } else {
+    set req.http.X-Loggedin = "logged-out";
+  }
+
+}
+
+sub vcl_fetch {
+  # Vary on the custom header, don't add if shield POP already added
+  if (beresp.http.Vary !~ "X-Loggedin") {
+    if (beresp.http.Vary) {
+	    set beresp.http.Vary = beresp.http.Vary ", X-Loggedin";
+    } else {
+	    set beresp.http.Vary = "X-Loggedin";
+    }
+  }
+}
diff --git a/config/fastly/snippets/stale_while_revalidate_controls.vcl b/config/fastly/snippets/stale_while_revalidate_controls.vcl
new file mode 100644
index 000000000..10fe02590
--- /dev/null
+++ b/config/fastly/snippets/stale_while_revalidate_controls.vcl
@@ -0,0 +1,5 @@
+sub vcl_fetch {
+  if (req.url == "/" || req.url == "/?i=i") {
+    set beresp.stale_while_revalidate = 15s;
+  }
+}
diff --git a/spec/services/fastly_config/snippets_spec.rb b/spec/services/fastly_config/snippets_spec.rb
index 0b95b0f0d..55172b594 100644
--- a/spec/services/fastly_config/snippets_spec.rb
+++ b/spec/services/fastly_config/snippets_spec.rb
@@ -20,7 +20,7 @@ RSpec.describe FastlyConfig::Snippets, type: :service do
       allow(fastly).to receive(:create).and_return(fastly_snippet)
       allow(fastly_snippet).to receive(:name).and_return("test")
       snippets_config.update(fastly_version)
-      expect(fastly).to have_received(:create)
+      expect(fastly).to have_received(:create).at_least(:once)
     end
 
     it "updates a snippet if one is found" do
@@ -31,7 +31,7 @@ RSpec.describe FastlyConfig::Snippets, type: :service do
       allow(fastly_snippet).to receive(:save!).and_return(fastly_snippet)
       allow(fastly_snippet).to receive(:name).and_return("test")
       snippets_config.update(fastly_version)
-      expect(fastly_snippet).to have_received(:save!)
+      expect(fastly_snippet).to have_received(:save!).at_least(:once)
     end
 
     it "logs success messages" do
@@ -47,7 +47,7 @@ RSpec.describe FastlyConfig::Snippets, type: :service do
 
       tags = hash_including(tags: array_including("snippet_update_type:update", "snippet_name:test", "new_version:#{fastly_version.number}"))
 
-      expect(DatadogStatsClient).to have_received(:increment).with("fastly.snippets", tags)
+      expect(DatadogStatsClient).to have_received(:increment).with("fastly.snippets", tags).at_least(:once)
     end
   end
 end