[deploy] Add new VCL snippets for Fastly (#7654)
* Add new VCL snippets * Fix specs * Partial match Fuzz Faster * Remove PHP
This commit is contained in:
Alex
GitHub
parent
2ad5403045
commit
b74b035e22
4 changed files with 166 additions and 3 deletions
139
config/fastly/snippets/bad_bot_detection.vcl
Normal file
139
config/fastly/snippets/bad_bot_detection.vcl
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
sub vcl_recv {
|
||||
# Disallow bot traffic based on user-agent
|
||||
if (
|
||||
req.http.user-agent ~ "^$"
|
||||
|| req.http.user-agent ~ "^Java"
|
||||
|| req.http.user-agent ~ "^Jakarta"
|
||||
|| req.http.user-agent ~ "IDBot"
|
||||
|| req.http.user-agent ~ "id-search"
|
||||
|| req.http.user-agent ~ "User-Agent"
|
||||
|| req.http.user-agent ~ "compatible ;"
|
||||
|| req.http.user-agent ~ "ConveraCrawler"
|
||||
|| req.http.user-agent ~ "^Mozilla$"
|
||||
|| req.http.user-agent ~ "libwww"
|
||||
|| req.http.user-agent ~ "lwp-trivial"
|
||||
|| req.http.user-agent ~ "urllib"
|
||||
|| req.http.user-agent ~ "GT:WWW"
|
||||
|| req.http.user-agent ~ "Snoopy"
|
||||
|| req.http.user-agent ~ "MFC_Tear_Sample"
|
||||
|| req.http.user-agent ~ "HTTP::Lite"
|
||||
|| req.http.user-agent ~ "PHPCrawl"
|
||||
|| req.http.user-agent ~ "URI::Fetch"
|
||||
|| req.http.user-agent ~ "Zend_Http_Client"
|
||||
|| req.http.user-agent ~ "http client"
|
||||
|| req.http.user-agent ~ "PECL::HTTP"
|
||||
|| req.http.user-agent ~ "panscient.com"
|
||||
|| req.http.user-agent ~ "IBM EVV"
|
||||
|| req.http.user-agent ~ "Bork-edition"
|
||||
|| req.http.user-agent ~ "Fetch API Request"
|
||||
|| req.http.user-agent ~ "PleaseCrawl"
|
||||
|| req.http.user-agent ~ "[A-Z][a-z]{3,} [a-z]{4,} [a-z]{4,}"
|
||||
|| req.http.user-agent ~ "layeredtech.com"
|
||||
|| req.http.user-agent ~ "WEP Search"
|
||||
|| req.http.user-agent ~ "Wells Search II"
|
||||
|| req.http.user-agent ~ "Missigua Locator"
|
||||
|| req.http.user-agent ~ "ISC Systems iRc Search 2.1"
|
||||
|| req.http.user-agent ~ "Microsoft URL Control"
|
||||
|| req.http.user-agent ~ "Indy Library"
|
||||
|| req.http.user-agent ~ "Fuzz Faster"
|
||||
|| req.http.user-agent == "8484 Boston Project v 1.0"
|
||||
|| req.http.user-agent == "Atomic_Email_Hunter/4.0"
|
||||
|| req.http.user-agent == "atSpider/1.0"
|
||||
|| req.http.user-agent == "autoemailspider"
|
||||
|| req.http.user-agent == "China Local Browse 2.6"
|
||||
|| req.http.user-agent == "ContactBot/0.2"
|
||||
|| req.http.user-agent == "ContentSmartz"
|
||||
|| req.http.user-agent == "DataCha0s/2.0"
|
||||
|| req.http.user-agent == "DataCha0s/2.0"
|
||||
|| req.http.user-agent == "DBrowse 1.4b"
|
||||
|| req.http.user-agent == "DBrowse 1.4d"
|
||||
|| req.http.user-agent == "Demo Bot DOT 16b"
|
||||
|| req.http.user-agent == "Demo Bot Z 16b"
|
||||
|| req.http.user-agent == "DSurf15a 01"
|
||||
|| req.http.user-agent == "DSurf15a 71"
|
||||
|| req.http.user-agent == "DSurf15a 81"
|
||||
|| req.http.user-agent == "DSurf15a VA"
|
||||
|| req.http.user-agent == "EBrowse 1.4b"
|
||||
|| req.http.user-agent == "Educate Search VxB"
|
||||
|| req.http.user-agent == "EmailSiphon"
|
||||
|| req.http.user-agent == "EmailWolf 1.00"
|
||||
|| req.http.user-agent == "ESurf15a 15"
|
||||
|| req.http.user-agent == "ExtractorPro"
|
||||
|| req.http.user-agent == "Franklin Locator 1.8"
|
||||
|| req.http.user-agent == "FSurf15a 01"
|
||||
|| req.http.user-agent == "Full Web Bot 0416B"
|
||||
|| req.http.user-agent == "Full Web Bot 0516B"
|
||||
|| req.http.user-agent == "Full Web Bot 2816B"
|
||||
|| req.http.user-agent == "Guestbook Auto Submitter"
|
||||
|| req.http.user-agent == "Industry Program 1.0.x"
|
||||
|| req.http.user-agent == "ISC Systems iRc Search 2.1"
|
||||
|| req.http.user-agent == "IUPUI Research Bot v 1.9a"
|
||||
|| req.http.user-agent == "LARBIN-EXPERIMENTAL (efp@gmx.net)"
|
||||
|| req.http.user-agent == "LetsCrawl.com/1.0 +http://letscrawl.com/"
|
||||
|| req.http.user-agent == "Lincoln State Web Browser"
|
||||
|| req.http.user-agent == "LMQueueBot/0.2"
|
||||
|| req.http.user-agent == "LWP::Simple/5.803"
|
||||
|| req.http.user-agent == "Mac Finder 1.0.xx"
|
||||
|| req.http.user-agent == "MFC Foundation Class Library 4.0"
|
||||
|| req.http.user-agent == "Microsoft URL Control - 6.00.8xxx"
|
||||
|| req.http.user-agent == "Missauga Locate 1.0.0"
|
||||
|| req.http.user-agent == "Missigua Locator 1.9"
|
||||
|| req.http.user-agent == "Missouri College Browse"
|
||||
|| req.http.user-agent == "Mizzu Labs 2.2"
|
||||
|| req.http.user-agent == "Mo College 1.9"
|
||||
|| req.http.user-agent == "Mozilla/2.0 (compatible; NEWT ActiveX; Win32)"
|
||||
|| req.http.user-agent == "Mozilla/3.0 (compatible; Indy Library)"
|
||||
|| req.http.user-agent == "Mozilla/4.0 (compatible; Advanced Email Extractor v2.xx)"
|
||||
|| req.http.user-agent == "Mozilla/4.0 (compatible; Iplexx Spider/1.0 http://www.iplexx.at)"
|
||||
|| req.http.user-agent == "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent"
|
||||
|| req.http.user-agent == "Mozilla/4.0 efp@gmx.net"
|
||||
|| req.http.user-agent == "Mozilla/5.0 (Version: xxxx Type:xx)"
|
||||
|| req.http.user-agent == "MVAClient"
|
||||
|| req.http.user-agent == "NameOfAgent (CMS Spider)"
|
||||
|| req.http.user-agent == "NASA Search 1.0"
|
||||
|| req.http.user-agent == "Nsauditor/1.x"
|
||||
|| req.http.user-agent == "PBrowse 1.4b"
|
||||
|| req.http.user-agent == "PEval 1.4b"
|
||||
|| req.http.user-agent == "Poirot"
|
||||
|| req.http.user-agent == "Port Huron Labs"
|
||||
|| req.http.user-agent == "Production Bot 0116B"
|
||||
|| req.http.user-agent == "Production Bot 2016B"
|
||||
|| req.http.user-agent == "Production Bot DOT 3016B"
|
||||
|| req.http.user-agent == "Program Shareware 1.0.2"
|
||||
|| req.http.user-agent == "PSurf15a 11"
|
||||
|| req.http.user-agent == "PSurf15a 51"
|
||||
|| req.http.user-agent == "PSurf15a VA"
|
||||
|| req.http.user-agent == "psycheclone"
|
||||
|| req.http.user-agent == "RSurf15a 41"
|
||||
|| req.http.user-agent == "RSurf15a 51"
|
||||
|| req.http.user-agent == "RSurf15a 81"
|
||||
|| req.http.user-agent == "searchbot admin@google.com"
|
||||
|| req.http.user-agent == "ShablastBot 1.0"
|
||||
|| req.http.user-agent == "snap.com beta crawler v0"
|
||||
|| req.http.user-agent == "Snapbot/1.0"
|
||||
|| req.http.user-agent == "sogou develop spider"
|
||||
|| req.http.user-agent == "Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
|
||||
|| req.http.user-agent == "sogou spider"
|
||||
|| req.http.user-agent == "Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
|
||||
|| req.http.user-agent == "sohu agent"
|
||||
|| req.http.user-agent == "SSurf15a 11"
|
||||
|| req.http.user-agent == "TSurf15a 11"
|
||||
|| req.http.user-agent == "Under the Rainbow 2.2"
|
||||
|| req.http.user-agent == "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"
|
||||
|| req.http.user-agent == "VadixBot"
|
||||
|| req.http.user-agent == "WebVulnCrawl.blogspot.com/1.0 libwww-perl/5.803"
|
||||
|| req.http.user-agent == "Wells Search II"
|
||||
|| req.http.user-agent == "WEP Search 00"
|
||||
) {
|
||||
error 703 "Forbidden Bots";
|
||||
}
|
||||
}
|
||||
|
||||
sub vcl_error {
|
||||
if (obj.status == 703) {
|
||||
set obj.status = 403;
|
||||
set obj.response = "Forbidden Bots";
|
||||
synthetic {""};
|
||||
return (deliver);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
sub vcl_recv {
|
||||
if (req.http.Cookie ~ "remember_user_token") {
|
||||
set req.http.X-Loggedin = "logged-in";
|
||||
} else {
|
||||
set req.http.X-Loggedin = "logged-out";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
sub vcl_fetch {
|
||||
# Vary on the custom header, don't add if shield POP already added
|
||||
if (beresp.http.Vary !~ "X-Loggedin") {
|
||||
if (beresp.http.Vary) {
|
||||
set beresp.http.Vary = beresp.http.Vary ", X-Loggedin";
|
||||
} else {
|
||||
set beresp.http.Vary = "X-Loggedin";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
sub vcl_fetch {
|
||||
if (req.url == "/" || req.url == "/?i=i") {
|
||||
set beresp.stale_while_revalidate = 15s;
|
||||
}
|
||||
}
|
||||
|
|
@ -20,7 +20,7 @@ RSpec.describe FastlyConfig::Snippets, type: :service do
|
|||
allow(fastly).to receive(:create).and_return(fastly_snippet)
|
||||
allow(fastly_snippet).to receive(:name).and_return("test")
|
||||
snippets_config.update(fastly_version)
|
||||
expect(fastly).to have_received(:create)
|
||||
expect(fastly).to have_received(:create).at_least(:once)
|
||||
end
|
||||
|
||||
it "updates a snippet if one is found" do
|
||||
|
|
@ -31,7 +31,7 @@ RSpec.describe FastlyConfig::Snippets, type: :service do
|
|||
allow(fastly_snippet).to receive(:save!).and_return(fastly_snippet)
|
||||
allow(fastly_snippet).to receive(:name).and_return("test")
|
||||
snippets_config.update(fastly_version)
|
||||
expect(fastly_snippet).to have_received(:save!)
|
||||
expect(fastly_snippet).to have_received(:save!).at_least(:once)
|
||||
end
|
||||
|
||||
it "logs success messages" do
|
||||
|
|
@ -47,7 +47,7 @@ RSpec.describe FastlyConfig::Snippets, type: :service do
|
|||
|
||||
tags = hash_including(tags: array_including("snippet_update_type:update", "snippet_name:test", "new_version:#{fastly_version.number}"))
|
||||
|
||||
expect(DatadogStatsClient).to have_received(:increment).with("fastly.snippets", tags)
|
||||
expect(DatadogStatsClient).to have_received(:increment).with("fastly.snippets", tags).at_least(:once)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue