diff --git a/config/fastly/snippets/bad_bot_detection.vcl b/config/fastly/snippets/bad_bot_detection.vcl index d27248431..1c9d0dc01 100644 --- a/config/fastly/snippets/bad_bot_detection.vcl +++ b/config/fastly/snippets/bad_bot_detection.vcl @@ -36,6 +36,12 @@ sub vcl_recv { || req.http.user-agent ~ "Microsoft URL Control" || req.http.user-agent ~ "Indy Library" || req.http.user-agent ~ "Fuzz Faster" + # Spoofed user agent, Firefox 62 was released in Sep 2018, this line added + # in Dec 2021. + || (req.http.user-agent ~ "Firefox/62.0" && req.http.user-agent ~ "Win64") + # DEV gets an order of magnitude more traffic from AhrefsBot than any other + # search crawler. + || req.http.user-agent ~ "AhrefsBot" || req.http.user-agent == "8484 Boston Project v 1.0" || req.http.user-agent == "Atomic_Email_Hunter/4.0" || req.http.user-agent == "atSpider/1.0" @@ -107,6 +113,8 @@ sub vcl_recv { || req.http.user-agent == "RSurf15a 41" || req.http.user-agent == "RSurf15a 51" || req.http.user-agent == "RSurf15a 81" + # Block Ruby bots unless they're interacting with the API + || (req.http.user-agent == "Ruby" && !(req.http.url ~ "^/api")) || req.http.user-agent == "searchbot admin@google.com" || req.http.user-agent == "ShablastBot 1.0" || req.http.user-agent == "snap.com beta crawler v0"