From 329f33448f7c2b13ba94a3bedba191f39526d9dd Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Fri, 17 Dec 2021 15:02:30 -0500 Subject: [PATCH] Disallow null user agent and obsolete browser (#15817) We were previously disallowing empty user agents, but we weren't blocking requests where the User-Agent header wasn't set at all. This commit blocks those requests. This commit also blocks Chrome 74. It's a 2.5-year-old version of an evergreen browser that releases every 6 weeks, so this is clearly a bot spoofing this header. --- config/fastly/snippets/bad_bot_detection.vcl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/fastly/snippets/bad_bot_detection.vcl b/config/fastly/snippets/bad_bot_detection.vcl index 1d419ac52..72f5396e7 100644 --- a/config/fastly/snippets/bad_bot_detection.vcl +++ b/config/fastly/snippets/bad_bot_detection.vcl @@ -1,7 +1,8 @@ sub vcl_recv { # Disallow bot traffic based on user-agent if ( - req.http.user-agent ~ "^$" + req.http.user-agent ~ "^$" # Disallow empty User-Agent + || req.http.user-agent !~ "." # Disallow null User-Agent || req.http.user-agent ~ "^Java" || req.http.user-agent ~ "^Jakarta" || req.http.user-agent ~ "IDBot" @@ -39,6 +40,9 @@ sub vcl_recv { # Spoofed user agent, Firefox 62 was released in Sep 2018, this line added # in Dec 2021. || (req.http.user-agent ~ "Firefox/62.0" && req.http.user-agent ~ "Win64") + # Spoofed user agent, Chrome 74 was released in April 2019, this line added + # in Dec 2021. + || req.http.user-agent ~ "Chrome/74" # DEV gets an order of magnitude more traffic from AhrefsBot than any other # search crawler. || req.http.user-agent ~ "AhrefsBot"