From a43d25f6fec53e42108cbddaa3f06a7ba6a7becc Mon Sep 17 00:00:00 2001 From: Ben Halpern Date: Wed, 9 Feb 2022 12:19:28 -0500 Subject: [PATCH] Add Disallow URLs to robots.txt for uselessly crawled endpoints (#16501) --- app/views/pages/robots.text.erb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/views/pages/robots.text.erb b/app/views/pages/robots.text.erb index 2c7eed485..0e7b31a6e 100644 --- a/app/views/pages/robots.text.erb +++ b/app/views/pages/robots.text.erb @@ -11,5 +11,7 @@ Disallow: /listings*?q=* Disallow: /mod/* Disallow: /mod?* Disallow: /admin/* +Disallow: /reactions?* +Disallow: /async_info/base_data Sitemap: <%= URL.url("sitemap-index.xml") %> \ No newline at end of file