From 2c07b085149900d9cd417aaaebcd0091d4601677 Mon Sep 17 00:00:00 2001 From: Ben Halpern Date: Fri, 7 Feb 2020 14:04:36 -0500 Subject: [PATCH] Add low-value landing pages to robots.txt disallow (#5944) [deploy] * Add low-value landing pages to robots.txt disallow * Update app/views/pages/robots.text.erb * Update app/views/pages/robots.text.erb --- app/views/pages/robots.text.erb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/views/pages/robots.text.erb b/app/views/pages/robots.text.erb index 7c762e2eb..fae5d9900 100644 --- a/app/views/pages/robots.text.erb +++ b/app/views/pages/robots.text.erb @@ -2,5 +2,9 @@ User-agent: * Disallow: /users/auth/twitter* Disallow: /users/auth/github* Disallow: /report-abuse?url=* +Disallow: /connect/@* +Disallow: /search?q=* +Disallow: /search/?q=* +Disallow: /listings*?q=* -Sitemap: https://<%= ApplicationConfig["AWS_BUCKET_NAME"] %>.s3.amazonaws.com/sitemaps/sitemap.xml.gz \ No newline at end of file +Sitemap: https://<%= ApplicationConfig["AWS_BUCKET_NAME"] %>.s3.amazonaws.com/sitemaps/sitemap.xml.gz