diff --git a/app/views/pages/robots.text.erb b/app/views/pages/robots.text.erb index 7c762e2eb..fae5d9900 100644 --- a/app/views/pages/robots.text.erb +++ b/app/views/pages/robots.text.erb @@ -2,5 +2,9 @@ User-agent: * Disallow: /users/auth/twitter* Disallow: /users/auth/github* Disallow: /report-abuse?url=* +Disallow: /connect/@* +Disallow: /search?q=* +Disallow: /search/?q=* +Disallow: /listings*?q=* -Sitemap: https://<%= ApplicationConfig["AWS_BUCKET_NAME"] %>.s3.amazonaws.com/sitemaps/sitemap.xml.gz \ No newline at end of file +Sitemap: https://<%= ApplicationConfig["AWS_BUCKET_NAME"] %>.s3.amazonaws.com/sitemaps/sitemap.xml.gz