From 82f2281b1c139d919de6b88065774f5bc742f075 Mon Sep 17 00:00:00 2001
From: Ben Halpern <bendhalpern@gmail.com>
Date: Tue, 28 Jan 2020 16:49:34 -0500
Subject: [PATCH] Create dynamic robots.txt page (#5794) [deploy]

* Create dynamic robots.txt page

* Add tests and remove cachebust
---
 app/controllers/pages_controller.rb | 7 ++++++-
 app/views/pages/robots.text.erb     | 6 ++++++
 config/routes.rb                    | 1 +
 public/robots.txt                   | 7 -------
 spec/requests/pages_spec.rb         | 7 +++++++
 5 files changed, 20 insertions(+), 8 deletions(-)
 create mode 100644 app/views/pages/robots.text.erb
 delete mode 100644 public/robots.txt

diff --git a/app/controllers/pages_controller.rb b/app/controllers/pages_controller.rb
index 94c456cc0..0cce46c05 100644
--- a/app/controllers/pages_controller.rb
+++ b/app/controllers/pages_controller.rb
@@ -1,6 +1,6 @@
 class PagesController < ApplicationController
   # No authorization required for entirely public controller
-  before_action :set_cache_control_headers, only: %i[show rlyweb now survey badge shecoded bounty faq]
+  before_action :set_cache_control_headers, only: %i[show rlyweb now survey badge shecoded bounty faq robots]
 
   def show
     @page = Page.find_by!(slug: params[:slug])
@@ -51,6 +51,11 @@ class PagesController < ApplicationController
     render "pages/report-abuse"
   end
 
+  def robots
+    respond_to :text
+    set_surrogate_key_header "robots_page"
+  end
+
   def rlyweb
     set_surrogate_key_header "rlyweb"
   end
diff --git a/app/views/pages/robots.text.erb b/app/views/pages/robots.text.erb
new file mode 100644
index 000000000..7c762e2eb
--- /dev/null
+++ b/app/views/pages/robots.text.erb
@@ -0,0 +1,6 @@
+User-agent: *
+Disallow: /users/auth/twitter*
+Disallow: /users/auth/github*
+Disallow: /report-abuse?url=*
+
+Sitemap: https://<%= ApplicationConfig["AWS_BUCKET_NAME"] %>.s3.amazonaws.com/sitemaps/sitemap.xml.gz
\ No newline at end of file
diff --git a/config/routes.rb b/config/routes.rb
index b741f133f..96e34afdd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -270,6 +270,7 @@ Rails.application.routes.draw do
 
   # You can have the root of your site routed with "root
   get "/about" => "pages#about"
+  get "/robots.:format" => "pages#robots"
   get "/api", to: redirect("https://docs.dev.to/api")
   get "/privacy" => "pages#privacy"
   get "/terms" => "pages#terms"
diff --git a/public/robots.txt b/public/robots.txt
deleted file mode 100644
index 22bf35a50..000000000
--- a/public/robots.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-agent: *
-# Disallow: /
-
-Sitemap: https://thepracticaldev.s3.amazonaws.com/sitemaps/sitemap.xml.gz
\ No newline at end of file
diff --git a/spec/requests/pages_spec.rb b/spec/requests/pages_spec.rb
index 6bbf6c1d6..a0b48e83e 100644
--- a/spec/requests/pages_spec.rb
+++ b/spec/requests/pages_spec.rb
@@ -114,4 +114,11 @@ RSpec.describe "Pages", type: :request do
       end
     end
   end
+
+  describe "GET /robots.txt" do
+    it "has proper text" do
+      get "/robots.txt"
+      expect(response.body).to include("Sitemap: https://#{ApplicationConfig['AWS_BUCKET_NAME']}.s3.amazonaws.com/sitemaps/sitemap.xml.gz")
+    end
+  end
 end