Create dynamic robots.txt page (#5794) [deploy]

* Create dynamic robots.txt page

* Add tests and remove cachebust

app/controllers/pages_controller.rb

@@ -1,6 +1,6 @@
 class PagesController < ApplicationController
   # No authorization required for entirely public controller
-  before_action :set_cache_control_headers, only: %i[show rlyweb now survey badge shecoded bounty faq]
+  before_action :set_cache_control_headers, only: %i[show rlyweb now survey badge shecoded bounty faq robots]
 
   def show
     @page = Page.find_by!(slug: params[:slug])
@@ -51,6 +51,11 @@ class PagesController < ApplicationController
     render "pages/report-abuse"
   end
 
+  def robots
+    respond_to :text
+    set_surrogate_key_header "robots_page"
+  end
+
   def rlyweb
     set_surrogate_key_header "rlyweb"
   end

app/views/pages/robots.text.erb

@@ -0,0 +1,6 @@
+User-agent: *
+Disallow: /users/auth/twitter*
+Disallow: /users/auth/github*
+Disallow: /report-abuse?url=*
+
+Sitemap: https://<%= ApplicationConfig["AWS_BUCKET_NAME"] %>.s3.amazonaws.com/sitemaps/sitemap.xml.gz

config/routes.rb

@@ -270,6 +270,7 @@ Rails.application.routes.draw do
 
   # You can have the root of your site routed with "root
   get "/about" => "pages#about"
+  get "/robots.:format" => "pages#robots"
   get "/api", to: redirect("https://docs.dev.to/api")
   get "/privacy" => "pages#privacy"
   get "/terms" => "pages#terms"

public/robots.txt

@@ -1,7 +0,0 @@
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-agent: *
-# Disallow: /
-
-Sitemap: https://thepracticaldev.s3.amazonaws.com/sitemaps/sitemap.xml.gz

spec/requests/pages_spec.rb

@@ -114,4 +114,11 @@ RSpec.describe "Pages", type: :request do
       end
     end
   end
+
+  describe "GET /robots.txt" do
+    it "has proper text" do
+      get "/robots.txt"
+      expect(response.body).to include("Sitemap: https://#{ApplicationConfig['AWS_BUCKET_NAME']}.s3.amazonaws.com/sitemaps/sitemap.xml.gz")
+    end
+  end
 end