diff --git a/Gemfile b/Gemfile index adefbbcb2..1032a7e99 100644 --- a/Gemfile +++ b/Gemfile @@ -100,7 +100,6 @@ gem "s3_direct_upload", "~> 0.1" # Direct Upload to Amazon S3 gem "sidekiq", "~> 6.2.2" # Sidekiq is used to process background jobs with the help of Redis gem "sidekiq-cron", "~> 1.1" # Allows execution of scheduled cron jobs as specific times gem "sidekiq-unique-jobs", "~> 7.0.12" # Ensures that Sidekiq jobs are unique when enqueued -gem "sitemap_generator", "~> 6.1" # SitemapGenerator is a framework-agnostic XML Sitemap generator gem "slack-notifier", "~> 2.4" # A slim ruby wrapper for posting to slack webhooks gem "sprockets", "~> 4.0" # Sprockets is a Rack-based asset packaging system gem "staccato", "~> 0.5" # Ruby Google Analytics Measurement diff --git a/Gemfile.lock b/Gemfile.lock index 547275f74..b00b039fe 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -759,8 +759,6 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.2) - sitemap_generator (6.1.2) - builder (~> 3.0) slack-notifier (2.4.0) smart_properties (1.15.0) spring (3.0.0) @@ -996,7 +994,6 @@ DEPENDENCIES sidekiq-cron (~> 1.1) sidekiq-unique-jobs (~> 7.0.12) simplecov (~> 0.21.2) - sitemap_generator (~> 6.1) slack-notifier (~> 2.4) spring (~> 3.0) spring-commands-rspec (~> 1.0) diff --git a/app/controllers/sitemaps_controller.rb b/app/controllers/sitemaps_controller.rb index 71cdcb615..e9f296b4a 100644 --- a/app/controllers/sitemaps_controller.rb +++ b/app/controllers/sitemaps_controller.rb @@ -5,22 +5,46 @@ class SitemapsController < ApplicationController RESULTS_LIMIT = Rails.env.production? ? 10_000 : 5 def show - if params[:sitemap].start_with? "sitemap-posts" - posts_sitemap + if params[:sitemap].start_with? "sitemap-index" + sitemap_index + elsif valid_resource_sitemap? + resource_sitemap(resource_string) else monthly_sitemap end set_cache_control_headers(@max_age, stale_while_revalidate: @stale_while_revalidate, stale_if_error: @stale_if_error) - render layout: false + render @view_template, layout: false end private - def posts_sitemap - @articles = Article.published.order("published_at DESC").limit(RESULTS_LIMIT).offset(offset).pluck(:path, :last_comment_at) - set_surrogate_controls(Time.now) + def sitemap_index + set_surrogate_controls(Time.current) + @articles_count = Article.published + .where("score >= ?", Settings::UserExperience.index_minimum_score).size + @page_limit = RESULTS_LIMIT + @view_template = "index" + end + + def resource_sitemap(resource) + case resource + when "users" + @users = User.order("comments_count DESC") + .where("score > -1") # Spam mitigation + .limit(RESULTS_LIMIT).offset(offset).pluck(:username, :profile_updated_at) + when "posts" + @articles = Article.published.order("published_at DESC") + .where("score >= ?", Settings::UserExperience.index_minimum_score) + .limit(RESULTS_LIMIT).offset(offset).pluck(:path, :last_comment_at) + when "tags" # tags + @tags = Tag.order("hotness_score DESC") + .where(supported: true) + .limit(RESULTS_LIMIT).offset(offset).pluck(:name, :updated_at) + end + set_surrogate_controls(Time.current) + @view_template = resource end def monthly_sitemap @@ -33,11 +57,12 @@ class SitemapsController < ApplicationController end @articles = Article.published - .where("published_at > ? AND published_at < ? AND score > ?", + .where("published_at > ? AND published_at < ? AND score >= ?", date, date.end_of_month, Settings::UserExperience.index_minimum_score) .pluck(:path, :last_comment_at) set_surrogate_controls(date) + @view_template = "posts" end def set_surrogate_controls(date) @@ -51,6 +76,14 @@ class SitemapsController < ApplicationController end end + def valid_resource_sitemap? + %w[posts users tags].include?(resource_string) + end + + def resource_string + params[:sitemap].gsub(".xml","").split("-")[1] + end + def offset params[:sitemap].split("-")[2].to_i * RESULTS_LIMIT # elvaluates to 0 if not present or not a number end diff --git a/app/views/pages/robots.text.erb b/app/views/pages/robots.text.erb index 7a1398b17..6d5856f8a 100644 --- a/app/views/pages/robots.text.erb +++ b/app/views/pages/robots.text.erb @@ -11,4 +11,4 @@ Disallow: /mod/* Disallow: /mod?* Disallow: /admin/* -Sitemap: https://<%= ApplicationConfig["AWS_BUCKET_NAME"] %>.s3.amazonaws.com/sitemaps/sitemap.xml.gz +Sitemap: <%= URL.url("sitemap-index.xml") %> \ No newline at end of file diff --git a/app/views/sitemaps/index.xml.erb b/app/views/sitemaps/index.xml.erb new file mode 100644 index 000000000..537bd699c --- /dev/null +++ b/app/views/sitemaps/index.xml.erb @@ -0,0 +1,20 @@ + + + <% if @articles_count > @page_limit %> + <% (0..(@articles_count/@page_limit)).each do |n| %> + + <%= app_url("sitemap-posts-#{n}.xml") %> + + <% end %> + <% else %> + + <%= app_url("sitemap-posts.xml") %> + + <% end %> + + <%= app_url("sitemap-users.xml") %> + + + <%= app_url("sitemap-tags.xml") %> + + diff --git a/app/views/sitemaps/show.xml.erb b/app/views/sitemaps/posts.xml.erb similarity index 100% rename from app/views/sitemaps/show.xml.erb rename to app/views/sitemaps/posts.xml.erb diff --git a/app/views/sitemaps/tags.xml.erb b/app/views/sitemaps/tags.xml.erb new file mode 100644 index 000000000..a90465ddf --- /dev/null +++ b/app/views/sitemaps/tags.xml.erb @@ -0,0 +1,9 @@ + + + <% @tags.each do |name, updated_at| %> + + <%= app_url("/t/#{name}") %> + <%= updated_at.strftime("%F") %> + + <% end %> + diff --git a/app/views/sitemaps/users.xml.erb b/app/views/sitemaps/users.xml.erb new file mode 100644 index 000000000..76ebdda58 --- /dev/null +++ b/app/views/sitemaps/users.xml.erb @@ -0,0 +1,9 @@ + + + <% @users.each do |username, updated_at| %> + + <%= app_url(username) %> + <%= updated_at.strftime("%F") %> + + <% end %> + diff --git a/app/workers/sitemap_refresh_worker.rb b/app/workers/sitemap_refresh_worker.rb deleted file mode 100644 index 3a182bc97..000000000 --- a/app/workers/sitemap_refresh_worker.rb +++ /dev/null @@ -1,13 +0,0 @@ -Rails.application.load_tasks - -class SitemapRefreshWorker - include Sidekiq::Worker - - sidekiq_options queue: :low_priority, retry: 10 - - def perform - sitemap_task = ForemInstance.local? ? "sitemap:refresh:no_ping" : "sitemap:refresh" - - Rake::Task[sitemap_task].invoke - end -end diff --git a/config/sitemap.rb b/config/sitemap.rb deleted file mode 100644 index 8b354e8c2..000000000 --- a/config/sitemap.rb +++ /dev/null @@ -1,53 +0,0 @@ -require "sitemap_generator/s3_adapter" - -# @forem/systems: It's fine if this doesn't 100% work correctly right now, as long as it doesn't break stuff at runtime. - -if Rails.env.production? - region = ApplicationConfig["AWS_UPLOAD_REGION"].presence || ApplicationConfig["AWS_DEFAULT_REGION"] - s3_config_hash = if ENV["FOREM_CONTEXT"] == "forem_cloud" # @forem/systems jdoss's special sauce. - # Excluding the aws_access_key_id and aws_secret_access_key causes use_iam_profile - # to be set to true by the S3Adapter - # https://github.com/kjvarga/sitemap_generator/blob/0b847f1e7a544ea8ef87bb643a732e30a07a14c9/lib/sitemap_generator/adapters/s3_adapter.rb#L39 - { - fog_provider: "AWS", - fog_directory: ApplicationConfig["AWS_BUCKET_NAME"], - fog_region: "us-east-2", - fog_public: false - } - elsif %w[AWS_ID AWS_SECRET AWS_BUCKET_NAME].all? { |key| ApplicationConfig[key] } - { - fog_provider: "AWS", - aws_access_key_id: ApplicationConfig["AWS_ID"], - aws_secret_access_key: ApplicationConfig["AWS_SECRET"], - fog_directory: ApplicationConfig["AWS_BUCKET_NAME"], - fog_region: region - } - end - - if s3_config_hash - SitemapGenerator::Sitemap.adapter = SitemapGenerator::S3Adapter.new(s3_config_hash) - SitemapGenerator::Sitemap.sitemaps_host = "https://#{ApplicationConfig['AWS_BUCKET_NAME']}.s3.amazonaws.com/" - SitemapGenerator::Sitemap.public_path = "tmp/" - else - SitemapGenerator::Sitemap.adapter = SitemapGenerator::FileAdapter.new - SitemapGenerator::Sitemap.public_path = "public/" - end - SitemapGenerator::Sitemap.sitemaps_path = "sitemaps/" -end - -SitemapGenerator::Sitemap.default_host = "#{ApplicationConfig['APP_PROTOCOL']}#{ApplicationConfig['APP_DOMAIN']}" - -SitemapGenerator::Sitemap.create do - Article.published.where("score > ? OR featured = ?", 12, true) - .limit(38_000).find_each do |article| - add article.path, lastmod: article.last_comment_at, changefreq: "daily" - end - - User.order(comments_count: :desc).where("updated_at > ?", 5.days.ago).limit(8000).find_each do |user| - add "/#{user.username}", changefreq: "daily" - end - - Tag.order(hotness_score: :desc).limit(250).find_each do |tag| - add "/t/#{tag.name}", changefreq: "daily" - end -end diff --git a/spec/requests/pages_spec.rb b/spec/requests/pages_spec.rb index 7811968d3..f4748947d 100644 --- a/spec/requests/pages_spec.rb +++ b/spec/requests/pages_spec.rb @@ -231,7 +231,7 @@ RSpec.describe "Pages", type: :request do it "has proper text" do get "/robots.txt" - text = "Sitemap: https://#{ApplicationConfig['AWS_BUCKET_NAME']}.s3.amazonaws.com/sitemaps/sitemap.xml.gz" + text = "Sitemap: #{URL.url("sitemap-index.xml")}" expect(response.body).to include(text) end end diff --git a/spec/requests/sitemaps_spec.rb b/spec/requests/sitemaps_spec.rb index 8b92ab1a7..e6e4e9920 100644 --- a/spec/requests/sitemaps_spec.rb +++ b/spec/requests/sitemaps_spec.rb @@ -41,38 +41,127 @@ RSpec.describe "Sitemaps", type: :request do expect(response.media_type).to eq("application/xml") end - it "renders most recent posts if /sitemap-posts", :aggregate_failures do - create_list(:article, 8) - get "/sitemap-posts.xml" - expect(response.body).to include(Article.order("published_at DESC").first.path) - expect(response.body).not_to include(Article.order("published_at DESC").last.path) + context "with index in param" do + it "renders basic index", :aggregate_failures do + get "/sitemap-index.xml" + expect(response.body).to include("