diff --git a/Gemfile b/Gemfile
index adefbbcb2..1032a7e99 100644
--- a/Gemfile
+++ b/Gemfile
@@ -100,7 +100,6 @@ gem "s3_direct_upload", "~> 0.1" # Direct Upload to Amazon S3
gem "sidekiq", "~> 6.2.2" # Sidekiq is used to process background jobs with the help of Redis
gem "sidekiq-cron", "~> 1.1" # Allows execution of scheduled cron jobs as specific times
gem "sidekiq-unique-jobs", "~> 7.0.12" # Ensures that Sidekiq jobs are unique when enqueued
-gem "sitemap_generator", "~> 6.1" # SitemapGenerator is a framework-agnostic XML Sitemap generator
gem "slack-notifier", "~> 2.4" # A slim ruby wrapper for posting to slack webhooks
gem "sprockets", "~> 4.0" # Sprockets is a Rack-based asset packaging system
gem "staccato", "~> 0.5" # Ruby Google Analytics Measurement
diff --git a/Gemfile.lock b/Gemfile.lock
index 547275f74..b00b039fe 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -759,8 +759,6 @@ GEM
simplecov_json_formatter (~> 0.1)
simplecov-html (0.12.3)
simplecov_json_formatter (0.1.2)
- sitemap_generator (6.1.2)
- builder (~> 3.0)
slack-notifier (2.4.0)
smart_properties (1.15.0)
spring (3.0.0)
@@ -996,7 +994,6 @@ DEPENDENCIES
sidekiq-cron (~> 1.1)
sidekiq-unique-jobs (~> 7.0.12)
simplecov (~> 0.21.2)
- sitemap_generator (~> 6.1)
slack-notifier (~> 2.4)
spring (~> 3.0)
spring-commands-rspec (~> 1.0)
diff --git a/app/controllers/sitemaps_controller.rb b/app/controllers/sitemaps_controller.rb
index 71cdcb615..e9f296b4a 100644
--- a/app/controllers/sitemaps_controller.rb
+++ b/app/controllers/sitemaps_controller.rb
@@ -5,22 +5,46 @@ class SitemapsController < ApplicationController
RESULTS_LIMIT = Rails.env.production? ? 10_000 : 5
def show
- if params[:sitemap].start_with? "sitemap-posts"
- posts_sitemap
+ if params[:sitemap].start_with? "sitemap-index"
+ sitemap_index
+ elsif valid_resource_sitemap?
+ resource_sitemap(resource_string)
else
monthly_sitemap
end
set_cache_control_headers(@max_age,
stale_while_revalidate: @stale_while_revalidate,
stale_if_error: @stale_if_error)
- render layout: false
+ render @view_template, layout: false
end
private
- def posts_sitemap
- @articles = Article.published.order("published_at DESC").limit(RESULTS_LIMIT).offset(offset).pluck(:path, :last_comment_at)
- set_surrogate_controls(Time.now)
+ def sitemap_index
+ set_surrogate_controls(Time.current)
+ @articles_count = Article.published
+ .where("score >= ?", Settings::UserExperience.index_minimum_score).size
+ @page_limit = RESULTS_LIMIT
+ @view_template = "index"
+ end
+
+ def resource_sitemap(resource)
+ case resource
+ when "users"
+ @users = User.order("comments_count DESC")
+ .where("score > -1") # Spam mitigation
+ .limit(RESULTS_LIMIT).offset(offset).pluck(:username, :profile_updated_at)
+ when "posts"
+ @articles = Article.published.order("published_at DESC")
+ .where("score >= ?", Settings::UserExperience.index_minimum_score)
+ .limit(RESULTS_LIMIT).offset(offset).pluck(:path, :last_comment_at)
+ when "tags" # tags
+ @tags = Tag.order("hotness_score DESC")
+ .where(supported: true)
+ .limit(RESULTS_LIMIT).offset(offset).pluck(:name, :updated_at)
+ end
+ set_surrogate_controls(Time.current)
+ @view_template = resource
end
def monthly_sitemap
@@ -33,11 +57,12 @@ class SitemapsController < ApplicationController
end
@articles = Article.published
- .where("published_at > ? AND published_at < ? AND score > ?",
+ .where("published_at > ? AND published_at < ? AND score >= ?",
date, date.end_of_month, Settings::UserExperience.index_minimum_score)
.pluck(:path, :last_comment_at)
set_surrogate_controls(date)
+ @view_template = "posts"
end
def set_surrogate_controls(date)
@@ -51,6 +76,14 @@ class SitemapsController < ApplicationController
end
end
+ def valid_resource_sitemap?
+ %w[posts users tags].include?(resource_string)
+ end
+
+ def resource_string
+ params[:sitemap].gsub(".xml","").split("-")[1]
+ end
+
def offset
params[:sitemap].split("-")[2].to_i * RESULTS_LIMIT # elvaluates to 0 if not present or not a number
end
diff --git a/app/views/pages/robots.text.erb b/app/views/pages/robots.text.erb
index 7a1398b17..6d5856f8a 100644
--- a/app/views/pages/robots.text.erb
+++ b/app/views/pages/robots.text.erb
@@ -11,4 +11,4 @@ Disallow: /mod/*
Disallow: /mod?*
Disallow: /admin/*
-Sitemap: https://<%= ApplicationConfig["AWS_BUCKET_NAME"] %>.s3.amazonaws.com/sitemaps/sitemap.xml.gz
+Sitemap: <%= URL.url("sitemap-index.xml") %>
\ No newline at end of file
diff --git a/app/views/sitemaps/index.xml.erb b/app/views/sitemaps/index.xml.erb
new file mode 100644
index 000000000..537bd699c
--- /dev/null
+++ b/app/views/sitemaps/index.xml.erb
@@ -0,0 +1,20 @@
+
+
+ <% if @articles_count > @page_limit %>
+ <% (0..(@articles_count/@page_limit)).each do |n| %>
+
+ <%= app_url("sitemap-posts-#{n}.xml") %>
+
+ <% end %>
+ <% else %>
+
+ <%= app_url("sitemap-posts.xml") %>
+
+ <% end %>
+
+ <%= app_url("sitemap-users.xml") %>
+
+
+ <%= app_url("sitemap-tags.xml") %>
+
+
diff --git a/app/views/sitemaps/show.xml.erb b/app/views/sitemaps/posts.xml.erb
similarity index 100%
rename from app/views/sitemaps/show.xml.erb
rename to app/views/sitemaps/posts.xml.erb
diff --git a/app/views/sitemaps/tags.xml.erb b/app/views/sitemaps/tags.xml.erb
new file mode 100644
index 000000000..a90465ddf
--- /dev/null
+++ b/app/views/sitemaps/tags.xml.erb
@@ -0,0 +1,9 @@
+
+
+ <% @tags.each do |name, updated_at| %>
+
+ <%= app_url("/t/#{name}") %>
+ <%= updated_at.strftime("%F") %>
+
+ <% end %>
+
diff --git a/app/views/sitemaps/users.xml.erb b/app/views/sitemaps/users.xml.erb
new file mode 100644
index 000000000..76ebdda58
--- /dev/null
+++ b/app/views/sitemaps/users.xml.erb
@@ -0,0 +1,9 @@
+
+
+ <% @users.each do |username, updated_at| %>
+
+ <%= app_url(username) %>
+ <%= updated_at.strftime("%F") %>
+
+ <% end %>
+
diff --git a/app/workers/sitemap_refresh_worker.rb b/app/workers/sitemap_refresh_worker.rb
deleted file mode 100644
index 3a182bc97..000000000
--- a/app/workers/sitemap_refresh_worker.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-Rails.application.load_tasks
-
-class SitemapRefreshWorker
- include Sidekiq::Worker
-
- sidekiq_options queue: :low_priority, retry: 10
-
- def perform
- sitemap_task = ForemInstance.local? ? "sitemap:refresh:no_ping" : "sitemap:refresh"
-
- Rake::Task[sitemap_task].invoke
- end
-end
diff --git a/config/sitemap.rb b/config/sitemap.rb
deleted file mode 100644
index 8b354e8c2..000000000
--- a/config/sitemap.rb
+++ /dev/null
@@ -1,53 +0,0 @@
-require "sitemap_generator/s3_adapter"
-
-# @forem/systems: It's fine if this doesn't 100% work correctly right now, as long as it doesn't break stuff at runtime.
-
-if Rails.env.production?
- region = ApplicationConfig["AWS_UPLOAD_REGION"].presence || ApplicationConfig["AWS_DEFAULT_REGION"]
- s3_config_hash = if ENV["FOREM_CONTEXT"] == "forem_cloud" # @forem/systems jdoss's special sauce.
- # Excluding the aws_access_key_id and aws_secret_access_key causes use_iam_profile
- # to be set to true by the S3Adapter
- # https://github.com/kjvarga/sitemap_generator/blob/0b847f1e7a544ea8ef87bb643a732e30a07a14c9/lib/sitemap_generator/adapters/s3_adapter.rb#L39
- {
- fog_provider: "AWS",
- fog_directory: ApplicationConfig["AWS_BUCKET_NAME"],
- fog_region: "us-east-2",
- fog_public: false
- }
- elsif %w[AWS_ID AWS_SECRET AWS_BUCKET_NAME].all? { |key| ApplicationConfig[key] }
- {
- fog_provider: "AWS",
- aws_access_key_id: ApplicationConfig["AWS_ID"],
- aws_secret_access_key: ApplicationConfig["AWS_SECRET"],
- fog_directory: ApplicationConfig["AWS_BUCKET_NAME"],
- fog_region: region
- }
- end
-
- if s3_config_hash
- SitemapGenerator::Sitemap.adapter = SitemapGenerator::S3Adapter.new(s3_config_hash)
- SitemapGenerator::Sitemap.sitemaps_host = "https://#{ApplicationConfig['AWS_BUCKET_NAME']}.s3.amazonaws.com/"
- SitemapGenerator::Sitemap.public_path = "tmp/"
- else
- SitemapGenerator::Sitemap.adapter = SitemapGenerator::FileAdapter.new
- SitemapGenerator::Sitemap.public_path = "public/"
- end
- SitemapGenerator::Sitemap.sitemaps_path = "sitemaps/"
-end
-
-SitemapGenerator::Sitemap.default_host = "#{ApplicationConfig['APP_PROTOCOL']}#{ApplicationConfig['APP_DOMAIN']}"
-
-SitemapGenerator::Sitemap.create do
- Article.published.where("score > ? OR featured = ?", 12, true)
- .limit(38_000).find_each do |article|
- add article.path, lastmod: article.last_comment_at, changefreq: "daily"
- end
-
- User.order(comments_count: :desc).where("updated_at > ?", 5.days.ago).limit(8000).find_each do |user|
- add "/#{user.username}", changefreq: "daily"
- end
-
- Tag.order(hotness_score: :desc).limit(250).find_each do |tag|
- add "/t/#{tag.name}", changefreq: "daily"
- end
-end
diff --git a/spec/requests/pages_spec.rb b/spec/requests/pages_spec.rb
index 7811968d3..f4748947d 100644
--- a/spec/requests/pages_spec.rb
+++ b/spec/requests/pages_spec.rb
@@ -231,7 +231,7 @@ RSpec.describe "Pages", type: :request do
it "has proper text" do
get "/robots.txt"
- text = "Sitemap: https://#{ApplicationConfig['AWS_BUCKET_NAME']}.s3.amazonaws.com/sitemaps/sitemap.xml.gz"
+ text = "Sitemap: #{URL.url("sitemap-index.xml")}"
expect(response.body).to include(text)
end
end
diff --git a/spec/requests/sitemaps_spec.rb b/spec/requests/sitemaps_spec.rb
index 8b92ab1a7..e6e4e9920 100644
--- a/spec/requests/sitemaps_spec.rb
+++ b/spec/requests/sitemaps_spec.rb
@@ -41,38 +41,127 @@ RSpec.describe "Sitemaps", type: :request do
expect(response.media_type).to eq("application/xml")
end
- it "renders most recent posts if /sitemap-posts", :aggregate_failures do
- create_list(:article, 8)
- get "/sitemap-posts.xml"
- expect(response.body).to include(Article.order("published_at DESC").first.path)
- expect(response.body).not_to include(Article.order("published_at DESC").last.path)
+ context "with index in param" do
+ it "renders basic index", :aggregate_failures do
+ get "/sitemap-index.xml"
+ expect(response.body).to include("