diff --git a/app/controllers/sitemaps_controller.rb b/app/controllers/sitemaps_controller.rb index acba7a896..17a5760ca 100644 --- a/app/controllers/sitemaps_controller.rb +++ b/app/controllers/sitemaps_controller.rb @@ -13,7 +13,8 @@ class SitemapsController < ApplicationController end @articles = Article.published - .where("published_at > ? AND published_at < ? AND score > ?", date, date.end_of_month, 3) + .where("published_at > ? AND published_at < ? AND score > ?", + date, date.end_of_month, Settings::UserExperience.index_minimum_score) .pluck(:path, :last_comment_at) set_surrogate_controls(date) diff --git a/app/lib/constants/settings/user_experience.rb b/app/lib/constants/settings/user_experience.rb index f89b8a8b7..15d18b17d 100644 --- a/app/lib/constants/settings/user_experience.rb +++ b/app/lib/constants/settings/user_experience.rb @@ -21,6 +21,10 @@ module Constants description: "Minimum score needed for a post to show up on the unauthenticated home page.", placeholder: "0" }, + index_minimum_score: { + description: "Minimum score needed for a post to be indexed by search engines.", + placeholder: "0" + }, primary_brand_color_hex: { description: "Determines background/border of buttons etc. Must be dark enough to contrast with white text.", placeholder: "#0a0a0a" diff --git a/app/models/article.rb b/app/models/article.rb index 4bea454ed..b1f251b1f 100644 --- a/app/models/article.rb +++ b/app/models/article.rb @@ -490,6 +490,18 @@ class Article < ApplicationRecord followers.uniq.compact end + def skip_indexing? + # should the article be skipped indexed by crawlers? + # true if unpublished, or spammy, + # or low score, not featured, and from a user with no comments + !published || + (score < Settings::UserExperience.index_minimum_score && + user.comments_count < 1 && + !featured) || + featured_number.to_i < 1_500_000_000 || + score < -1 + end + private def search_score diff --git a/app/models/settings/user_experience.rb b/app/models/settings/user_experience.rb index b801821db..0e00748b7 100644 --- a/app/models/settings/user_experience.rb +++ b/app/models/settings/user_experience.rb @@ -11,6 +11,7 @@ module Settings # basic (current default), rich (cover image on all posts), compact (more minimal) setting :feed_style, type: :string, default: "basic" setting :home_feed_minimum_score, type: :integer, default: 0 + setting :index_minimum_score, type: :integer, default: 0 setting :primary_brand_color_hex, type: :string, default: "#3b49df", validates: { format: { with: HEX_COLOR_REGEX, diff --git a/app/views/admin/settings/forms/_user_experience.html.erb b/app/views/admin/settings/forms/_user_experience.html.erb index 3b761cc05..dfb4fe93e 100644 --- a/app/views/admin/settings/forms/_user_experience.html.erb +++ b/app/views/admin/settings/forms/_user_experience.html.erb @@ -43,6 +43,14 @@ value: Settings::UserExperience.home_feed_minimum_score, placeholder: Constants::Settings::UserExperience::DETAILS[:home_feed_minimum_score][:placeholder] %> +
")) ||
- @article.featured_number.to_i < 1500000000 ||
- @article.score < -1 %>
+ <% if @article.skip_indexing? %>
<% end %>
diff --git a/spec/requests/stories_show_spec.rb b/spec/requests/stories_show_spec.rb
index d276a7c50..bfb42d069 100644
--- a/spec/requests/stories_show_spec.rb
+++ b/spec/requests/stories_show_spec.rb
@@ -201,27 +201,13 @@ RSpec.describe "StoriesShow", type: :request do
expect(response.body).to include("noindex")
end
- it "has noindex if article has low score even with " do
- article = create(:article, score: -5)
- article.update_column(:processed_html, "hello")
- get article.path
- expect(response.body).to include("noindex")
- end
-
it "does not have noindex if article has high score" do
article = create(:article, score: 6)
get article.path
expect(response.body).not_to include("noindex")
end
- it "does not have noindex if article intermediate score and " do
- article = create(:article, score: 3)
- article.update_column(:processed_html, "hello")
- get article.path
- expect(response.body).not_to include("noindex")
- end
-
- it "does not have noindex if article w/ intermediate score w/ 1 comment " do
+ it "does not have noindex if article w/ intermediate score w/ 1 comment" do
article = create(:article, score: 3)
article.user.update_column(:comments_count, 1)
get article.path