diff --git a/app/controllers/sitemaps_controller.rb b/app/controllers/sitemaps_controller.rb index acba7a896..17a5760ca 100644 --- a/app/controllers/sitemaps_controller.rb +++ b/app/controllers/sitemaps_controller.rb @@ -13,7 +13,8 @@ class SitemapsController < ApplicationController end @articles = Article.published - .where("published_at > ? AND published_at < ? AND score > ?", date, date.end_of_month, 3) + .where("published_at > ? AND published_at < ? AND score > ?", + date, date.end_of_month, Settings::UserExperience.index_minimum_score) .pluck(:path, :last_comment_at) set_surrogate_controls(date) diff --git a/app/lib/constants/settings/user_experience.rb b/app/lib/constants/settings/user_experience.rb index f89b8a8b7..15d18b17d 100644 --- a/app/lib/constants/settings/user_experience.rb +++ b/app/lib/constants/settings/user_experience.rb @@ -21,6 +21,10 @@ module Constants description: "Minimum score needed for a post to show up on the unauthenticated home page.", placeholder: "0" }, + index_minimum_score: { + description: "Minimum score needed for a post to be indexed by search engines.", + placeholder: "0" + }, primary_brand_color_hex: { description: "Determines background/border of buttons etc. Must be dark enough to contrast with white text.", placeholder: "#0a0a0a" diff --git a/app/models/article.rb b/app/models/article.rb index 4bea454ed..b1f251b1f 100644 --- a/app/models/article.rb +++ b/app/models/article.rb @@ -490,6 +490,18 @@ class Article < ApplicationRecord followers.uniq.compact end + def skip_indexing? + # should the article be skipped indexed by crawlers? + # true if unpublished, or spammy, + # or low score, not featured, and from a user with no comments + !published || + (score < Settings::UserExperience.index_minimum_score && + user.comments_count < 1 && + !featured) || + featured_number.to_i < 1_500_000_000 || + score < -1 + end + private def search_score diff --git a/app/models/settings/user_experience.rb b/app/models/settings/user_experience.rb index b801821db..0e00748b7 100644 --- a/app/models/settings/user_experience.rb +++ b/app/models/settings/user_experience.rb @@ -11,6 +11,7 @@ module Settings # basic (current default), rich (cover image on all posts), compact (more minimal) setting :feed_style, type: :string, default: "basic" setting :home_feed_minimum_score, type: :integer, default: 0 + setting :index_minimum_score, type: :integer, default: 0 setting :primary_brand_color_hex, type: :string, default: "#3b49df", validates: { format: { with: HEX_COLOR_REGEX, diff --git a/app/views/admin/settings/forms/_user_experience.html.erb b/app/views/admin/settings/forms/_user_experience.html.erb index 3b761cc05..dfb4fe93e 100644 --- a/app/views/admin/settings/forms/_user_experience.html.erb +++ b/app/views/admin/settings/forms/_user_experience.html.erb @@ -43,6 +43,14 @@ value: Settings::UserExperience.home_feed_minimum_score, placeholder: Constants::Settings::UserExperience::DETAILS[:home_feed_minimum_score][:placeholder] %> +
+ <%= admin_config_label :index_minimum_score, model: Settings::UserExperience %> + <%= admin_config_description Constants::Settings::UserExperience::DETAILS[:index_minimum_score][:description] %> + <%= f.number_field :index_minimum_score, + class: "crayons-textfield", + value: Settings::UserExperience.index_minimum_score, + placeholder: Constants::Settings::UserExperience::DETAILS[:index_minimum_score][:placeholder] %> +
<%= admin_config_label :default_font, model: Settings::UserExperience %> <%= admin_config_description Constants::Settings::UserExperience::DETAILS[:default_font][:description] %> diff --git a/app/views/articles/show.html.erb b/app/views/articles/show.html.erb index 7b235cf7d..8d404a9dc 100644 --- a/app/views/articles/show.html.erb +++ b/app/views/articles/show.html.erb @@ -51,13 +51,7 @@ <% end %> - <% if !@article.published || - (@article.score < 5 && - @article.user.comments_count < 1 && - !@article.featured && - @article.processed_html.exclude?("")) || - @article.featured_number.to_i < 1500000000 || - @article.score < -1 %> + <% if @article.skip_indexing? %> <% end %> diff --git a/spec/requests/stories_show_spec.rb b/spec/requests/stories_show_spec.rb index d276a7c50..bfb42d069 100644 --- a/spec/requests/stories_show_spec.rb +++ b/spec/requests/stories_show_spec.rb @@ -201,27 +201,13 @@ RSpec.describe "StoriesShow", type: :request do expect(response.body).to include("noindex") end - it "has noindex if article has low score even with " do - article = create(:article, score: -5) - article.update_column(:processed_html, "hello") - get article.path - expect(response.body).to include("noindex") - end - it "does not have noindex if article has high score" do article = create(:article, score: 6) get article.path expect(response.body).not_to include("noindex") end - it "does not have noindex if article intermediate score and " do - article = create(:article, score: 3) - article.update_column(:processed_html, "hello") - get article.path - expect(response.body).not_to include("noindex") - end - - it "does not have noindex if article w/ intermediate score w/ 1 comment " do + it "does not have noindex if article w/ intermediate score w/ 1 comment" do article = create(:article, score: 3) article.user.update_column(:comments_count, 1) get article.path