Remove unneeded checks for code tags when filtering articles from search indexing (#14801)

* Move logic to determine noindex from view to model

This brings attention to the code and allows for faster unit tests.

I expect we'll be modifying this soon to deemphasize the code tag as a
marker of quality.

* Replace "5" constant with home feed minimum score

* Remove check for "code" tags in low quality article check

This made sense for DEV and other devrel communities, but is no longer
necessary for Forems.

* Add UserExperience setting for minimum index score

- add setting, I used `index_minimum_score` to match the
`home_feed_minimum_score`, `minimum_index_score` feels natural but
breaks the pattern.

- use this to determine whether to include article in the sitemap

- use this to determine whether to add noindex meta to article show page

* Remove unneeded tests

Since there's no longer any check for <code> tags in the body, omit
those tests (which passed anyway).

* Add new setting to admin page

app/controllers/sitemaps_controller.rb

@@ -13,7 +13,8 @@ class SitemapsController < ApplicationController
     end
 
     @articles = Article.published
-      .where("published_at > ? AND published_at < ? AND score > ?", date, date.end_of_month, 3)
+      .where("published_at > ? AND published_at < ? AND score > ?",
+             date, date.end_of_month, Settings::UserExperience.index_minimum_score)
       .pluck(:path, :last_comment_at)
 
     set_surrogate_controls(date)

app/lib/constants/settings/user_experience.rb

@@ -21,6 +21,10 @@ module Constants
           description: "Minimum score needed for a post to show up on the unauthenticated home page.",
           placeholder: "0"
         },
+        index_minimum_score: {
+          description: "Minimum score needed for a post to be indexed by search engines.",
+          placeholder: "0"
+        },
         primary_brand_color_hex: {
           description: "Determines background/border of buttons etc. Must be dark enough to contrast with white text.",
           placeholder: "#0a0a0a"

app/models/article.rb

@@ -490,6 +490,18 @@ class Article < ApplicationRecord
     followers.uniq.compact
   end
 
+  def skip_indexing?
+    # should the article be skipped indexed by crawlers?
+    # true if unpublished, or spammy,
+    # or low score, not featured, and from a user with no comments
+    !published ||
+      (score < Settings::UserExperience.index_minimum_score &&
+       user.comments_count < 1 &&
+       !featured) ||
+      featured_number.to_i < 1_500_000_000 ||
+      score < -1
+  end
+
   private
 
   def search_score

app/models/settings/user_experience.rb

@@ -11,6 +11,7 @@ module Settings
     # basic (current default), rich (cover image on all posts), compact (more minimal)
     setting :feed_style, type: :string, default: "basic"
     setting :home_feed_minimum_score, type: :integer, default: 0
+    setting :index_minimum_score, type: :integer, default: 0
     setting :primary_brand_color_hex, type: :string, default: "#3b49df", validates: {
       format: {
         with: HEX_COLOR_REGEX,

app/views/admin/settings/forms/_user_experience.html.erb

@@ -43,6 +43,14 @@
                              value: Settings::UserExperience.home_feed_minimum_score,
                              placeholder: Constants::Settings::UserExperience::DETAILS[:home_feed_minimum_score][:placeholder] %>
         </div>
+        <div class="crayons-field">
+          <%= admin_config_label :index_minimum_score, model: Settings::UserExperience %>
+          <%= admin_config_description Constants::Settings::UserExperience::DETAILS[:index_minimum_score][:description] %>
+          <%= f.number_field :index_minimum_score,
+                             class: "crayons-textfield",
+                             value: Settings::UserExperience.index_minimum_score,
+                             placeholder: Constants::Settings::UserExperience::DETAILS[:index_minimum_score][:placeholder] %>
+        </div>
         <div class="crayons-field">
           <%= admin_config_label :default_font, model: Settings::UserExperience %>
           <%= admin_config_description Constants::Settings::UserExperience::DETAILS[:default_font][:description] %>

app/views/articles/show.html.erb

@@ -51,13 +51,7 @@
       <meta property="og:image" content="<%= article_social_image_url(@article) %>" />
       <meta name="twitter:image:src" content="<%= article_social_image_url(@article) %>">
     <% end %>
-    <% if !@article.published ||
-      (@article.score < 5 &&
-        @article.user.comments_count < 1 &&
-        !@article.featured &&
-        @article.processed_html.exclude?("<code>")) ||
-      @article.featured_number.to_i < 1500000000 ||
-      @article.score < -1 %>
+    <% if @article.skip_indexing? %>
       <meta name="robots" content="noindex">
       <meta name="robots" content="nofollow">
     <% end %>

spec/requests/stories_show_spec.rb

@@ -201,27 +201,13 @@ RSpec.describe "StoriesShow", type: :request do
       expect(response.body).to include("noindex")
     end
 
-    it "has noindex if article has low score even with <code>" do
-      article = create(:article, score: -5)
-      article.update_column(:processed_html, "<code>hello</code>")
-      get article.path
-      expect(response.body).to include("noindex")
-    end
-
     it "does not have noindex if article has high score" do
       article = create(:article, score: 6)
       get article.path
       expect(response.body).not_to include("noindex")
     end
 
-    it "does not have noindex if article intermediate score and <code>" do
-      article = create(:article, score: 3)
-      article.update_column(:processed_html, "<code>hello</code>")
-      get article.path
-      expect(response.body).not_to include("noindex")
-    end
-
-    it "does not have noindex if article w/ intermediate score w/ 1 comment " do
+    it "does not have noindex if article w/ intermediate score w/ 1 comment" do
       article = create(:article, score: 3)
       article.user.update_column(:comments_count, 1)
       get article.path