Fix human language detection (#2338)

* Update language detection

* Modify feed to account for languages

Gemfile

@@ -30,6 +30,7 @@ gem "bourbon", "~> 5.1"
 gem "buffer", "~> 0.1"
 gem "carrierwave", "~> 1.3"
 gem "carrierwave-bombshelter", "~> 0.2"
+gem "cld", "~> 0.8"
 gem "cloudinary", "~> 1.11"
 gem "counter_culture", "~> 2.1"
 gem "csv_shaper", "~> 1.3"

Gemfile.lock

@@ -199,6 +199,8 @@ GEM
     chromedriver-helper (2.1.0)
       archive-zip (~> 0.10)
       nokogiri (~> 1.8)
+    cld (0.8.0)
+      ffi
     cloudinary (1.11.1)
       aws_cf_signer
       rest-client
@@ -1009,6 +1011,7 @@ DEPENDENCIES
   carrierwave (~> 1.3)
   carrierwave-bombshelter (~> 0.2)
   chromedriver-helper (~> 2.1)
+  cld (~> 0.8)
   cloudinary (~> 1.11)
   counter_culture (~> 2.1)
   csv_shaper (~> 1.3)

app/assets/javascripts/initializers/initializeFetchFollowedArticles.js.erb

@@ -35,6 +35,7 @@ function insertNewArticles(user){
       var intersectedTags = intersect_arrays(user.followed_tag_names, article.cached_tag_list_array)
       var followedPoints = 1
       var experienceDifference = Math.abs(article['experience_level_rating'] - user.experience_level || 5)
+      var containsPreferredLanguage = findOne([article.language || 'en'], user.preferred_languages_array || ['en']);
       JSON.parse(user.followed_tags).map(function(tag) {
         if (intersectedTags.includes(tag.name)) {
           followedPoints = followedPoints + tag.points
@@ -47,6 +48,11 @@ function insertNewArticles(user){
       if (containsOrganizationID) {
         articlePoints = articlePoints + 16
       }
+      if (containsPreferredLanguage) {
+        articlePoints = articlePoints + 1
+      } else {
+        articlePoints = articlePoints - 10
+      }
       var rand = Math.random();
       if (rand < 0.3) {
         articlePoints = articlePoints + 1
@@ -80,6 +86,7 @@ function insertTopArticles(user){
       var intersectedTags = intersect_arrays(user.followed_tag_names, article.cached_tag_list_array)
       var followedPoints = 1
       var experienceDifference = Math.abs(article['experience_level_rating'] - user.experience_level || 5)
+      var containsPreferredLanguage = findOne([article.language || 'en'], user.preferred_languages_array || ['en'])
       JSON.parse(user.followed_tags).map(function(tag) {
         if (intersectedTags.includes(tag.name)) {
           followedPoints = followedPoints + tag.points
@@ -98,6 +105,11 @@ function insertTopArticles(user){
       } else if (rand < 0.6) {
         articlePoints = articlePoints + 3
       }
+      if (containsPreferredLanguage) {
+        articlePoints = articlePoints + 1
+      } else {
+        articlePoints = articlePoints - 10
+      }
       articlePoints = articlePoints - (experienceDifference/2);
       article['points'] = articlePoints
     });
@@ -115,6 +127,7 @@ function insertTopArticles(user){
 
 function algoliaFollowedArticles(){
     var user = userData();
+
     if (user && user.followed_tag_names && user.followed_tag_names.length > 0) {
       insertInitialArticles(user)
       var followedUsersArray = [];

app/controllers/async_info_controller.rb

@@ -48,6 +48,7 @@ class AsyncInfoController < ApplicationController
         display_sponsors: @user.display_sponsors,
         trusted: @user.trusted,
         experience_level: @user.experience_level,
+        preferred_languages_array: @user.preferred_languages_array,
         config_body_class: @user.config_body_class
       }
     end

app/controllers/stories_controller.rb

@@ -104,7 +104,7 @@ class StoriesController < ApplicationController
   def handle_base_index
     @home_page = true
     @page = (params[:page] || 1).to_i
-    num_articles = 30
+    num_articles = 35
     @stories = article_finder(num_articles)
     add_param_context(:page, :timeframe)
     if %w[week month year infinity].include?(params[:timeframe])

app/decorators/user_decorator.rb

@@ -83,4 +83,12 @@ class UserDecorator < ApplicationDecorator
     ]
     colors[id % 10]
   end
+
+  def preferred_languages_array
+    languages = []
+    language_settings.keys.each do |setting|
+      languages << setting.split("prefer_language_")[1] if language_settings[setting] && setting.include?("prefer_language_")
+    end
+    languages
+  end
 end

app/labor/language_detector.rb

@@ -5,26 +5,17 @@ class LanguageDetector
 
   def detect
     response = get_language
-    response["result"] if response["confidence"] > 0.8
+    response[:code] if response[:reliable]
   rescue StandardError
     nil
   end
 
   def get_language
-    return { "result" => "en", "confidence" => 0.9 } unless Rails.env.production?
-
-    client = Algorithmia.client(ApplicationConfig["ALGORITHMIA_KEY"])
-    algo = client.algo("miguelher/LanguageDetector/0.1.0")
-    algo.pipe(text).result
+    CLD.detect_language(text)
   end
 
   def text
-    @article.title + "\n" +
-      non_default_description.to_s +
+    @article.title + ". " +
       FrontMatterParser::Parser.new(:md).call(@article.body_markdown).content.split("`")[0]
   end
-
-  def non_default_description
-    @article.description + "\n" unless @article.description.include? "From the DEV community"
-  end
 end

app/models/article.rb

@@ -80,7 +80,7 @@ class Article < ApplicationRecord
     :comments_count, :positive_reactions_count, :cached_tag_list,
     :main_image, :main_image_background_hex_color, :updated_at, :slug,
     :video, :user_id, :organization_id, :video_source_url, :video_code,
-    :video_thumbnail_url, :video_closed_caption_track_url,
+    :video_thumbnail_url, :video_closed_caption_track_url, :language,
     :experience_level_rating, :experience_level_rating_distribution,
     :published_at, :crossposted_at, :boost_states, :description, :reading_time, :video_duration_in_seconds)
   }
@@ -142,7 +142,7 @@ class Article < ApplicationRecord
                   id: :index_id,
                   per_environment: true,
                   enqueue: :trigger_delayed_index do
-      attributes :title, :path, :class_name, :comments_count, :reading_time,
+      attributes :title, :path, :class_name, :comments_count, :reading_time, :language,
         :tag_list, :positive_reactions_count, :id, :hotness_score, :score, :readable_publish_date, :flare_tag, :user_id,
         :organization_id, :cloudinary_video_url, :video_duration_in_minutes, :experience_level_rating, :experience_level_rating_distribution
       attribute :published_at_int do

app/models/user.rb

@@ -189,6 +189,7 @@ class User < ApplicationRecord
     prefer_language_es Boolean, default: false
     prefer_language_fr Boolean, default: false
     prefer_language_it Boolean, default: false
+    prefer_language_pt Boolean, default: false
   end
 
   def self.trigger_delayed_index(record, remove)

app/policies/user_policy.rb

@@ -106,6 +106,7 @@ class UserPolicy < ApplicationPolicy
        prefer_language_es
        prefer_language_fr
        prefer_language_it
+       prefer_language_pt
        prefer_language_ja
        profile_image
        seeking_mentorship

app/views/stories/_main_stories_feed.html.erb

@@ -10,7 +10,7 @@
     <div id="new-articles-object" data-articles="
       <%= @new_stories.to_json(
             only: %i[title path id user_id comments_count positive_reactions_count organization_id
-                     reading_time video_thumbnail_url video video_duration_in_minutes
+                     reading_time video_thumbnail_url video video_duration_in_minutes language
                      experience_level_rating experience_level_rating_distribution],
             methods: %i[readable_publish_date cached_tag_list_array flare_tag class_name
                         cloudinary_video_url video_duration_in_minutes published_at_int],
@@ -23,7 +23,7 @@
     <div id="home-articles-object" data-articles="
       <%= @stories.to_json(
             only: %i[title path id user_id comments_count positive_reactions_count organization_id
-                     reading_time video_thumbnail_url video video_duration_in_minutes
+                     reading_time video_thumbnail_url video video_duration_in_minutes language
                      experience_level_rating experience_level_rating_distribution],
             methods: %i[readable_publish_date cached_tag_list_array flare_tag class_name
                         cloudinary_video_url video_duration_in_minutes published_at_int],

app/views/users/_misc.html.erb

@@ -39,12 +39,12 @@
 <%= form_for(@user) do |f| %>
   <div class="sub-field">
     <%= f.label :config_theme, "Site Theme" %>
-    <%= f.select(:config_theme, options_for_select(["default", "night theme"], @user.config_theme.gsub("_", " "))) %>
+    <%= f.select(:config_theme, options_for_select(["default", "night theme"], @user.config_theme.tr("_", " "))) %>
     <sub><em>Default is light style. Night theme is in public alpha.</em></sub>
   </div>
   <div class="sub-field">
     <%= f.label :config_font, "Base Post Font" %>
-    <%= f.select(:config_font, options_for_select(["default", "sans serif", "comic sans"], @user.config_font.gsub("_", " "))) %>
+    <%= f.select(:config_font, options_for_select(["default", "sans serif", "comic sans"], @user.config_font.tr("_", " "))) %>
     <sub><em>Default is serif style.</em></sub>
   </div>
   <div class="field">
@@ -81,6 +81,10 @@
       <%= f.check_box :prefer_language_it %>
       <%= f.label :prefer_language_it, "Italian" %>
     </div>
+    <div class="sub-field">
+      <%= f.check_box :prefer_language_pt %>
+      <%= f.label :prefer_language_pt, "Portugese" %>
+    </div>
   </div>
   <div class="field">
     <label></label>

spec/labor/language_detector_spec.rb

@@ -3,8 +3,22 @@ require "rails_helper"
 RSpec.describe LanguageDetector do
   let(:user) { create(:user) }
   let(:article) { create(:article, user_id: user.id) }
+  let(:article_1) { create(:article, user_id: user.id) }
+  let(:article_2) { create(:article, user_id: user.id) }
 
-  it "returns language" do
+  it "detects english" do
+    article.update_column(:body_markdown, "This is definitely english.")
+    article.update_column(:title, "I love the english language.")
     expect(described_class.new(article).detect).to eq("en")
   end
+  it "detects french" do
+    article_1.update_column(:body_markdown, "C'est vraiment francais, bien oui?")
+    article_1.update_column(:title, "C'est vraiment francais, bien oui?")
+    expect(described_class.new(article_1).detect).to eq("fr")
+  end
+  it "detects nil if non-sensicle" do
+    article_2.update_column(:body_markdown, "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec pharetra sapien orci, sit amet auctor nunc tempor quis.")
+    article_2.update_column(:title, "Mauris commodo felis et lacus volutpat fermentum.")
+    expect(described_class.new(article_2).detect).to eq(nil)
+  end
 end

spec/models/user_spec.rb

@@ -380,6 +380,12 @@ RSpec.describe User, type: :model do
       new_user = user_from_authorization_service(:twitter, nil, "navbar_basic")
       new_user.estimate_default_language_without_delay!
     end
+
+    it "returns proper preferred_languages_array" do
+      user.email = "ben@hello.jp"
+      user.estimate_default_language_without_delay!
+      expect(user.decorate.preferred_languages_array).to include("ja")
+    end
   end
 
   it "follows users" do