[deploy] Remove AWS SDK and private endpoints completely (#9978)

* Initial work to remove aws_sdk

* More work

* Add tests

* Typos and cleanup

* Fixed typo and misc

* Fix test typos

* Update spec

* Remove functioncaller code

* Add comment to bump tests :)

* Fix spaminess calc

.env_sample

@@ -51,12 +51,6 @@ export AWS_SECRET="Optional"
 export AWS_BUCKET_NAME="Optional"
 export AWS_UPLOAD_REGION=""
 
-# AWS SDK
-# (https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html)
-export AWS_DEFAULT_REGION="us-east-1"
-export AWS_SDK_KEY="Optional"
-export AWS_SDK_SECRET="Optional"
-
 # AWS for video storage
 export AWS_S3_INPUT_BUCKET="Optional"
 export AWS_S3_VIDEO_ID="Optional"

Gemfile

@@ -19,7 +19,6 @@ gem "ahoy_email", "~> 1.1" # Email analytics for Rails
 gem "ahoy_matey", "~> 3.0" # Tracking analytics for Rails
 gem "ancestry", "~> 3.1" # Ancestry allows the records of a ActiveRecord model to be organized in a tree structure
 gem "autoprefixer-rails", "~> 9.8" # Parse CSS and add vendor prefixes to CSS rules using values from the Can I Use website
-gem "aws-sdk-lambda", "~> 1.48" # Official AWS Ruby gem for AWS Lambda
 gem "blazer", "~> 2.2.6" # Allows admins to query data
 gem "bootsnap", ">= 1.1.0", require: false # Boot large ruby/rails apps faster
 gem "buffer", "~> 0.1" # Buffer is a Ruby Wrapper for the Buffer API

Gemfile.lock

@@ -104,15 +104,6 @@ GEM
     autoprefixer-rails (9.8.6.2)
       execjs
     aws-eventstream (1.1.0)
-    aws-partitions (1.356.0)
-    aws-sdk-core (3.104.3)
-      aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
-      aws-sigv4 (~> 1.1)
-      jmespath (~> 1.0)
-    aws-sdk-lambda (1.48.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
-      aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.2)
       aws-eventstream (~> 1, >= 1.0.2)
     aws_cf_signer (0.1.3)
@@ -420,7 +411,6 @@ GEM
     ipaddress (0.8.3)
     jbuilder (2.10.0)
       activesupport (>= 5.0.0)
-    jmespath (1.4.0)
     jquery-fileupload-rails (0.4.7)
       actionpack (>= 3.1)
       railties (>= 3.1)
@@ -872,7 +862,6 @@ DEPENDENCIES
   ancestry (~> 3.1)
   approvals (~> 0.0)
   autoprefixer-rails (~> 9.8)
-  aws-sdk-lambda (~> 1.48)
   better_errors (~> 2.7)
   binding_of_caller (~> 0.8)
   blazer (~> 2.2.6)

app/black_box/black_box.rb

@@ -1,6 +1,7 @@
 class BlackBox
+  OUR_EPOCH_NUMBER = "2010-01-01 00:00:01".to_time.to_i # Arbitrary date, but the one we went with.
   class << self
-    def article_hotness_score(article, function_caller = FunctionCaller)
+    def article_hotness_score(article)
       usable_date = article.crossposted_at || article.published_at
       reaction_points = article.score
       super_super_recent_bonus = usable_date > 1.hour.ago ? 28 : 0
@@ -16,7 +17,7 @@ class BlackBox
         reaction_points = (reaction_points * 0.8).to_i # watercooler posts shouldn't get as much love in feed
       end
 
-      article_hotness = last_mile_hotness_calc(article, function_caller)
+      article_hotness = last_mile_hotness_calc(article)
 
       (
         article_hotness + reaction_points + recency_bonus + super_recent_bonus +
@@ -32,14 +33,15 @@ class BlackBox
       (rep_points + descendants_points + bonus_points - spaminess_rating).to_i
     end
 
-    def calculate_spaminess(story, function_caller = FunctionCaller)
-      # accepts comment or article as story
-      return 100 unless story.user
-      return 0 if ENV["AWS_SDK_KEY"].blank? # Skip this if we don't have a private spam score for now
-      return 0 if ENV["AWS_SDK_KEY"] == "foobarbaz" # Also skip if placeholder
+    def calculate_spaminess(story)
+      user = story.user
+      return 100 unless user
+      return 0 if user.trusted
+      return 0 if user.badge_achievements_count.positive?
 
-      function_caller.call("blackbox-production-spamScore",
-                           { story: story, user: story.user }.to_json).to_i
+      base_spaminess = 0
+      base_spaminess += 25 if social_auth_registration_recent?(user) && user.registered_at > 25.days.ago
+      base_spaminess
     end
 
     private
@@ -50,20 +52,18 @@ class BlackBox
       size_bonus + code_bonus
     end
 
-    def last_mile_hotness_calc(article, function_caller)
-      if ENV["AWS_SDK_KEY"].present? && ENV["AWS_SDK_KEY"] != "foobarbaz"
-        function_caller.call(
-          "blackbox-production-articleHotness",
-          { article: article, user: article.user }.to_json,
-        ).to_i
-      else
-        # Simple calculation that takes in published at time and scores
-        # Same order of magnitude calculation as existing private function
-        # Gives credit to new articles and articles which score well from users and mods
-        article.published_at.to_i / 10_000 +
-          (article.score * 5000) +
-          (article.comment_score * 5000)
-      end
+    def last_mile_hotness_calc(article)
+      score_from_epoch = article.featured_number.to_i - OUR_EPOCH_NUMBER # Approximate time of publish - epoch time
+      score_from_epoch / 1000 +
+        (article.score * 50) +
+        (article.comment_score * 50) -
+        (article.spaminess_rating * 5)
+    end
+
+    def social_auth_registration_recent?(user)
+      # was the social auth account created very recently?
+      social_auth_date_plus_two_days = ((user.github_created_at || user.twitter_created_at || 3.days.ago) + 2.days)
+      user.registered_at < social_auth_date_plus_two_days
     end
   end
 end

app/labor/coupon_generator.rb

@@ -1,20 +0,0 @@
-class CouponGenerator
-  attr_accessor :id, :version
-
-  def initialize(id, version)
-    @id = id
-    @version = version
-  end
-
-  def generate
-    "#{version}_#{lambda_generated_code}"
-  end
-
-  private
-
-  def lambda_generated_code
-    response = FunctionCaller.new("blackbox-production-couponCode",
-                                  { inputNumber: id, version: version }.to_json).call
-    response.to_s(36)
-  end
-end

app/labor/function_caller.rb

@@ -1,22 +0,0 @@
-class FunctionCaller
-  def initialize(function_name, payload, aws_lambda_client = AWS_LAMBDA)
-    @function_name = function_name
-    @payload = payload
-    @aws_lambda_client = aws_lambda_client
-  end
-
-  def self.call(*args)
-    new(*args).call
-  end
-
-  def call
-    response = aws_lambda_client.invoke(function_name: function_name, payload: payload)
-    payload_json = response.payload.as_json[0]
-    body = payload_json ? JSON.parse(payload_json)["body"] : nil
-    body ? JSON.parse(body)["message"] : nil
-  end
-
-  private
-
-  attr_reader :function_name, :payload, :aws_lambda_client
-end

app/views/admin/users/show.html.erb

@@ -31,16 +31,6 @@
       <% else %>
         <dd>N/A</dd>
       <% end %>
-      <dt>Sustaining Member:</dt>
-      <dd><%= @user.a_sustaining_member? %></dd>
-      <% if @user.a_sustaining_member? %>
-        <dt>Monthly Dues:</dt>
-        <dd><%= @user.monthly_dues %></dd>
-        <dt>25% Off Coupon:</dt>
-        <dd><%= CouponGenerator.new(current_user.id, "member_discount").generate %> || "N/A" </dd>
-        <dt>1x Gift:</dt>
-        <dd><%= CouponGenerator.new(current_user.id, "tee_pack").generate %> || "N/A" </dd>
-      <% end %>
     </dl>
   </div>
   </p>

config/initializers/aws_sdk.rb

@@ -1,18 +0,0 @@
-Rails.application.reloader.to_prepare do
-  AWS_LAMBDA = if Rails.env.production? && ApplicationConfig["AWS_SDK_KEY"].present?
-                 Aws::Lambda::Client.new(
-                   region: ApplicationConfig["AWS_DEFAULT_REGION"],
-                   access_key_id: ApplicationConfig["AWS_SDK_KEY"],
-                   secret_access_key: ApplicationConfig["AWS_SDK_SECRET"],
-                 )
-               else
-                 # Fake Aws::Lambda::Client
-                 Class.new do
-                   def invoke(*)
-                     # rubocop:disable Performance/OpenStruct
-                     OpenStruct.new(payload: [{ body: { message: 0 }.to_json }.to_json])
-                     # rubocop:enable Performance/OpenStruct
-                   end
-                 end.new
-               end
-end

spec/black_box/black_box_spec.rb

@@ -1,51 +1,23 @@
 require "rails_helper"
 
 RSpec.describe BlackBox, type: :black_box do
-  let!(:function_caller) { double }
-
   describe "#article_hotness_score" do
     let!(:article) { build_stubbed(:article, published_at: Time.current) }
 
-    it "calls function caller if AWS_SDK_KEY present" do
-      ENV["AWS_SDK_KEY"] = "valid_key"
-      allow(function_caller).to receive(:call).and_return(5)
-      described_class.article_hotness_score(article, function_caller)
-      expect(function_caller).to have_received(:call).once
-      ENV["AWS_SDK_KEY"] = nil
-    end
-
-    it "does not call function caller if AWS_SDK_KEY is placeholder" do
-      ENV["AWS_SDK_KEY"] = "foobarbaz"
-      allow(function_caller).to receive(:call).and_return(5)
-      described_class.article_hotness_score(article, function_caller)
-      expect(function_caller).not_to have_received(:call)
-      ENV["AWS_SDK_KEY"] = nil
-    end
-
-    it "doesn't fail when function caller returns nil" do
-      allow(function_caller).to receive(:call).and_return(nil)
-      described_class.article_hotness_score(article, function_caller)
-    end
-
-    xit "returns the correct value" do
+    it "returns higher value for higher score" do
       article = build_stubbed(:article, score: 99, published_at: Time.current)
-      allow(function_caller).to receive(:call).and_return(5)
-      # recent bonuses (28 + 31 + 80 + 395 + 330 + 330 = 1194)
-      # + score (99)
-      # + value from the function caller (5)
-      score = described_class.article_hotness_score(article, function_caller)
-      expect(score).to eq(657_758)
+      lower_article = build_stubbed(:article, score: 70, published_at: Time.current)
+      score = described_class.article_hotness_score(article)
+      lower_score = described_class.article_hotness_score(lower_article)
+      expect(score).to be > lower_score
     end
 
-    xit "returns the lower correct value if article tagged with watercooler" do
-      article = build_stubbed(:article, score: 99, cached_tag_list: "hello, discuss, watercooler",
-                                        published_at: Time.current)
-      allow(function_caller).to receive(:call).and_return(5)
-      # recent bonuses (28 + 31 + 80 + 395 + 330 + 330 = 1194)
-      # + score (99)
-      # + value from the function caller (5)
-      score = described_class.article_hotness_score(article, function_caller)
-      expect(score).to be < 657_758 # lower because watercooler tag
+    it "returns higher value for more recent article" do
+      article = build_stubbed(:article, score: 99, published_at: Time.current)
+      lower_article = build_stubbed(:article, score: 70, published_at: 5.days.ago)
+      score = described_class.article_hotness_score(article)
+      lower_score = described_class.article_hotness_score(lower_article)
+      expect(score).to be > lower_score
     end
   end
 
@@ -55,12 +27,6 @@ RSpec.describe BlackBox, type: :black_box do
       reactions = double
       allow(comment).to receive(:reactions).and_return(reactions)
       allow(reactions).to receive(:sum).with(:points).and_return(22)
-      # rep_points + descendants_points + bonus_points - spaminess_rating
-      # rep_points - 22
-      # descendants_points - 0
-      # bonus_points - 2 + 1 = 3
-      # spaminess_rating - 0
-      # 22 + 0 + 3 - 0 = 25
       expect(described_class.comment_quality_score(comment)).to eq(25)
     end
   end
@@ -69,44 +35,43 @@ RSpec.describe BlackBox, type: :black_box do
     let(:user) { build_stubbed(:user) }
     let(:comment) { build_stubbed(:comment, user: user) }
 
-    before do
-      allow(function_caller).to receive(:call).and_return(1)
-    end
-
     it "returns 100 if there is no user" do
       story = instance_double("Comment", user: nil)
-      expect(described_class.calculate_spaminess(story, function_caller)).to eq(100)
-      expect(function_caller).not_to have_received(:call)
+      expect(described_class.calculate_spaminess(story)).to eq(100)
     end
 
-    it "calls the function_caller if AWS_SDK_KEY present" do
-      ENV["AWS_SDK_KEY"] = "valid_key"
-      described_class.calculate_spaminess(comment, function_caller)
-      expect(function_caller).to have_received(:call).with("blackbox-production-spamScore",
-                                                           { story: comment, user: user }.to_json).once
-      ENV["AWS_SDK_KEY"] = nil
+    it "returns 25 if there is a recent github_created_at" do
+      user = create(:user)
+      user.update_column(:github_created_at, 1.day.ago)
+      user.update_column(:registered_at, 1.hour.ago)
+      article = create(:article, score: 99, published_at: Time.current, user: user)
+      expect(described_class.calculate_spaminess(article)).to eq(25)
     end
 
-    it "does not call the function_caller if AWS_SDK_KEY is placeholder" do
-      ENV["AWS_SDK_KEY"] = "foobarbaz"
-      described_class.calculate_spaminess(comment, function_caller)
-      expect(function_caller).not_to have_received(:call).with("blackbox-production-spamScore",
-                                                               { story: comment, user: user }.to_json)
-      ENV["AWS_SDK_KEY"] = nil
+    it "returns 0 if there is non-recent github_created_at " do
+      user = create(:user)
+      user.update_column(:github_created_at, 10.days.ago)
+      user.update_column(:registered_at, 1.hour.ago)
+      article = create(:article, score: 99, published_at: Time.current)
+      expect(described_class.calculate_spaminess(article)).to eq(0)
     end
 
-    it "returns the function_caller spaminess if AWS_SDK_KEY present" do
-      ENV["AWS_SDK_KEY"] = "valid_key"
-      spaminess = described_class.calculate_spaminess(comment, function_caller)
-      expect(spaminess).to eq(1)
-      ENV["AWS_SDK_KEY"] = nil
+    it "returns 0 if user is trusted even if new social" do
+      user = create(:user)
+      user.update_column(:github_created_at, 1.day.ago)
+      user.update_column(:registered_at, 1.hour.ago)
+      user.add_role(:trusted)
+      article = create(:article, score: 99, published_at: Time.current)
+      expect(described_class.calculate_spaminess(article)).to eq(0)
     end
 
-    it "returns the default retrun value if AWS_SDK_KEY is placeholder" do
-      ENV["AWS_SDK_KEY"] = "foobarbaz"
-      spaminess = described_class.calculate_spaminess(comment, function_caller)
-      expect(spaminess).to eq(0)
-      ENV["AWS_SDK_KEY"] = nil
+    it "returns 0 if badge_achievements_count is high" do
+      user = create(:user)
+      user.update_column(:github_created_at, 1.day.ago)
+      user.update_column(:registered_at, 1.hour.ago)
+      user.update_column(:badge_achievements_count, 2)
+      article = create(:article, score: 99, published_at: Time.current)
+      expect(described_class.calculate_spaminess(article)).to eq(0)
     end
   end
 end

spec/labor/coupon_generator_spec.rb

@@ -1,12 +0,0 @@
-require "rails_helper"
-
-RSpec.describe CouponGenerator, type: :labor, vcr: {} do
-  let(:versions) { %w[member_discount sticker_pack tee_pack] }
-
-  describe "#expect" do
-    it "generates code with proper prefix ( [version]_[code] )" do
-      version = versions.sample
-      expect(described_class.new(1, version).generate).to include("#{version}_")
-    end
-  end
-end

spec/labor/function_caller_spec.rb

@@ -1,36 +0,0 @@
-require "rails_helper"
-
-RSpec.describe FunctionCaller, type: :labor do
-  let(:payload) { { user_id: 1, name: "hello" } }
-  let(:dummy_client) { double }
-  let(:result_struct) { Struct.new(:payload, keyword_init: true) }
-  let(:result) { result_struct.new(payload: [{ body: { message: "hi" }.to_json }.to_json]) }
-
-  before do
-    allow(dummy_client).to receive(:invoke).and_return(result)
-  end
-
-  it "calls the aws_lambda_client" do
-    described_class.call("some_function", payload, dummy_client)
-    expect(dummy_client).to have_received(:invoke).with(function_name: "some_function", payload: payload)
-  end
-
-  it "returns parsed value" do
-    value = described_class.call("some_function", payload, dummy_client)
-    expect(value).to eq("hi")
-  end
-
-  it "doesn't fail when payload is empty" do
-    empty_result = result_struct.new(payload: [nil])
-    empty_client = double
-    allow(empty_client).to receive(:invoke).and_return(empty_result)
-    expect(described_class.call("some_function", payload, empty_client)).to be_nil
-  end
-
-  it "doesn't fail when payload is empty and is a hash" do
-    empty_result = result_struct.new(payload: [{}.to_json])
-    empty_client = double
-    allow(empty_client).to receive(:invoke).and_return(empty_result)
-    expect(described_class.call("some_function", payload, empty_client)).to be_nil
-  end
-end