Export articles/posts (#576)

* Add EditorConfig file for all editors

* Add article export service

* Add setting to request an export of all articles

* On the outside they are called posts

* Add articles exported email to mailer and service

* Refactor to one public method and test flags

* Trigger the export if it was requested and send the email

* Recreated migration file with generic export fields

* Rename fields and switch to a whitelist

* Refactor ArticleExportService into a more generic structure

* Rename articles_exported_email to export_email

* Fix notify mailer spec

* Rename Exporter::Exporter to Exporter::Service

* Remove commented out line

* Removed body_html, coordinates and updated_at from export

* Invert DJ config

* Update spec

.editorconfig

@@ -0,0 +1,10 @@
+# editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true

Gemfile

@@ -78,6 +78,7 @@ gem "redcarpet", "~> 3.4"
 gem "reverse_markdown", "~> 1.1"
 gem "rolify", "~> 5.2"
 gem "rouge", "~> 3.2"
+gem "rubyzip", "~> 1.2", ">= 1.2.2"
 gem "s3_direct_upload", "~> 0.1"
 gem "sass-rails", "~> 5.0"
 gem "sdoc", "~> 0.4", group: :doc

Gemfile.lock

@@ -997,6 +997,7 @@ DEPENDENCIES
   rubocop (~> 0.59)
   rubocop-rspec (~> 1.29)
   ruby-prof (~> 0.17)
+  rubyzip (~> 1.2, >= 1.2.2)
   s3_direct_upload (~> 0.1)
   sass-rails (~> 5.0)
   sdoc (~> 0.4)

app/controllers/users_controller.rb

@@ -25,6 +25,12 @@ class UsersController < ApplicationController
     if @user.update(permitted_attributes(@user))
       RssReader.new.delay.fetch_user(@user) if @user.feed_url.present?
       notice = "Your profile was successfully updated."
+
+      if @user.export_requested?
+        notice = notice + " The export will be emailed to you shortly."
+        Exporter::Service.new(@user).delay.export(send_email: true)
+      end
+
       follow_hiring_tag(@user)
       redirect_to "/settings/#{@tab}", notice: notice
     else

app/mailers/notify_mailer.rb

@@ -94,4 +94,11 @@ class NotifyMailer < ApplicationMailer
     subject = "You have been matched with a new DEV mentee!"
     mail(to: @mentor.email, subject: subject, from: "Liana (from dev.to) <liana@dev.to>")
   end
+
+  def export_email(user, attachment)
+    @user = user
+    export_filename = "devto-export-#{Date.current.iso8601}.zip"
+    attachments[export_filename] = attachment
+    mail(to: @user.email, subject: "The export of your data is ready")
+  end
 end

app/policies/user_policy.rb

@@ -103,7 +103,8 @@ class UserPolicy < ApplicationPolicy
        summary
        text_color_hex
        username
-       website_url]
+       website_url
+       export_requested]
   end
 
   private

app/services/exporter/articles.rb

@@ -0,0 +1,67 @@
+module Exporter
+  class Articles
+    attr_reader :name
+    attr_reader :user
+
+    def initialize(user)
+      @name = :articles
+      @user = user
+    end
+
+    def export(slug: nil)
+      articles = user.articles
+      articles = articles.where(slug: slug) if slug.present?
+      json_articles = jsonify_articles(articles)
+
+      { "articles.json" => json_articles }
+    end
+
+    private
+
+    def whitelisted_attributes
+      %i[
+        body_markdown
+        cached_tag_list
+        cached_user_name
+        cached_user_username
+        canonical_url
+        comments_count
+        created_at
+        crossposted_at
+        description
+        edited_at
+        feed_source_url
+        language
+        last_comment_at
+        main_image
+        main_image_background_hex_color
+        path
+        positive_reactions_count
+        processed_html
+        published
+        published_at
+        published_from_feed
+        reactions_count
+        show_comments
+        slug
+        social_image
+        title
+        video
+        video_closed_caption_track_url
+        video_code
+        video_source_url
+        video_thumbnail_url
+      ]
+    end
+
+    def jsonify_articles(articles)
+      articles_to_jsonify = []
+      # load articles in batches, we don't want to hog the DB
+      # if a user has lots and lots of articles
+      articles.find_each do |article|
+        articles_to_jsonify << article
+      end
+      articles_to_jsonify.to_json(only: whitelisted_attributes)
+    end
+  end
+end

app/services/exporter/service.rb

@@ -0,0 +1,64 @@
+require "zip"
+
+module Exporter
+  class Service
+    attr_reader :user
+
+    EXPORTERS = [
+      Articles,
+    ].freeze
+
+    def initialize(user)
+      @user = user
+    end
+
+    def export(send_email: false, config: {})
+      exports = {}
+
+      # export content with filenames
+      EXPORTERS.each do |exporter|
+        files = exporter.new(user).export(config.fetch(exporter.name.to_sym, {}))
+        files.each do |name, content|
+          exports[name] = content
+        end
+      end
+
+      zipped_exports = zip_exports(exports)
+
+      send_exports_by_email(zipped_exports) if send_email
+
+      update_user_export_fields
+
+      zipped_exports.rewind
+      zipped_exports
+    end
+
+    private
+
+    def zip_exports(exports)
+      buffer = StringIO.new
+      Zip::OutputStream.write_buffer(buffer) do |stream|
+        exports.each do |name, content|
+          stream.put_next_entry(
+            name,
+            nil, # comment
+            nil, # extra
+            Zip::Entry::DEFLATED,
+            Zlib::BEST_COMPRESSION,
+          )
+          stream.write content
+        end
+      end
+      buffer
+    end
+
+    def send_exports_by_email(zipped_exports)
+      zipped_exports.rewind
+      NotifyMailer.export_email(user, zipped_exports.read).deliver
+    end
+
+    def update_user_export_fields
+      user.update!(export_requested: false, exported_at: Time.current)
+    end
+  end
+end

app/views/mailers/notify_mailer/export_email.html.erb

@@ -0,0 +1,4 @@
+<h3 style="font-weight:300">Your data has been exported.</h3>
+<p>
+Please check the attached file.
+</p>

app/views/mailers/notify_mailer/export_email.text.erb

@@ -0,0 +1,3 @@
+Your data has been exported.
+
+Please check the attached file.

app/views/users/_misc.html.erb

@@ -50,7 +50,7 @@
       <%= f.check_box :prefer_language_it %>
       <%= f.label :prefer_language_it, "Italian" %>
     </div>
-  </div>  
+  </div>
   <div class="field">
     <label></label>
     <%= f.hidden_field :tab, value: @tab %>
@@ -85,3 +85,32 @@
     <%= f.submit "SUBMIT", class: "cta" %>
   </div>
 <% end %>
+
+<h2>Export posts</h2>
+
+<% if @user.export_requested? %>
+  <h4 style="font-weight:400">
+    You have recently requested an export of your data.
+    Please check your email.
+  </h4>
+<% else %>
+  <h4 style="font-weight:400">
+    You can request an export of all your data.
+    Currently we only support the export of posts.
+    It will be emailed to your inbox.
+  </h4>
+
+  <%= form_for(@user) do |f| %>
+    <div class="checkbox-field">
+      <div class="sub-field">
+        <%= f.check_box :export_requested %>
+        <%= f.label :export_requested, "Request an export of your data" %>
+      </div>
+    </div>
+    <div class="field">
+      <label></label>
+      <%= f.hidden_field :tab, value: @tab %>
+      <%= f.submit "SUBMIT", class: "cta" %>
+    </div>
+  <% end %>
+<% end %>

db/migrate/20181026112019_add_export_fields_to_users.rb

@@ -0,0 +1,6 @@
+class AddExportFieldsToUsers < ActiveRecord::Migration[5.1]
+  def change
+    add_column :users, :export_requested, :boolean, default: false
+    add_column :users, :exported_at, :datetime
+  end
+end

db/schema.rb

@@ -711,6 +711,8 @@ ActiveRecord::Schema.define(version: 20181120170350) do
     t.string "employment_title"
     t.string "encrypted_password", default: "", null: false
     t.integer "experience_level"
+    t.boolean "export_requested", default: false
+    t.datetime "exported_at"
     t.string "facebook_url"
     t.boolean "feed_admin_publish_permission", default: true
     t.boolean "feed_mark_canonical", default: false

spec/mailers/notify_mailer_spec.rb

@@ -120,6 +120,7 @@ RSpec.describe NotifyMailer, type: :mailer do
         mentee_email = described_class.mentee_email(mentee, mentor)
         expect(mentee_email.subject).to eq "You have been matched with a DEV mentor!"
       end
+
       it "renders proper from" do
         mentee_email = described_class.mentee_email(mentee, mentor)
         expect(mentee_email.from).to include "liana@dev.to"
@@ -134,10 +135,34 @@ RSpec.describe NotifyMailer, type: :mailer do
         mentor_email = described_class.mentor_email(mentor, mentee)
         expect(mentor_email.subject).to eq "You have been matched with a new DEV mentee!"
       end
-      it "renders proper subject" do
+
+      it "renders proper from" do
         mentor_email = described_class.mentor_email(mentor, mentee)
         expect(mentor_email.from).to include "liana@dev.to"
       end
     end
+
+    describe "#export_email" do
+      it "renders proper subject" do
+        export_email = described_class.export_email(user, "attachment")
+        expect(export_email.subject).to include("export of your data is ready")
+      end
+
+      it "renders proper receiver" do
+        export_email = described_class.export_email(user, "attachment")
+        expect(export_email.to).to eq([user.email])
+      end
+
+      it "attaches a zip file" do
+        export_email = described_class.export_email(user, "attachment")
+        expect(export_email.attachments[0].content_type).to include("application/zip")
+      end
+
+      it "adds the correct filename" do
+        export_email = described_class.export_email(user, "attachment")
+        expected_filename = "devto-export-#{Date.current.iso8601}.zip"
+        expect(export_email.attachments[0].filename).to eq(expected_filename)
+      end
+    end
   end
 end

spec/requests/user_settings_spec.rb

@@ -53,6 +53,10 @@ RSpec.describe "UserSettings", type: :request do
   describe "PUT /update/:id" do
     before { login_as user }
 
+    after do
+      Delayed::Worker.delay_jobs = false
+    end
+
     it "updates summary" do
       put "/users/#{user.id}", params: { user: { tab: "profile", summary: "Hello new summary" } }
       expect(user.summary).to eq("Hello new summary")
@@ -62,6 +66,50 @@ RSpec.describe "UserSettings", type: :request do
       put "/users/#{user.id}", params: { user: { tab: "profile", username: "h" } }
       expect(response.body).to include("Username is too short")
     end
+
+    context "when requesting an export of the articles" do
+      def send_request(flag = true)
+        Delayed::Worker.delay_jobs = true
+        put "/users/#{user.id}", params: {
+          user: { tab: "misc", export_requested: flag }
+        }
+      end
+
+      it "updates export_requested flag" do
+        send_request
+        expect(user.reload.export_requested).to be(true)
+      end
+
+      it "displays a flash with a reminder for the user to expect an email" do
+        send_request
+        expect(flash[:notice]).to include("The export will be emailed to you shortly.")
+      end
+
+      it "hides the checkbox" do
+        send_request
+        follow_redirect!
+        expect(response.body).not_to include("Request an export of your posts")
+      end
+
+      it "tells the user they recently requested an export" do
+        send_request
+        follow_redirect!
+        expect(response.body).to include("You have recently requested an export")
+      end
+
+      it "sends an email" do
+        expect do
+          send_request
+          Delayed::Worker.new.work_off
+          # Delayed::Worker.delay_jobs = false
+        end.to change { ActionMailer::Base.deliveries.count }.by(1)
+      end
+
+      it "does not send an email if there was no request" do
+        Delayed::Worker.delay_jobs = false
+        expect { send_request(false) }.not_to(change { ActionMailer::Base.deliveries.count })
+      end
+    end
   end
 
   describe "DELETE /users/remove_association" do

spec/services/exporter/articles_spec.rb

@@ -0,0 +1,115 @@
+require "rails_helper"
+
+RSpec.describe Exporter::Articles do
+  let(:user) { create(:user) }
+  let(:article) { create(:article, user: user) }
+  let(:other_user) { create(:user) }
+  let(:other_user_article) { create(:article, user: other_user) }
+
+  def valid_instance(user)
+    described_class.new(user)
+  end
+
+  def expected_fields
+    %w[
+      body_markdown
+      cached_tag_list
+      cached_user_name
+      cached_user_username
+      canonical_url
+      comments_count
+      created_at
+      crossposted_at
+      description
+      edited_at
+      feed_source_url
+      language
+      last_comment_at
+      main_image
+      main_image_background_hex_color
+      path
+      positive_reactions_count
+      processed_html
+      published
+      published_at
+      published_from_feed
+      reactions_count
+      show_comments
+      slug
+      social_image
+      title
+      video
+      video_closed_caption_track_url
+      video_code
+      video_source_url
+      video_thumbnail_url
+    ]
+  end
+
+  def load_articles(data)
+    JSON.parse(data["articles.json"])
+  end
+
+  describe "#initialize" do
+    it "accepts a user" do
+      exporter = valid_instance(user)
+      expect(exporter.user).to be(user)
+    end
+
+    it "names itself articles" do
+      exporter = valid_instance(user)
+      expect(exporter.name).to eq(:articles)
+    end
+  end
+
+  describe "#export" do
+    context "when slug is unknown" do
+      it "returns no articles if the slug is not found" do
+        exporter = valid_instance(user)
+        result = exporter.export(slug: "not found")
+        articles = load_articles(result)
+        expect(articles).to be_empty
+      end
+
+      it "no articles if slug belongs to another user" do
+        exporter = valid_instance(user)
+        result = exporter.export(slug: other_user_article.slug)
+        articles = load_articles(result)
+        expect(articles).to be_empty
+      end
+    end
+
+    context "when slug is known" do
+      it "returns the article" do
+        exporter = valid_instance(user)
+        result = exporter.export(slug: article.slug)
+        articles = load_articles(result)
+        expect(articles.length).to eq(1)
+      end
+
+      it "returns only expected fields for the article" do
+        exporter = valid_instance(user)
+        result = exporter.export(slug: article.slug)
+        articles = load_articles(result)
+        expect(articles.first.keys).to eq(expected_fields)
+      end
+    end
+
+    context "when all articles are requested" do
+      it "returns all the articles as json" do
+        exporter = valid_instance(article.user)
+        result = exporter.export
+        articles = load_articles(result)
+        user.reload
+        expect(articles.length).to eq(user.articles.size)
+      end
+
+      it "returns only expected fields for the article" do
+        exporter = valid_instance(article.user)
+        result = exporter.export
+        articles = load_articles(result)
+        expect(articles.first.keys).to eq(expected_fields)
+      end
+    end
+  end
+end

spec/services/exporter/service_spec.rb

@@ -0,0 +1,132 @@
+require "rails_helper"
+require "zip"
+
+RSpec.describe Exporter::Service do
+  let(:user) { create(:user) }
+  let(:article) { create(:article, user: user) }
+  let(:other_user) { create(:user) }
+  let(:other_user_article) { create(:article, user: other_user) }
+
+  before do
+    ActionMailer::Base.deliveries.clear
+  end
+
+  def valid_instance(user)
+    described_class.new(user)
+  end
+
+  def extract_zipped_exports(buffer)
+    exports = {}
+
+    buffer.rewind
+    Zip::InputStream.open(buffer) do |stream|
+      loop do
+        entry = stream.get_next_entry
+        break if entry.blank?
+        exports[entry.name] = stream.read
+      end
+    end
+
+    exports
+  end
+
+  def expected_fields
+    [
+      "body_html",
+      "body_markdown",
+      "cached_tag_list",
+      "cached_user_name",
+      "cached_user_username",
+      "canonical_url",
+      "comments_count",
+      "created_at",
+      "crossposted_at",
+      "description",
+      "edited_at",
+      "feed_source_url",
+      "language",
+      "last_comment_at",
+      "lat",
+      "long",
+      "main_image",
+      "main_image_background_hex_color",
+      "path",
+      "positive_reactions_count",
+      "processed_html",
+      "published",
+      "published_at",
+      "published_from_feed",
+      "reactions_count",
+      "show_comments",
+      "slug",
+      "social_image",
+      "title",
+      "updated_at",
+      "video",
+      "video_closed_caption_track_url",
+      "video_code",
+      "video_source_url",
+      "video_thumbnail_url",
+    ]
+  end
+
+  describe "EXPORTERS" do
+    it "is a list of supported exporters" do
+      expect(described_class::EXPORTERS).to eq([Exporter::Articles])
+    end
+  end
+
+  describe "#initialize" do
+    it "accepts a user" do
+      service = valid_instance(user)
+      expect(service.user).to be(user)
+    end
+  end
+
+  describe "#export" do
+    it "exports a zip file with files" do
+      service = valid_instance(article.user)
+      zipped_exports = service.export
+      exports = extract_zipped_exports(zipped_exports)
+      expect(exports.keys).to eq(["articles.json"])
+    end
+
+    it "passes configuration to an exporter" do
+      service = valid_instance(article.user)
+      zipped_exports = service.export(config: { articles: { slug: article.slug } })
+      exports = extract_zipped_exports(zipped_exports)
+      expect(exports.length).to eq(1)
+    end
+
+    context "when emailing the user" do
+      it "delivers one email" do
+        service = valid_instance(article.user)
+        service.export(send_email: true)
+        expect(ActionMailer::Base.deliveries.count).to eq(1)
+      end
+
+      it "delivers an email with the export" do
+        service = valid_instance(article.user)
+        zipped_export = service.export(send_email: true)
+        attachment = ActionMailer::Base.deliveries.last.attachments[0].decoded
+
+        exports = extract_zipped_exports(zipped_export)
+        expect(exports).to eq(extract_zipped_exports(StringIO.new(attachment)))
+      end
+    end
+
+    it "sets the requested flag as false" do
+      service = valid_instance(article.user)
+      service.export
+      expect(user.export_requested).to be(false)
+    end
+
+    it "sets the exported at datetime as the current one" do
+      Timecop.freeze(Time.current) do
+        service = valid_instance(article.user)
+        service.export
+        expect(user.exported_at).to eq(Time.current)
+      end
+    end
+  end
+end