Dispatch events to the webhook endpoints #3715 (#3872)

* Start with webhooks: table and model

* Start with webhooks api

* Start with webhooks api

* Webhook::Event class and events list

* Remove commented callbacks

* Start with sending events to webhook endpoints

* A couple of tests for Articles::Creator

* Send event to webhook endpoint on article destroy

* Dispatch event on article update

* Dispatch event when an article updated from admin

* Spec for the webhook job

* One more test for the dispatch event job

* Integration-like spec for event dispatching to webhook endpoints when creating an article

* Use Addressable::URI to parse endpoint url

* Add oj as a faster fast_jsonapi backend

* Renamed serializers

* Move article serialization out of a model

* Don't allow to create a webhook event with invalid type

* Add fields for ArticleSerializer

* Fix webhook event job specs

* Specify timeout when dispatching events

* Fix webhook job spec

* Change webhook events queue name

* Change serialized article fields for the dispatchable events

* Include user data when serializing an article for webhook event

* Moved decorating out of Webhook::DispatchEvent, fixed most specs

* Fix route in ArticleSerializer

* Move Article decoration to Webhook::PayloadAdapter

* Refactor image url helpers to avoid including helpers into serializer

* Fix specs

* Default url options for production
This commit is contained in:
Anna Buianova 2019-09-07 20:17:45 +03:00 committed by Ben Halpern
parent f7d6874956
commit 4c9a40be41
32 changed files with 530 additions and 77 deletions

View file

@ -38,6 +38,7 @@ gem "dry-struct", "~> 1.0" # Typed structs and value objects
gem "email_validator", "~> 2.0" # Email validator for Rails and ActiveModel
gem "emoji_regex", "~> 2.0" # A pair of Ruby regular expressions for matching Unicode Emoji symbols
gem "envied", "~> 0.9" # Ensure presence and type of your app's ENV-variables
gem "fast_jsonapi", "~> 1.5" # Serializer for Ruby objects
gem "fastly", "~> 1.15" # Client library for the Fastly acceleration system
gem "fastly-rails", "~> 0.8" # Fastly dynamic caching integration for Rails
gem "feedjira", "~> 3.0" # A feed fetching and parsing library
@ -57,6 +58,7 @@ gem "kaminari", "~> 1.1" # A Scope & Engine based, clean, powerful, customizable
gem "liquid", "~> 4.0" # A secure, non-evaling end user template engine with aesthetic markup
gem "nokogiri", "~> 1.10" # HTML, XML, SAX, and Reader parser
gem "octokit", "~> 4.14" # Simple wrapper for the GitHub API
gem "oj", "~> 3.9" # JSON parser and object serializer
gem "omniauth", "~> 1.9" # A generalized Rack framework for multiple-provider authentication
gem "omniauth-github", "~> 1.3" # OmniAuth strategy for GitHub
gem "omniauth-twitter", "~> 1.4" # OmniAuth strategy for Twitter

View file

@ -323,6 +323,8 @@ GEM
multipart-post (>= 1.2, < 3)
faraday_middleware (0.13.1)
faraday (>= 0.7.4, < 1.0)
fast_jsonapi (1.5)
activesupport (>= 4.2)
fastimage (2.1.1)
fastly (1.15.0)
fastly-rails (0.8.0)
@ -511,6 +513,7 @@ GEM
rack (>= 1.2, < 3)
octokit (4.14.0)
sawyer (~> 0.8.0, >= 0.5.3)
oj (3.9.1)
omniauth (1.9.0)
hashie (>= 3.4.6, < 3.7.0)
rack (>= 1.6.2, < 3)
@ -884,6 +887,7 @@ DEPENDENCIES
erb_lint (~> 0.0)
factory_bot_rails (~> 4.11)
faker (~> 2.2)
fast_jsonapi (~> 1.5)
fastly (~> 1.15)
fastly-rails (~> 0.8)
feedjira (~> 3.0)
@ -911,6 +915,7 @@ DEPENDENCIES
nakayoshi_fork (~> 0.0.4)
nokogiri (~> 1.10)
octokit (~> 4.14)
oj (~> 3.9)
omniauth (~> 1.9)
omniauth-github (~> 1.3)
omniauth-twitter (~> 1.4)

View file

@ -17,6 +17,20 @@ module Admin
end
end
def update
if requested_resource.update(resource_params)
Webhook::DispatchEvent.call("article_updated", requested_resource)
redirect_to(
[namespace, requested_resource],
notice: translate_with_resource("update.success"),
)
else
render :edit, locals: {
page: Administrate::Page::Form.new(dashboard, requested_resource)
}
end
end
def destroy
Articles::Destroyer.call(requested_resource)
if requested_resource.destroyed?

View file

@ -145,6 +145,7 @@ class ArticlesController < ApplicationController
end
updated = @article.update(article_params_json.merge(edited_at: edited_at_date))
handle_notifications(updated)
Webhook::DispatchEvent.call("article_updated", @article) if updated
respond_to do |format|
format.html do
# TODO: JSON should probably not be returned in the format.html section

View file

@ -107,23 +107,7 @@ module ApplicationHelper
end
def cloud_cover_url(url)
return if url.blank?
return asset_path("triple-unicorn") if Rails.env.test?
return url if Rails.env.development?
width = 1000
height = 420
quality = "auto"
cl_image_path(url,
type: "fetch",
width: width,
height: height,
crop: "imagga_scale",
quality: quality,
flags: "progressive",
fetch_format: "auto",
sign_url: true)
CloudCoverUrl.new(url).call
end
def tag_colors(tag)

View file

@ -18,49 +18,10 @@ module SocialImageHelper
end
def article_social_image_url(article)
image = user_defined_image(article)
if image.present?
return cl_image_path(image,
type: "fetch",
width: "1000",
height: "500",
crop: "imagga_scale",
quality: "auto",
flags: "progressive",
fetch_format: "auto",
sign_url: true)
end
return legacy_article_social_image(article) unless use_new_social_url?(article)
article_social_preview_url(article, format: :png)
end
def legacy_article_social_image(article)
cache_key = "article-social-img-#{article}-#{article.updated_at}-#{article.comments_count}"
Rails.cache.fetch(cache_key, expires_in: 1.hour) do
src = GeneratedImage.new(article).social_image
return src if src.start_with? "https://res.cloudinary.com/"
cl_image_path(src,
type: "fetch",
width: "1000",
height: "500",
crop: "imagga_scale",
quality: "auto",
flags: "progressive",
fetch_format: "auto",
sign_url: true)
end
Articles::SocialImage.new(article).url
end
def use_new_social_url?(resource)
resource.updated_at > SOCIAL_PREVIEW_MIGRATION_DATETIME
end
def user_defined_image(article)
return article.social_image if article.social_image.present?
return article.main_image if article.main_image.present?
return article.video_thumbnail_url if article.video_thumbnail_url.present?
end
end

View file

@ -0,0 +1,10 @@
module Webhook
class DispatchEventJob < ApplicationJob
queue_as :webhook_dispatch_events
def perform(endpoint_url:, payload:, client: HTTParty)
uri = Addressable::URI.parse(endpoint_url)
client.post(uri, headers: { "Content-Type" => "application/json" }, body: payload, timeout: 10)
end
end
end

View file

@ -6,13 +6,20 @@ module Webhook
article_destroyed
].freeze
def initialize(event_name, payload = {})
@event_name = event_name
attr_reader :event_type, :payload, :timestamp
def initialize(event_type:, payload: {})
raise InvalidEvent unless EVENT_TYPES.include?(event_type)
@event_type = event_type
@payload = payload
@timestamp = Time.current.rfc3339
end
private
attr_reader :event_name, :payload
def as_json(*_args)
Webhook::EventSerializer.new(self).serializable_hash
end
end
class InvalidEvent < StandardError; end
end

View file

@ -0,0 +1,7 @@
module Webhook
class ArticleDestroyedSerializer
include FastJsonapi::ObjectSerializer
set_type :article
attributes :title
end
end

View file

@ -0,0 +1,36 @@
module Webhook
class ArticleSerializer
include FastJsonapi::ObjectSerializer
set_type :article
attributes :title, :description, :readable_publish_date, :cached_tag_list, :cached_tag_list_array,
:slug, :path, :url, :comments_count, :positive_reactions_count, :body_markdown
attribute :canonical_url, &:processed_canonical_url
attribute :body_html, &:processed_html
attribute :created_at do |a|
a.created_at.utc.iso8601
end
attribute :edited_at do |a|
a.edited_at&.utc&.iso8601
end
attribute :crossposted_at do |a|
a.crossposted_at&.utc&.iso8601
end
attribute :published_at do |a|
a.published_at&.utc&.iso8601
end
attribute :last_comment_at do |a|
a.last_comment_at&.utc&.iso8601
end
attribute :cover_image do |a|
CloudCoverUrl.new(a.url).call
end
attribute :social_image do |article|
Articles::SocialImage.new(article).url
end
attribute :user do |a|
UserSerializer.new(a.user).serializable_hash
end
end
end

View file

@ -0,0 +1,10 @@
module Webhook
class EventSerializer
include FastJsonapi::ObjectSerializer
set_type :webhook_event
set_id do |event|
"#{event.event_type}_#{event.timestamp}"
end
attributes :event_type, :timestamp, :payload
end
end

View file

@ -0,0 +1,14 @@
module Webhook
class UserSerializer
include FastJsonapi::ObjectSerializer
attributes :name, :username, :twitter_username, :github_username
attribute :website_url, &:processed_website_url
attribute :profile_image do |user|
ProfileImage.new(user).get(640)
end
attribute :profile_image_90 do |user|
ProfileImage.new(user).get(90)
end
end
end

View file

@ -1,8 +1,9 @@
module Articles
class Creator
def initialize(user, article_params)
def initialize(user, article_params, event_dispatcher = Webhook::DispatchEvent)
@user = user
@article_params = article_params
@event_dispatcher = event_dispatcher
end
def self.call(*args)
@ -12,8 +13,29 @@ module Articles
def call
raise if RateLimitChecker.new(user).limit_by_action("published_article_creation")
tags = article_params[:tags]
article = save_article
if article.persisted?
NotificationSubscription.create(user: user, notifiable_id: article.id, notifiable_type: "Article", config: "all_comments")
Notification.send_to_followers(article, "Published") if article.published
dispatch_event(article)
end
article.decorate
end
private
attr_reader :user, :article_params, :event_dispatcher
def dispatch_event(article)
event_dispatcher.call("article_created", article)
end
def save_article
series = article_params[:series]
tags = article_params[:tags]
# convert tags from array to a string
if tags.present?
@ -25,15 +47,8 @@ module Articles
article.user_id = user.id
article.show_comments = true
article.collection = Collection.find_series(series, user) if series.present?
if article.save
NotificationSubscription.create(user: user, notifiable_id: article.id, notifiable_type: "Article", config: "all_comments")
Notification.send_to_followers(article, "Published") if article.published
end
article.decorate
article.save
article
end
private
attr_reader :user, :article_params
end
end

View file

@ -2,10 +2,11 @@ module Articles
module Destroyer
module_function
def call(article)
def call(article, event_dispatcher = Webhook::DispatchEvent)
article.destroy!
Notification.remove_all_without_delay(notifiable_id: article.id, notifiable_type: "Article", action: "Published")
Notification.remove_all(notifiable_id: article.id, notifiable_type: "Article", action: "Reaction")
event_dispatcher.call("article_destroyed", article)
end
end
end

View file

@ -1,9 +1,10 @@
module Articles
class Updater
def initialize(user, article_id, article_params)
def initialize(user, article_id, article_params, event_dispatcher = Webhook::DispatchEvent)
@user = user
@article_id = article_id
@article_params = article_params
@event_dispatcher = event_dispatcher
end
def self.call(*args)
@ -37,13 +38,18 @@ module Articles
send_notification = article.published && article.saved_change_to_published_at.present?
Notification.send_to_followers(article, "Published") if send_notification
dispatch_event(article)
article.decorate
end
private
attr_reader :user, :article_id
attr_accessor :article_params
attr_reader :user, :article_id, :article_params, :event_dispatcher
def dispatch_event(article)
event_dispatcher.call("article_updated", article)
end
def load_article
relation = user.has_role?(:super_admin) ? Article.includes(:user) : user.articles

View file

@ -0,0 +1,26 @@
module Webhook
class DispatchEvent
def initialize(event_type, record)
@event_type = event_type
@record = record
end
def self.call(*args)
new(*args).call
end
def call
endpoint_urls = Endpoint.for_events([event_type]).pluck(:target_url)
return if endpoint_urls.empty?
event_json = Event.new(event_type: event_type, payload: PayloadAdapter.new(record).hash).to_json
endpoint_urls.each do |url|
DispatchEventJob.perform_later(endpoint_url: url, payload: event_json)
end
end
private
attr_reader :event_type, :record
end
end

View file

@ -0,0 +1,30 @@
module Webhook
class PayloadAdapter
def initialize(object)
raise InvalidPayloadObject unless object.is_a?(Article)
@object = object
end
def hash
serializer.new(prepared_object).serializable_hash
end
private
attr_reader :object
# decorate article before serializing
def prepared_object
return object unless object.is_a?(Article) && !object.decorated?
object.decorate
end
def serializer
object.destroyed? ? ArticleDestroyedSerializer : ArticleSerializer
end
end
class InvalidPayloadObject < StandardError; end
end

View file

@ -0,0 +1,63 @@
module Articles
class SocialImage
include Rails.application.routes.url_helpers
include CloudinaryHelper
def initialize(article)
@article = article
end
SOCIAL_PREVIEW_MIGRATION_DATETIME = Time.zone.parse("2019-04-22T00:00:00Z")
def url
image = user_defined_image
if image.present?
return cl_image_path(image,
type: "fetch",
width: "1000",
height: "500",
crop: "imagga_scale",
quality: "auto",
flags: "progressive",
fetch_format: "auto",
sign_url: true)
end
return legacy_article_social_image unless use_new_social_url?
article_social_preview_url(article, format: :png)
end
private
attr_reader :article
def legacy_article_social_image
cache_key = "article-social-img-#{article}-#{article.updated_at}-#{article.comments_count}"
Rails.cache.fetch(cache_key, expires_in: 1.hour) do
src = GeneratedImage.new(article).social_image
return src if src.start_with? "https://res.cloudinary.com/"
cl_image_path(src,
type: "fetch",
width: "1000",
height: "500",
crop: "imagga_scale",
quality: "auto",
flags: "progressive",
fetch_format: "auto",
sign_url: true)
end
end
def use_new_social_url?
article.updated_at > SOCIAL_PREVIEW_MIGRATION_DATETIME
end
def user_defined_image
return article.social_image if article.social_image.present?
return article.main_image if article.main_image.present?
return article.video_thumbnail_url if article.video_thumbnail_url.present?
end
end
end

View file

@ -0,0 +1,31 @@
class CloudCoverUrl
include CloudinaryHelper
include ActionView::Helpers::AssetUrlHelper
def initialize(url)
@url = url
end
def call
return if url.blank?
return url if Rails.env.development? || Rails.env.test?
width = 1000
height = 420
quality = "auto"
cl_image_path(url,
type: "fetch",
width: width,
height: height,
crop: "imagga_scale",
quality: quality,
flags: "progressive",
fetch_format: "auto",
sign_url: true)
end
private
attr_reader :url
end

View file

@ -43,6 +43,7 @@ module PracticalDeveloper
config.autoload_paths += Dir["#{config.root}/app/sanitizers"]
config.autoload_paths += Dir["#{config.root}/app/facades"]
config.autoload_paths += Dir["#{config.root}/app/errors"]
config.autoload_paths += Dir["#{config.root}/app/view_objects"]
config.autoload_paths += Dir["#{config.root}/lib/"]
config.active_record.observers = :article_observer, :reaction_observer, :comment_observer

View file

@ -107,3 +107,5 @@ Rails.application.configure do
Bullet.rails_logger = true
end
end
Rails.application.routes.default_url_options = { host: Rails.application.config.app_domain }

View file

@ -125,3 +125,5 @@ Rails.application.configure do
config.middleware.use Rack::HostRedirect,
"practicaldev.herokuapp.com" => "dev.to"
end
Rails.application.routes.default_url_options = { host: Rails.application.config.app_domain }

View file

@ -68,3 +68,5 @@ Rails.application.configure do
Bullet.add_whitelist(type: :n_plus_one_query, class_name: "ActsAsTaggableOn::Tagging", association: :tag)
end
end
Rails.application.routes.default_url_options = { host: "test.host" }

View file

@ -0,0 +1,27 @@
require "rails_helper"
RSpec.describe Webhook::DispatchEventJob, type: :job do
include_examples "#enqueues_job", "webhook_dispatch_events", "https://example.com", ""
describe "#perform_now" do
let(:article) { create(:article) }
let(:payload) { Webhook::PayloadAdapter.new(article).hash }
let(:json) { Webhook::Event.new(event_type: "article_updated", payload: payload).to_json }
let(:url) { Faker::Internet.url }
it "posts an event" do
client = double
allow(client).to receive(:post)
described_class.perform_now(endpoint_url: url, payload: json, client: client)
expect(client).to have_received(:post).once.
with(Addressable::URI.parse(url), headers: { "Content-Type" => "application/json" },
body: json,
timeout: 10)
end
it "doesn't fail" do
stub_request(:post, url).to_return(status: 200)
described_class.perform_now(endpoint_url: url, payload: json)
end
end
end

View file

@ -0,0 +1,45 @@
require "rails_helper"
RSpec.describe Webhook::Event, type: :model do
let(:article) { create(:article) }
let!(:payload) { Webhook::PayloadAdapter.new(article).hash }
it "rases an exception" do
expect do
described_class.new(event_type: "cool_event")
end.to raise_error(Webhook::InvalidEvent)
end
describe "#as_json" do
it "provides correct json (sample)" do
event = described_class.new(event_type: "article_created", payload: { title: "Hello, world" })
hash = event.as_json
attributes = hash[:data][:attributes]
expect(attributes[:event_type]).to eq("article_created")
expect(attributes[:timestamp]).to be_truthy
expect(attributes[:payload][:title]).to eq("Hello, world")
end
it "provides correct json including article" do
event = described_class.new(event_type: "article_updated", payload: payload)
hash = event.as_json
attributes = hash[:data][:attributes]
expect(attributes[:event_type]).to eq("article_updated")
expect(attributes[:payload][:data][:attributes][:title]).to eq(article.title)
end
end
describe "#to_json" do
it "provides correct json including article" do
event = described_class.new(event_type: "article_updated", payload: payload)
json = event.to_json
hash = JSON.parse(json)
attributes = hash["data"]["attributes"]
expect(attributes["event_type"]).to eq("article_updated")
expect(attributes["payload"]["data"]["attributes"]["title"]).to eq(article.title)
end
end
end

View file

@ -47,4 +47,33 @@ RSpec.describe "ArticlesCreate", type: :request do
}
expect(Collection.last.slug).to eq("helloyo")
end
context "when scheduling jobs" do
let(:url) { Faker::Internet.url(scheme: "https") }
let(:article_params) do
{
article: {
title: "NEW TITLE #{rand(100)}",
body_markdown: "---\ntitle: hey hey hahuu\npublished: false\nseries: helloyo\n---\nYo ho ho#{rand(100)}"
}
}
end
before do
create(:webhook_endpoint, events: %w[article_created article_updated], target_url: url)
end
it "schedules a dispatching event job" do
expect do
post "/articles", params: article_params
end.to have_enqueued_job(Webhook::DispatchEventJob).once
end
it "doesn't fail when executing jobs" do
stub_request(:post, url).to_return(status: 200)
perform_enqueued_jobs do
post "/articles", params: article_params
end
end
end
end

View file

@ -108,4 +108,13 @@ RSpec.describe "ArticlesUpdate", type: :request do
expect(response).to redirect_to "#{article.path}/edit"
expect(article.reload.video_thumbnail_url).to include "https://i.imgur.com/HPiu7N4.jpg"
end
it "schedules a dispatching event job" do
create(:webhook_endpoint, events: %w[article_created article_updated])
expect do
put "/articles/#{article.id}", params: {
article: { title: "new_title", body_markdown: "Yo ho ho#{rand(100)}", tag_list: "yo" }
}
end.to have_enqueued_job(Webhook::DispatchEventJob).once
end
end

View file

@ -30,6 +30,13 @@ RSpec.describe Articles::Creator do
described_class.call(user, valid_attributes)
end.to change(NotificationSubscription, :count).by(1)
end
it "calls an event dispatcher" do
event_dispatcher = double
allow(event_dispatcher).to receive(:call)
article = described_class.call(user, valid_attributes, event_dispatcher)
expect(event_dispatcher).to have_received(:call).with("article_created", article.object)
end
end
context "when valid attributes" do
@ -63,5 +70,12 @@ RSpec.describe Articles::Creator do
described_class.call(user, invalid_attributes)
end.not_to change(NotificationSubscription, :count)
end
it "doesn't call an event dispatcher" do
event_dispatcher = double
allow(event_dispatcher).to receive(:call)
described_class.call(user, invalid_attributes, event_dispatcher)
expect(event_dispatcher).not_to have_received(:call)
end
end
end

View file

@ -0,0 +1,23 @@
require "rails_helper"
RSpec.describe Articles::Destroyer do
let(:article) { create(:article) }
it "destroys an article" do
described_class.call(article)
expect(Article.find_by(id: article.id)).to be_nil
end
it "schedules removing notifications" do
expect do
described_class.call(article)
end.to have_enqueued_job(Notifications::RemoveAllJob).once
end
it "calls events dispatcher" do
event_dispatcher = double
allow(event_dispatcher).to receive(:call)
described_class.call(article, event_dispatcher)
expect(event_dispatcher).to have_received(:call).with("article_destroyed", article)
end
end

View file

@ -0,0 +1,20 @@
require "rails_helper"
RSpec.describe Articles::Updater do
let(:user) { create(:user) }
let!(:article) { create(:article, user: user) }
let(:attributes) { { body_markdown: "sample" } }
it "updates an article" do
described_class.call(user, article.id, attributes)
article.reload
expect(article.body_markdown).to eq("sample")
end
it "calls events dispatcher" do
event_dispatcher = double
allow(event_dispatcher).to receive(:call)
described_class.call(user, article.id, attributes, event_dispatcher)
expect(event_dispatcher).to have_received(:call).with("article_updated", article)
end
end

View file

@ -0,0 +1,21 @@
require "rails_helper"
RSpec.describe Webhook::DispatchEvent, type: :service do
let!(:article) { create(:article) }
it "does nothing if there are no corresponding endpoints" do
create(:webhook_endpoint, events: %w[article_created])
expect do
described_class.call("article_destroyed", article)
end.not_to have_enqueued_job(Webhook::DispatchEventJob)
end
it "schedules jobs" do
create(:webhook_endpoint, events: %w[article_created], target_url: "https://create-webhooks.example.com/accept")
create(:webhook_endpoint, events: %w[article_created article_updated article_destroyed], target_url: "https://all-webhooks.example.com/accept")
create(:webhook_endpoint, events: %w[article_destroyed], target_url: "https://destroy-webhooks.example.com/accept")
expect do
described_class.call("article_created", article)
end.to have_enqueued_job(Webhook::DispatchEventJob).twice
end
end

View file

@ -0,0 +1,35 @@
require "rails_helper"
RSpec.describe Webhook::PayloadAdapter, type: :service do
it "raises an exception when invalid object is passed" do
expect do
described_class.new(User.new).hash
end.to raise_error(Webhook::InvalidPayloadObject)
end
describe "#hash" do
let(:user) { create(:user) }
let!(:article) { create(:article, title: "I'm super", user: user) }
it "returns a hash for a persisted article" do
data = described_class.new(article).hash
expect(data).to be_kind_of(Hash)
expect(data[:data][:attributes][:title]).to eq(article.title)
expect(data[:data][:attributes][:body_markdown]).to be_truthy
end
it "returns a hash with a user" do
data = described_class.new(article).hash
expect(data[:data][:attributes][:user][:data][:attributes][:username]).to eq(user.username)
end
it "returns a hash for a destroyed article" do
article = create(:article, title: "hello")
article.destroy
data = described_class.new(article).hash
expect(data).to be_kind_of(Hash)
expect(data[:data][:attributes][:title]).to eq("hello")
expect(data[:data][:attributes][:body_markdown]).to be_falsey
end
end
end