Pro: add referrers to dashboard and single article stats (#3295)

* Reorganize PageViewsController

* Add domain and path to PageView model

* Add before_create callback to populate domain and path

* Add list of referrers to AnalyticsService

* Add referrers to the UI

* Remove useless referrers card and tweak table line height

* Add referrer stats to article stats page

* Add not null and empty default to domain and path

* Refactor JS analytics client

* create_list is a step back here

* Revert "Add not null and empty default to domain and path"

This reverts commit bc02440076047a887c65d300bccd4661ecc8ffd0.

* Add index on domain concurrently

* Make the script more robust
This commit is contained in:
rhymes 2019-06-25 19:58:09 +02:00 committed by Ben Halpern
parent 16b59db15c
commit 7da7a16d8d
23 changed files with 357 additions and 212 deletions

View file

@ -36,11 +36,7 @@
display: inline-block;
width: fit-content;
margin-bottom: 10px;
@include themeable(
color,
theme-color,
$black
);
@include themeable(color, theme-color, $black);
@include themeable(
background,
theme-container-background,
@ -55,16 +51,8 @@
}
&.active {
@include themeable(
background,
theme-secondary-color,
$dark-gray
);
@include themeable(
color,
theme-container-background,
white
);
@include themeable(background, theme-secondary-color, $dark-gray);
@include themeable(color, theme-container-background, white);
}
&.back-nav {
@ -402,7 +390,7 @@
& li.ellipsis-menu-item {
list-style: none;
& form * {
background-color: transparent;
}
@ -721,6 +709,7 @@
padding: 20px;
margin: 5px;
@include themeable(background-color, theme-container-background, #fff);
h4 {
font-size: 0.8em;
margin: 0px;
@ -730,6 +719,28 @@
.stat-percentage {
font-size: 0.8em;
}
table {
margin: 1rem auto;
text-align: left;
width: 50%;
thead {
line-height: 3rem;
th:last-child {
text-align: right;
}
}
tbody {
line-height: 2rem;
td:last-child {
text-align: right;
}
}
}
}
.featured-stat {

View file

@ -35,6 +35,15 @@ module Api
render json: data.to_json
end
def referrers
analytics = AnalyticsService.new(
@owner,
start_date: params[:start], end_date: params[:end], article_id: params[:article_id],
)
data = analytics.referrers
render json: data.to_json
end
private
def authorize_pro_user

View file

@ -1,28 +1,27 @@
class PageViewsController < ApplicationController
# No policy needed. All views are for all users
def create
if user_signed_in?
PageView.create(user_id: current_user.id,
article_id: page_view_params[:article_id],
referrer: page_view_params[:referrer],
user_agent: page_view_params[:user_agent])
else
PageView.create(counts_for_number_of_views: 10,
article_id: page_view_params[:article_id],
referrer: page_view_params[:referrer],
user_agent: page_view_params[:user_agent])
end
page_view_create_params = if user_signed_in?
page_view_params.merge(user_id: current_user.id)
else
page_view_params.merge(counts_for_number_of_views: 10)
end
PageView.create(page_view_create_params)
update_article_page_views
head :ok
end
def update
if user_signed_in?
page_view = PageView.where(article_id: params[:id], user_id: current_user.id).last
page_view ||= PageView.create(user_id: current_user.id,
article_id: params[:id]) # pageview is sometimes missing if failure on prior creation.
# pageview is sometimes missing if failure on prior creation.
page_view ||= PageView.create(user_id: current_user.id, article_id: params[:id])
page_view.update_column(:time_tracked_in_seconds, page_view.time_tracked_in_seconds + 15)
end
head :ok
end
@ -34,7 +33,6 @@ class PageViewsController < ApplicationController
@article = Article.find(page_view_params[:article_id])
new_page_views_count = @article.page_views.sum(:counts_for_number_of_views)
@article.update_column(:page_views_count, new_page_views_count) if new_page_views_count > @article.page_views_count
return if Rails.env.production? && rand(20) != 1 # We need to do this operation even less often.
update_organic_page_views
end
@ -44,11 +42,19 @@ class PageViewsController < ApplicationController
end
def update_organic_page_views
organic_count = @article.page_views.where(referrer: "https://www.google.com/").sum(:counts_for_number_of_views)
return if Rails.env.production? && rand(20) != 1 # We need to do this operation only once in a while.
page_views_from_google_com = @article.page_views.where(referrer: "https://www.google.com/")
organic_count = page_views_from_google_com.sum(:counts_for_number_of_views)
@article.update_column(:organic_page_views_count, organic_count) if organic_count > @article.organic_page_views_count
organic_count_past_week_count = @article.page_views.where(referrer: "https://www.google.com/").where("created_at > ?", 1.week.ago).sum(:counts_for_number_of_views)
organic_count_past_week_count = page_views_from_google_com.
where("created_at > ?", 1.week.ago).sum(:counts_for_number_of_views)
@article.update_column(:organic_page_views_past_week_count, organic_count_past_week_count) if organic_count_past_week_count > @article.organic_page_views_past_week_count
organic_count_past_month_count = @article.page_views.where(referrer: "https://www.google.com/").where("created_at > ?", 1.month.ago).sum(:counts_for_number_of_views)
organic_count_past_month_count = page_views_from_google_com.
where("created_at > ?", 1.month.ago).sum(:counts_for_number_of_views)
@article.update_column(:organic_page_views_past_month_count, organic_count_past_month_count) if organic_count_past_month_count > @article.organic_page_views_past_month_count
end
end

View file

@ -0,0 +1,45 @@
import handleFetchAPIErrors from '../src/utils/errors';
function callAnalyticsAPI(path, date, { organizationId, articleId }, callback) {
let url = `${path}?start=${date.toISOString().split('T')[0]}`;
if (organizationId) {
url = `${url}&organization_id=${organizationId}`;
}
if (articleId) {
url = `${url}&article_id=${articleId}`;
}
fetch(url)
.then(handleFetchAPIErrors)
.then(response => response.json())
.then(callback)
// eslint-disable-next-line no-console
.catch(error => console.error(error)); // we should come up with better error handling
}
export function callHistoricalAPI(
date,
{ organizationId, articleId },
callback,
) {
callAnalyticsAPI(
'/api/analytics/historical',
date,
{ organizationId, articleId },
callback,
);
}
export function callReferrersAPI(
date,
{ organizationId, articleId },
callback,
) {
callAnalyticsAPI(
'/api/analytics/referrers',
date,
{ organizationId, articleId },
callback,
);
}

View file

@ -1,5 +1,5 @@
import Chart from 'chart.js';
import handleFetchAPIErrors from '../src/utils/errors';
import { callHistoricalAPI, callReferrersAPI } from './client';
function resetActive(activeButton) {
const buttons = document.getElementsByClassName('timerange-button');
@ -84,7 +84,7 @@ function drawCharts(data, timeRangeLabel) {
const readers = parsedData.map(date => date.page_views.total);
drawChart({
canvas: document.getElementById('reactionsChart'),
canvas: document.getElementById('reactions-chart'),
title: `Reactions ${timeRangeLabel}`,
labels,
datasets: [
@ -120,7 +120,7 @@ function drawCharts(data, timeRangeLabel) {
});
drawChart({
canvas: document.getElementById('commentsChart'),
canvas: document.getElementById('comments-chart'),
title: `Comments ${timeRangeLabel}`,
labels,
datasets: [
@ -135,7 +135,7 @@ function drawCharts(data, timeRangeLabel) {
});
drawChart({
canvas: document.getElementById('followersChart'),
canvas: document.getElementById('followers-chart'),
title: `New Followers ${timeRangeLabel}`,
labels,
datasets: [
@ -150,7 +150,7 @@ function drawCharts(data, timeRangeLabel) {
});
drawChart({
canvas: document.getElementById('readersChart'),
canvas: document.getElementById('readers-chart'),
title: `Reads ${timeRangeLabel}`,
labels,
datasets: [
@ -165,48 +165,65 @@ function drawCharts(data, timeRangeLabel) {
});
}
function callAnalyticsApi(date, timeRangeLabel, { organizationId, articleId }) {
let url = `/api/analytics/historical?start=${
date.toISOString().split('T')[0]
}`;
function renderReferrers(data) {
const container = document.getElementById('referrers-container');
const tableBody = data.domains
.filter(referrer => referrer.domain)
.map(referrer => {
return `
<tr>
<td>${referrer.domain}</td>
<td>${referrer.count}</td>
</tr>
`;
});
if (organizationId) {
url = `${url}&organization_id=${organizationId}`;
}
if (articleId) {
url = `${url}&article_id=${articleId}`;
// add referrers with empty domains if present
const emptyDomainReferrer = data.domains.filter(
referrer => !referrer.domain,
)[0];
if (emptyDomainReferrer) {
tableBody.push(`
<tr>
<td>All other external referrers</td>
<td>${emptyDomainReferrer.count}</td>
</tr>
`);
}
fetch(url)
.then(handleFetchAPIErrors)
.then(response => response.json())
.then(data => {
drawCharts(data, timeRangeLabel);
writeCards(data, timeRangeLabel);
})
// eslint-disable-next-line no-console
.catch(error => console.error(error)); // we should come up with better error handling
container.innerHTML = tableBody.join('');
}
function callAnalyticsAPI(date, timeRangeLabel, { organizationId, articleId }) {
callHistoricalAPI(date, { organizationId, articleId }, data => {
writeCards(data, timeRangeLabel);
drawCharts(data, timeRangeLabel);
});
callReferrersAPI(date, { organizationId, articleId }, data => {
renderReferrers(data);
});
}
function drawWeekCharts({ organizationId, articleId }) {
resetActive(document.getElementById('week-button'));
const oneWeekAgo = new Date();
oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);
callAnalyticsApi(oneWeekAgo, 'this Week', { organizationId, articleId });
callAnalyticsAPI(oneWeekAgo, 'this Week', { organizationId, articleId });
}
function drawMonthCharts({ organizationId, articleId }) {
resetActive(document.getElementById('month-button'));
const oneMonthAgo = new Date();
oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1);
callAnalyticsApi(oneMonthAgo, 'this Month', { organizationId, articleId });
callAnalyticsAPI(oneMonthAgo, 'this Month', { organizationId, articleId });
}
function drawInfinityCharts({ organizationId, articleId }) {
resetActive(document.getElementById('infinity-button'));
// April 1st is when the DEV analytics feature went into place
const beginningOfTime = new Date('2019-4-1');
callAnalyticsApi(beginningOfTime, '', { organizationId, articleId });
callAnalyticsAPI(beginningOfTime, '', { organizationId, articleId });
}
export default function initCharts({ organizationId, articleId }) {

View file

@ -4,6 +4,8 @@ class PageView < ApplicationRecord
belongs_to :user, optional: true
belongs_to :article
before_create :extract_domain_and_path
algoliasearch index_name: "UserHistory", per_environment: true, if: :belongs_to_pro_user? do
attributes :referrer, :user_agent, :article_tags
@ -46,6 +48,14 @@ class PageView < ApplicationRecord
private
def extract_domain_and_path
return unless referrer
parsed_url = Addressable::URI.parse(referrer)
self.domain = parsed_url.domain
self.path = parsed_url.path
end
def belongs_to_pro_user?
user&.pro?
end

View file

@ -8,7 +8,7 @@ class AnalyticsService
load_data
end
# Totals computes total counts for comments, reactions, follows and page views
# Computes total counts for comments, reactions, follows and page views
def totals
{
comments: { total: comment_data.size },
@ -18,7 +18,7 @@ class AnalyticsService
}
end
# Grouped by day computes counts for comments, reactions, follows and page views per each day
# Computes counts for comments, reactions, follows and page views per each day
def grouped_by_day
return {} unless start_date && end_date
@ -49,6 +49,16 @@ class AnalyticsService
end
end
# Returns the list of referrers
def referrers(top: 20)
# count_all is the name of the field autogenerated by Rails with COUNT(*)
counts = page_view_data.group(:domain).order(count_all: :desc).limit(top).count
# we transform this in a list of hashes in case we need to add more keys
domains = counts.map { |domain, count| { domain: domain, count: count } }
{ domains: domains }
end
private
attr_reader(

View file

@ -14,49 +14,7 @@
<h1 class="pro-header">Stats for "<%= @article.title %>"</h1>
</section>
<div class="toggles">
<button class="selected timerange-button" id="week-button">Week</button>
<button class="timerange-button" id="month-button">Month</button>
<button class="timerange-button" id="infinity-button">Infinity</button>
</div>
<div class="summary-stats">
<div class="card" id="readers-card"></div>
<div class="card" id="reactions-card"></div>
<div class="card" id="comments-card"></div>
<div class="card" id="followers-card"></div>
</div>
<div class="graphs">
<div class="row">
<div class="card">
<h2>Readers Summary</h2>
<div class="charts-container">
<canvas id="readersChart"></canvas>
</div>
</div>
<div class="card">
<h2>New Followers Summary</h2>
<div class="charts-container">
<canvas id="followersChart"></canvas>
</div>
</div>
</div>
<div class="row">
<div class="card">
<h2>Reactions Summary ❤🦄🔖</h2>
<div class="charts-container">
<canvas id="reactionsChart"></canvas>
</div>
</div>
<div class="card">
<h2>Comments Summary 💬</h2>
<div class="charts-container">
<canvas id="commentsChart"></canvas>
</div>
</div>
</div>
</div>
<%= render "shared/stats" %>
</div>
<%= javascript_pack_tag "analyticsArticle", defer: true %>

View file

@ -22,49 +22,7 @@
<p>This dashboard will highlight deep insights especially relevant to developer relations authors and serious bloggers.</p>
</section>
<div class="toggles">
<button class="selected timerange-button" id="week-button">Week</button>
<button class="timerange-button" id="month-button">Month</button>
<button class="timerange-button" id="infinity-button">Infinity</button>
</div>
<div class="summary-stats">
<div class="card" id="readers-card"></div>
<div class="card" id="reactions-card"></div>
<div class="card" id="comments-card"></div>
<div class="card" id="followers-card"></div>
</div>
<div class="graphs">
<div class="row">
<div class="card">
<h2>Readers Summary</h2>
<div class="charts-container">
<canvas id="readersChart"></canvas>
</div>
</div>
<div class="card">
<h2>New Followers Summary</h2>
<div class="charts-container">
<canvas id="followersChart"></canvas>
</div>
</div>
</div>
<div class="row">
<div class="card">
<h2>Reactions Summary ❤🦄🔖</h2>
<div class="charts-container">
<canvas id="reactionsChart"></canvas>
</div>
</div>
<div class="card">
<h2>Comments Summary 💬</h2>
<div class="charts-container">
<canvas id="commentsChart"></canvas>
</div>
</div>
</div>
</div>
<%= render "shared/stats" %>
</div>
<%= javascript_pack_tag "analyticsDashboard", defer: true %>

View file

@ -0,0 +1,59 @@
<div class="toggles">
<button class="selected timerange-button" id="week-button">Week</button>
<button class="timerange-button" id="month-button">Month</button>
<button class="timerange-button" id="infinity-button">Infinity</button>
</div>
<div class="summary-stats">
<div class="card" id="readers-card"></div>
<div class="card" id="reactions-card"></div>
<div class="card" id="comments-card"></div>
<div class="card" id="followers-card"></div>
</div>
<div class="graphs">
<div class="row">
<div class="card">
<h2>Readers Summary</h2>
<div class="charts-container">
<canvas id="readers-chart"></canvas>
</div>
</div>
<div class="card">
<h2>New Followers Summary</h2>
<div class="charts-container">
<canvas id="followers-chart"></canvas>
</div>
</div>
</div>
<div class="row">
<div class="card">
<h2>Reactions Summary ❤🦄🔖</h2>
<div class="charts-container">
<canvas id="reactions-chart"></canvas>
</div>
</div>
<div class="card">
<h2>Comments Summary 💬</h2>
<div class="charts-container">
<canvas id="comments-chart"></canvas>
</div>
</div>
</div>
<div class="row">
<div class="card">
<h2>Traffic Source Summary 🚦</h2>
<table>
<thead>
<tr>
<th>Source</th>
<th>Views</th>
</tr>
</thead>
<tbody id="referrers-container">
</tbody>
</table>
</div>
</div>
</div>

View file

@ -90,9 +90,11 @@ Rails.application.routes.draw do
post "/update_or_create", to: "github_repos#update_or_create"
end
end
get "/analytics/totals", to: "analytics#totals"
get "/analytics/historical", to: "analytics#historical"
get "/analytics/past_day", to: "analytics#past_day"
get "/analytics/referrers", to: "analytics#referrers"
end
end

View file

@ -0,0 +1,6 @@
class AddDomainAndPathToPageViews < ActiveRecord::Migration[5.2]
def change
add_column :page_views, :domain, :string
add_column :page_views, :path, :string
end
end

View file

@ -0,0 +1,7 @@
class AddIndexToPageViewDomain < ActiveRecord::Migration[5.2]
disable_ddl_transaction!
def change
add_index :page_views, :domain, algorithm: :concurrently
end
end

View file

@ -12,7 +12,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2019_06_19_153428) do
ActiveRecord::Schema.define(version: 2019_06_25_143841) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -598,6 +598,8 @@ ActiveRecord::Schema.define(version: 2019_06_19_153428) do
t.bigint "article_id"
t.integer "counts_for_number_of_views", default: 1
t.datetime "created_at", null: false
t.string "domain"
t.string "path"
t.string "referrer"
t.integer "time_tracked_in_seconds", default: 15
t.datetime "updated_at", null: false
@ -605,6 +607,7 @@ ActiveRecord::Schema.define(version: 2019_06_19_153428) do
t.bigint "user_id"
t.index ["article_id"], name: "index_page_views_on_article_id"
t.index ["created_at"], name: "index_page_views_on_created_at"
t.index ["domain"], name: "index_page_views_on_domain"
t.index ["user_id"], name: "index_page_views_on_user_id"
end

View file

View file

View file

@ -1,30 +0,0 @@
# namespace :db do
#
# desc "Copy production database to local"
#
# task :copy_production => :environment do
# puts "FIRING UP!"
# # Download latest dump
# system("echo hey")
# system("heroku pg:backups --remote heroku capture")
# system("curl -o latest.dump `heroku pg:backups --remote heroku public-url`")
#
#
# # get user and database name
# # config = Rails.configuration.database_configuration["development"]
# # database = config["database"]
# # user = config["username"]
# #
# # # import
# # system("pg_restore --verbose --clean --no-acl --no-owner -h localhost -d #{database} #{Rails.root}/tmp/latest.dump")
# end
#
# end
#
#
# heroku pg:backups --remote heroku capture
# curl -o latest.dump `heroku pg:backups --remote heroku public-url`
# rake db:reset
# pg_restore --verbose --no-acl --no-owner -t articles -t users -t podcasts -t podcast_episodes -t sponsors -t identities -t organizations -h localhost -d PracticalDeveloper_development latest.dump
# rake db:migrate
# pg_restore --verbose --clean --no-acl --no-owner -t articles -t users -t podcasts -t podcast_episodes -t sponsors -t identities -t organizations -h localhost -d PracticalDeveloper_development latest.dump

View file

@ -1,29 +0,0 @@
if %w[development test].include? Rails.env
namespace :lint do
desc "eslint"
task :eslint do
cmd = "cd client && npm run eslint . -- --ext .jsx,.js"
puts "Running eslint via `#{cmd}`"
sh cmd
end
desc "jscs"
task :jscs do
cmd = "cd client && npm run jscs ."
puts "Running jscs via `#{cmd}`"
sh cmd
end
desc "JS Linting"
task js: %i[eslint jscs] do
puts "Completed running all JavaScript Linters"
end
task lint: [:js] do
puts "Completed all linting"
end
end
desc "Runs all linters. Run `rake -D lint` to see all available lint options"
task lint: ["lint:lint"]
end

View file

@ -0,0 +1,23 @@
namespace :page_views do
desc "Update domain and path from page views referrers"
task update_domain_path: :environment do
puts "Going to update #{PageView.count} users"
ActiveRecord::Base.transaction do
PageView.find_each do |pv|
next unless pv.referrer
next if pv.domain # allows retries
begin
parsed_url = Addressable::URI.parse(pv.referrer)
pv.update!(domain: parsed_url.domain, path: parsed_url.path)
rescue StandardError => e
Rails.logger.error("#{pv.id}: #{e.message}")
next
end
end
end
puts "All done now!"
end
end

View file

@ -2,5 +2,6 @@ FactoryBot.define do
factory :page_view do
user
article
referrer { Faker::Internet.url }
end
end

View file

@ -1,6 +1,22 @@
require "rails_helper"
RSpec.describe PageView, type: :model do
let(:article) { create(:article) }
it { is_expected.to belong_to(:user).optional }
it { is_expected.to belong_to(:article) }
describe "#domain" do
it "is automatically set when a new page view is created" do
pv = create(:page_view, referrer: "http://example.com/page")
expect(pv.reload.domain).to eq("example.com")
end
end
describe "#path" do
it "is automatically set when a new page view is created" do
pv = create(:page_view, referrer: "http://example.com/page")
expect(pv.reload.path).to eq("/page")
end
end
end

View file

@ -38,4 +38,8 @@ RSpec.describe "Api::V0::Analytics", type: :request do
describe "GET /api/analytics/past_day" do
include_examples "GET /api/analytics/:endpoint authorization examples", "past_day"
end
describe "GET /api/analytics/referrers" do
include_examples "GET /api/analytics/:endpoint authorization examples", "referrers"
end
end

View file

@ -9,6 +9,11 @@ RSpec.describe AnalyticsService, type: :service do
after { Timecop.return }
def format_date(datetime)
# PostgreSQL DATE(..) function uses UTC.
datetime.utc.to_date.iso8601
end
describe "initialization" do
it "raises an error if start date is invalid" do
expect(-> { described_class.new(user, start_date: "2000-") }).to raise_error(ArgumentError)
@ -165,11 +170,6 @@ RSpec.describe AnalyticsService, type: :service do
end
describe "#grouped_by_day" do
def format_date(datetime)
# Postgre's DATE(..) method uses UTC.
datetime.utc.to_date.iso8601
end
it "returns stats grouped by day" do
stats = described_class.new(
user, start_date: "2019-04-01", end_date: "2019-04-04"
@ -370,4 +370,63 @@ RSpec.describe AnalyticsService, type: :service do
end
end
end
describe "#referrers" do
let(:analytics_service) { described_class.new(user) }
context "when working on domains" do
before { PageView.where(article: article).delete_all }
it "returns unique domains with a count" do
url = Faker::Internet.url
create(:page_view, user: user, article: article, referrer: url)
create(:page_view, user: user, article: article, referrer: url)
domains = analytics_service.referrers[:domains]
expect(domains.first).to eq(domain: Addressable::URI.parse(url).domain, count: 2)
end
it "returns unique domains with a count when there is a date range" do
url = Faker::Internet.url
create(:page_view, user: user, article: article, referrer: url)
create(:page_view, user: user, article: article, referrer: url)
date = format_date(article.created_at)
analytics_service = described_class.new(user, start_date: date)
domains = analytics_service.referrers[:domains]
expect(domains.first).to eq(domain: Addressable::URI.parse(url).domain, count: 2)
end
it "returns nothing if there is no data" do
expect(analytics_service.referrers[:domains]).to be_empty
end
it "returns the domains ordered by count" do
other_url = Faker::Internet.url
create_list(:page_view, 2, user: user, article: article, referrer: other_url)
top_url = Faker::Internet.url
create_list(:page_view, 3, user: user, article: article, referrer: top_url)
expected_result = [
{ domain: Addressable::URI.parse(top_url).domain, count: 3 },
{ domain: Addressable::URI.parse(other_url).domain, count: 2 },
]
expect(analytics_service.referrers[:domains]).to eq(expected_result)
end
it "returns 20 domains at most by default" do
21.times { create(:page_view, user: user, article: article, referrer: Faker::Internet.url) } # rubocop:disable FactoryBot/CreateList
expect(analytics_service.referrers[:domains].size).to eq(20)
end
it "returns the most visited domain if asked for only one result" do
top_url = Faker::Internet.url
create_list(:page_view, 3, user: user, article: article, referrer: top_url)
other_url = Faker::Internet.url
create_list(:page_view, 2, user: user, article: article, referrer: other_url)
top_domain = Addressable::URI.parse(top_url).domain
result = analytics_service.referrers(top: 1)[:domains]
expect(result).to eq([{ domain: top_domain, count: 3 }])
end
end
end
end