improve metrics gathered using in our traffic section

this also pulls out the middleware into its own home and inserts in front
This commit is contained in:
Sam 2015-02-05 16:08:52 +11:00
parent ebc2ad7c81
commit 08b790b3c2
14 changed files with 188 additions and 104 deletions

View File

@ -24,7 +24,8 @@ export default Discourse.Route.extend({
c.set('top_referrers', topReferrers);
}
['disk_space','admins', 'moderators', 'blocked', 'suspended', 'top_traffic_sources', 'top_referred_topics', 'updated_at'].forEach(function(x) {
[ 'disk_space','admins', 'moderators', 'blocked', 'suspended',
'top_traffic_sources', 'top_referred_topics', 'updated_at'].forEach(function(x) {
c.set(x, d[x]);
});

View File

@ -75,10 +75,15 @@
</tr>
</thead>
{{#unless loading}}
{{ render 'admin_report_counts' anon_reqs }}
{{ render 'admin_report_counts' logged_in_reqs }}
{{ render 'admin_report_counts' crawler_reqs}}
{{ render 'admin_report_counts' total_reqs}}
{{ render 'admin_report_counts' topic_anon_reqs }}
{{ render 'admin_report_counts' topic_logged_in_reqs }}
{{ render 'admin_report_counts' topic_crawler_reqs }}
{{ render 'admin_report_counts' background_reqs }}
{{ render 'admin_report_counts' success_reqs }}
{{ render 'admin_report_counts' redirect_reqs }}
{{ render 'admin_report_counts' server_error_reqs }}
{{ render 'admin_report_counts' client_error_reqs }}
{{ render 'admin_report_counts' total_reqs }}
{{/unless}}
</table>
</div>

View File

@ -16,12 +16,8 @@ class AdminDashboardData
'system_private_messages',
'moderator_warning_private_messages',
'notify_moderators_private_messages',
'notify_user_private_messages',
'anon_reqs',
'crawler_reqs',
'logged_in_reqs',
'total_reqs'
]
'notify_user_private_messages'
] + ApplicationRequest.req_types.keys.map{|r| r + "_reqs"}
def problems
[ rails_env_check,

View File

@ -1,9 +1,9 @@
class ApplicationRequest < ActiveRecord::Base
enum req_type: %i(anon logged_in crawler)
enum req_type: %i(total success background topic_anon topic_logged_in topic_crawler server_error client_error redirect)
cattr_accessor :autoflush
# auto flush if backlog is larger than this
self.autoflush = 100
self.autoflush = 200
def self.increment!(type, opts=nil)
key = redis_key(type)

View File

@ -33,28 +33,27 @@ class Report
def self.find(type, opts=nil)
opts ||= {}
report_method = :"report_#{type}"
return nil unless respond_to?(report_method)
# Load the report
report = Report.new(type)
report.start_date = opts[:start_date] if opts[:start_date]
report.end_date = opts[:end_date] if opts[:end_date]
report_method = :"report_#{type}"
if respond_to?(report_method)
send(report_method, report)
elsif type =~ /_reqs$/
req_report(report, type.split(/_reqs$/)[0].to_sym)
else
return nil
end
report
end
def self.req_report(report, filter=nil)
data = ApplicationRequest.all
if mapped = ApplicationRequest.req_types[filter]
data = data.where(req_type: mapped)
end
data = ApplicationRequest.where(req_type: ApplicationRequest.req_types[filter])
filtered_results = data.where('date >= ? AND date <= ?', report.start_date.to_date, report.end_date.to_date)
report.data = []
filtered_results.group(:date)
.sum(:count)
@ -67,21 +66,6 @@ class Report
report.prev30Days = filtered_results.sum(:count)
end
def self.report_anon_reqs(report)
req_report(report, :anon)
end
def self.report_crawler_reqs(report)
req_report(report, :crawler)
end
def self.report_logged_in_reqs(report)
req_report(report, :logged_in)
end
def self.report_total_reqs(report)
req_report(report)
end
def self.report_visits(report)
basic_report_about report, UserVisit, :by_day, report.start_date, report.end_date

View File

@ -0,0 +1,6 @@
# no reason to track this in development, that is 300+ redis calls saved per
# page view (we serve all assets out of thin in development)
if Rails.env != 'development'
require 'middleware/request_tracker'
Rails.configuration.middleware.unshift Middleware::RequestTracker
end

View File

@ -588,18 +588,38 @@ en:
title: "Top Referred Topics"
xaxis: "Topic"
num_clicks: "Clicks"
anon_reqs:
title: "Anonymous"
topic_anon_reqs:
title: "Anonymous Topics Views"
xaxis: "Day"
yaxis: "Anonymous application requests"
logged_in_reqs:
title: "Logged In"
yaxis: "Anonymous Topic Views"
topic_logged_in_reqs:
title: "Logged In Topic Views"
xaxis: "Day"
yaxis: "Logged In application requests"
crawler_reqs:
title: "Crawler"
yaxis: "Logged In Topic Views"
topic_crawler_reqs:
title: "Crawler Topic Views"
xaxis: "Day"
yaxis: "Crawler application requests"
yaxis: "Crawler Topic Views"
background_reqs:
title: "Background requests"
xaxis: "Day"
yaxis: "Requests used for live update and tracking"
success_reqs:
title: "Successful requests"
xaxis: "Day"
yaxis: "Successful requests (Status 2xx)"
redirect_reqs:
title: "Redirect requests"
xaxis: "Day"
yaxis: "Redirect requests (Status 3xx)"
server_error_reqs:
title: "Server Errors"
xaxis: "Day"
yaxis: "Server Errors (Status 5xx)"
client_error_reqs:
title: "Bad requests"
xaxis: "Day"
yaxis: "Client Errors (Status 4xx)"
total_reqs:
title: "Total"
xaxis: "Day"

View File

@ -0,0 +1,10 @@
class ResetApplicationRequests < ActiveRecord::Migration
def up
# enum changed we need to clear the data
execute 'TRUNCATE TABLE application_requests'
end
def down
end
end

View File

@ -114,29 +114,6 @@ module Middleware
@app = app
end
def self.log_request_on_site(env, helper=nil)
host = RailsMultisite::ConnectionManagement.host(env)
RailsMultisite::ConnectionManagement.with_hostname(host) do
log_request(env,helper)
end
end
def self.log_request(env,helper=nil)
helper ||= Helper.new(env)
type =
if helper.is_crawler?
:crawler
elsif helper.has_auth_cookie?
:logged_in
else
:anon
end
ApplicationRequest.increment!(type)
end
def call(env)
helper = Helper.new(env)
@ -146,10 +123,6 @@ module Middleware
@app.call(env)
end
ensure
Scheduler::Defer.later "Track view" do
self.class.log_request_on_site(env,helper)
end
end
end

View File

@ -0,0 +1,66 @@
require_dependency 'middleware/anonymous_cache'
class Middleware::RequestTracker
def initialize(app, settings={})
@app = app
end
def self.log_request_on_site(result, env, helper=nil)
host = RailsMultisite::ConnectionManagement.host(env)
RailsMultisite::ConnectionManagement.with_hostname(host) do
log_request(result,env,helper)
end
end
PATH_PARAMS = "action_dispatch.request.path_parameters".freeze
def self.log_request(result,env,helper=nil)
helper ||= Middleware::AnonymousCache::Helper.new(env)
params = env[PATH_PARAMS]
request = Rack::Request.new(env)
ApplicationRequest.increment!(:total)
status,_ = result
status = status.to_i
if status >= 500
ApplicationRequest.increment!(:server_error)
elsif status >= 400
ApplicationRequest.increment!(:client_error)
elsif status >= 300
ApplicationRequest.increment!(:redirect)
end
if request.path =~ /^\/message-bus\// || request.path == /\/topics\/timings/
ApplicationRequest.increment!(:background)
elsif status >= 200 && status < 300
ApplicationRequest.increment!(:success)
end
if params && params[:controller] == "topics" && params[:action] == "show"
if helper.is_crawler?
ApplicationRequest.increment!(:topic_crawler)
elsif helper.has_auth_cookie?
ApplicationRequest.increment!(:topic_logged_in)
else
ApplicationRequest.increment!(:topic_anon)
end
end
rescue => ex
Discourse.handle_exception(ex, {message: "Failed to log request"})
end
def call(env)
result = @app.call(env)
ensure
Scheduler::Defer.later("Track view", _db=nil) do
self.class.log_request_on_site(result,env)
end
end
end

View File

@ -14,10 +14,10 @@ module Scheduler
@async = val
end
def later(desc = nil, &blk)
def later(desc = nil, db=RailsMultisite::ConnectionManagement.current_db, &blk)
if @async
start_thread unless @thread.alive?
@queue << [RailsMultisite::ConnectionManagement.current_db, blk, desc]
@queue << [db, blk, desc]
else
blk.call
end
@ -48,7 +48,7 @@ module Scheduler
def do_work
db, job, desc = @queue.deq
begin
RailsMultisite::ConnectionManagement.establish_connection(db: db)
RailsMultisite::ConnectionManagement.establish_connection(db: db) if db
job.call
rescue => ex
Discourse.handle_exception(ex, {message: "Running deferred code '#{desc}'"})

View File

@ -16,21 +16,6 @@ describe Middleware::AnonymousCache::Helper do
Middleware::AnonymousCache::Helper.new(env(opts))
end
context "log_request" do
it "can log requests correctly" do
freeze_time Time.now
ApplicationRequest.clear_cache!
Middleware::AnonymousCache.log_request(env "HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)")
Middleware::AnonymousCache.log_request(env)
ApplicationRequest.write_cache!
ApplicationRequest.crawler.first.count.should == 1
ApplicationRequest.anon.first.count.should == 1
end
end
context "cachable?" do
it "true by default" do

View File

@ -0,0 +1,38 @@
require "spec_helper"
require_dependency "middleware/request_tracker"
describe Middleware::RequestTracker do
def env(opts={})
{
"HTTP_HOST" => "http://test.com",
"REQUEST_URI" => "/path?bla=1",
"REQUEST_METHOD" => "GET",
"rack.input" => ""
}.merge(opts)
end
context "log_request" do
it "can log requests correctly" do
freeze_time Time.now
ApplicationRequest.clear_cache!
Middleware::RequestTracker.log_request(["200"], env(
"HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)",
"action_dispatch.request.path_parameters" => {controller: "topics", action: "show"}
))
Middleware::RequestTracker.log_request(["200"], env(
"action_dispatch.request.path_parameters" => {controller: "topics", action: "show"}
))
ApplicationRequest.write_cache!
ApplicationRequest.total.first.count.should == 2
ApplicationRequest.success.first.count.should == 2
ApplicationRequest.topic_anon.first.count.should == 1
ApplicationRequest.topic_crawler.first.count.should == 1
end
end
end

View File

@ -18,9 +18,9 @@ describe ApplicationRequest do
it 'can automatically flush' do
t1 = Time.now.utc.at_midnight
freeze_time(t1)
inc(:anon)
inc(:anon)
inc(:anon, autoflush: 3)
inc(:total)
inc(:total)
inc(:total, autoflush: 3)
ApplicationRequest.first.count.should == 3
end
@ -28,9 +28,9 @@ describe ApplicationRequest do
it 'flushes yesterdays results' do
t1 = Time.now.utc.at_midnight
freeze_time(t1)
inc(:anon)
inc(:total)
freeze_time(t1.tomorrow)
inc(:anon)
inc(:total)
ApplicationRequest.write_cache!
ApplicationRequest.count.should == 2
@ -49,15 +49,15 @@ describe ApplicationRequest do
time = Time.now.at_midnight
freeze_time(time)
3.times { inc(:anon) }
2.times { inc(:logged_in) }
4.times { inc(:crawler) }
3.times { inc(:total) }
2.times { inc(:success) }
4.times { inc(:redirect) }
ApplicationRequest.write_cache!
ApplicationRequest.anon.first.count.should == 3
ApplicationRequest.logged_in.first.count.should == 2
ApplicationRequest.crawler.first.count.should == 4
ApplicationRequest.total.first.count.should == 3
ApplicationRequest.success.first.count.should == 2
ApplicationRequest.redirect.first.count.should == 4
end
end