improve metrics gathered using in our traffic section
this also pulls out the middleware into its own home and inserts in front
This commit is contained in:
parent
ebc2ad7c81
commit
08b790b3c2
|
@ -24,7 +24,8 @@ export default Discourse.Route.extend({
|
|||
c.set('top_referrers', topReferrers);
|
||||
}
|
||||
|
||||
['disk_space','admins', 'moderators', 'blocked', 'suspended', 'top_traffic_sources', 'top_referred_topics', 'updated_at'].forEach(function(x) {
|
||||
[ 'disk_space','admins', 'moderators', 'blocked', 'suspended',
|
||||
'top_traffic_sources', 'top_referred_topics', 'updated_at'].forEach(function(x) {
|
||||
c.set(x, d[x]);
|
||||
});
|
||||
|
||||
|
|
|
@ -75,10 +75,15 @@
|
|||
</tr>
|
||||
</thead>
|
||||
{{#unless loading}}
|
||||
{{ render 'admin_report_counts' anon_reqs }}
|
||||
{{ render 'admin_report_counts' logged_in_reqs }}
|
||||
{{ render 'admin_report_counts' crawler_reqs}}
|
||||
{{ render 'admin_report_counts' total_reqs}}
|
||||
{{ render 'admin_report_counts' topic_anon_reqs }}
|
||||
{{ render 'admin_report_counts' topic_logged_in_reqs }}
|
||||
{{ render 'admin_report_counts' topic_crawler_reqs }}
|
||||
{{ render 'admin_report_counts' background_reqs }}
|
||||
{{ render 'admin_report_counts' success_reqs }}
|
||||
{{ render 'admin_report_counts' redirect_reqs }}
|
||||
{{ render 'admin_report_counts' server_error_reqs }}
|
||||
{{ render 'admin_report_counts' client_error_reqs }}
|
||||
{{ render 'admin_report_counts' total_reqs }}
|
||||
{{/unless}}
|
||||
</table>
|
||||
</div>
|
||||
|
|
|
@ -16,12 +16,8 @@ class AdminDashboardData
|
|||
'system_private_messages',
|
||||
'moderator_warning_private_messages',
|
||||
'notify_moderators_private_messages',
|
||||
'notify_user_private_messages',
|
||||
'anon_reqs',
|
||||
'crawler_reqs',
|
||||
'logged_in_reqs',
|
||||
'total_reqs'
|
||||
]
|
||||
'notify_user_private_messages'
|
||||
] + ApplicationRequest.req_types.keys.map{|r| r + "_reqs"}
|
||||
|
||||
def problems
|
||||
[ rails_env_check,
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
class ApplicationRequest < ActiveRecord::Base
|
||||
enum req_type: %i(anon logged_in crawler)
|
||||
enum req_type: %i(total success background topic_anon topic_logged_in topic_crawler server_error client_error redirect)
|
||||
|
||||
cattr_accessor :autoflush
|
||||
# auto flush if backlog is larger than this
|
||||
self.autoflush = 100
|
||||
self.autoflush = 200
|
||||
|
||||
def self.increment!(type, opts=nil)
|
||||
key = redis_key(type)
|
||||
|
|
|
@ -33,28 +33,27 @@ class Report
|
|||
|
||||
def self.find(type, opts=nil)
|
||||
opts ||= {}
|
||||
report_method = :"report_#{type}"
|
||||
return nil unless respond_to?(report_method)
|
||||
|
||||
# Load the report
|
||||
report = Report.new(type)
|
||||
|
||||
report.start_date = opts[:start_date] if opts[:start_date]
|
||||
report.end_date = opts[:end_date] if opts[:end_date]
|
||||
report_method = :"report_#{type}"
|
||||
|
||||
if respond_to?(report_method)
|
||||
send(report_method, report)
|
||||
elsif type =~ /_reqs$/
|
||||
req_report(report, type.split(/_reqs$/)[0].to_sym)
|
||||
else
|
||||
return nil
|
||||
end
|
||||
report
|
||||
end
|
||||
|
||||
def self.req_report(report, filter=nil)
|
||||
data = ApplicationRequest.all
|
||||
|
||||
if mapped = ApplicationRequest.req_types[filter]
|
||||
data = data.where(req_type: mapped)
|
||||
end
|
||||
data = ApplicationRequest.where(req_type: ApplicationRequest.req_types[filter])
|
||||
|
||||
filtered_results = data.where('date >= ? AND date <= ?', report.start_date.to_date, report.end_date.to_date)
|
||||
|
||||
|
||||
report.data = []
|
||||
filtered_results.group(:date)
|
||||
.sum(:count)
|
||||
|
@ -67,21 +66,6 @@ class Report
|
|||
report.prev30Days = filtered_results.sum(:count)
|
||||
end
|
||||
|
||||
def self.report_anon_reqs(report)
|
||||
req_report(report, :anon)
|
||||
end
|
||||
|
||||
def self.report_crawler_reqs(report)
|
||||
req_report(report, :crawler)
|
||||
end
|
||||
|
||||
def self.report_logged_in_reqs(report)
|
||||
req_report(report, :logged_in)
|
||||
end
|
||||
|
||||
def self.report_total_reqs(report)
|
||||
req_report(report)
|
||||
end
|
||||
|
||||
def self.report_visits(report)
|
||||
basic_report_about report, UserVisit, :by_day, report.start_date, report.end_date
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
# no reason to track this in development, that is 300+ redis calls saved per
|
||||
# page view (we serve all assets out of thin in development)
|
||||
if Rails.env != 'development'
|
||||
require 'middleware/request_tracker'
|
||||
Rails.configuration.middleware.unshift Middleware::RequestTracker
|
||||
end
|
|
@ -588,18 +588,38 @@ en:
|
|||
title: "Top Referred Topics"
|
||||
xaxis: "Topic"
|
||||
num_clicks: "Clicks"
|
||||
anon_reqs:
|
||||
title: "Anonymous"
|
||||
topic_anon_reqs:
|
||||
title: "Anonymous Topics Views"
|
||||
xaxis: "Day"
|
||||
yaxis: "Anonymous application requests"
|
||||
logged_in_reqs:
|
||||
title: "Logged In"
|
||||
yaxis: "Anonymous Topic Views"
|
||||
topic_logged_in_reqs:
|
||||
title: "Logged In Topic Views"
|
||||
xaxis: "Day"
|
||||
yaxis: "Logged In application requests"
|
||||
crawler_reqs:
|
||||
title: "Crawler"
|
||||
yaxis: "Logged In Topic Views"
|
||||
topic_crawler_reqs:
|
||||
title: "Crawler Topic Views"
|
||||
xaxis: "Day"
|
||||
yaxis: "Crawler application requests"
|
||||
yaxis: "Crawler Topic Views"
|
||||
background_reqs:
|
||||
title: "Background requests"
|
||||
xaxis: "Day"
|
||||
yaxis: "Requests used for live update and tracking"
|
||||
success_reqs:
|
||||
title: "Successful requests"
|
||||
xaxis: "Day"
|
||||
yaxis: "Successful requests (Status 2xx)"
|
||||
redirect_reqs:
|
||||
title: "Redirect requests"
|
||||
xaxis: "Day"
|
||||
yaxis: "Redirect requests (Status 3xx)"
|
||||
server_error_reqs:
|
||||
title: "Server Errors"
|
||||
xaxis: "Day"
|
||||
yaxis: "Server Errors (Status 5xx)"
|
||||
client_error_reqs:
|
||||
title: "Bad requests"
|
||||
xaxis: "Day"
|
||||
yaxis: "Client Errors (Status 4xx)"
|
||||
total_reqs:
|
||||
title: "Total"
|
||||
xaxis: "Day"
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
class ResetApplicationRequests < ActiveRecord::Migration
|
||||
|
||||
def up
|
||||
# enum changed we need to clear the data
|
||||
execute 'TRUNCATE TABLE application_requests'
|
||||
end
|
||||
|
||||
def down
|
||||
end
|
||||
end
|
|
@ -114,29 +114,6 @@ module Middleware
|
|||
@app = app
|
||||
end
|
||||
|
||||
def self.log_request_on_site(env, helper=nil)
|
||||
host = RailsMultisite::ConnectionManagement.host(env)
|
||||
RailsMultisite::ConnectionManagement.with_hostname(host) do
|
||||
log_request(env,helper)
|
||||
end
|
||||
end
|
||||
|
||||
def self.log_request(env,helper=nil)
|
||||
|
||||
helper ||= Helper.new(env)
|
||||
|
||||
type =
|
||||
if helper.is_crawler?
|
||||
:crawler
|
||||
elsif helper.has_auth_cookie?
|
||||
:logged_in
|
||||
else
|
||||
:anon
|
||||
end
|
||||
|
||||
ApplicationRequest.increment!(type)
|
||||
end
|
||||
|
||||
def call(env)
|
||||
helper = Helper.new(env)
|
||||
|
||||
|
@ -146,10 +123,6 @@ module Middleware
|
|||
@app.call(env)
|
||||
end
|
||||
|
||||
ensure
|
||||
Scheduler::Defer.later "Track view" do
|
||||
self.class.log_request_on_site(env,helper)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
|
|
@ -0,0 +1,66 @@
|
|||
require_dependency 'middleware/anonymous_cache'
|
||||
|
||||
class Middleware::RequestTracker
|
||||
|
||||
def initialize(app, settings={})
|
||||
@app = app
|
||||
end
|
||||
|
||||
def self.log_request_on_site(result, env, helper=nil)
|
||||
host = RailsMultisite::ConnectionManagement.host(env)
|
||||
RailsMultisite::ConnectionManagement.with_hostname(host) do
|
||||
log_request(result,env,helper)
|
||||
end
|
||||
end
|
||||
|
||||
PATH_PARAMS = "action_dispatch.request.path_parameters".freeze
|
||||
|
||||
def self.log_request(result,env,helper=nil)
|
||||
|
||||
helper ||= Middleware::AnonymousCache::Helper.new(env)
|
||||
params = env[PATH_PARAMS]
|
||||
request = Rack::Request.new(env)
|
||||
|
||||
ApplicationRequest.increment!(:total)
|
||||
|
||||
status,_ = result
|
||||
status = status.to_i
|
||||
|
||||
if status >= 500
|
||||
ApplicationRequest.increment!(:server_error)
|
||||
elsif status >= 400
|
||||
ApplicationRequest.increment!(:client_error)
|
||||
elsif status >= 300
|
||||
ApplicationRequest.increment!(:redirect)
|
||||
end
|
||||
|
||||
if request.path =~ /^\/message-bus\// || request.path == /\/topics\/timings/
|
||||
ApplicationRequest.increment!(:background)
|
||||
elsif status >= 200 && status < 300
|
||||
ApplicationRequest.increment!(:success)
|
||||
end
|
||||
|
||||
if params && params[:controller] == "topics" && params[:action] == "show"
|
||||
if helper.is_crawler?
|
||||
ApplicationRequest.increment!(:topic_crawler)
|
||||
elsif helper.has_auth_cookie?
|
||||
ApplicationRequest.increment!(:topic_logged_in)
|
||||
else
|
||||
ApplicationRequest.increment!(:topic_anon)
|
||||
end
|
||||
end
|
||||
|
||||
rescue => ex
|
||||
Discourse.handle_exception(ex, {message: "Failed to log request"})
|
||||
end
|
||||
|
||||
|
||||
def call(env)
|
||||
result = @app.call(env)
|
||||
ensure
|
||||
Scheduler::Defer.later("Track view", _db=nil) do
|
||||
self.class.log_request_on_site(result,env)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
|
@ -14,10 +14,10 @@ module Scheduler
|
|||
@async = val
|
||||
end
|
||||
|
||||
def later(desc = nil, &blk)
|
||||
def later(desc = nil, db=RailsMultisite::ConnectionManagement.current_db, &blk)
|
||||
if @async
|
||||
start_thread unless @thread.alive?
|
||||
@queue << [RailsMultisite::ConnectionManagement.current_db, blk, desc]
|
||||
@queue << [db, blk, desc]
|
||||
else
|
||||
blk.call
|
||||
end
|
||||
|
@ -48,7 +48,7 @@ module Scheduler
|
|||
def do_work
|
||||
db, job, desc = @queue.deq
|
||||
begin
|
||||
RailsMultisite::ConnectionManagement.establish_connection(db: db)
|
||||
RailsMultisite::ConnectionManagement.establish_connection(db: db) if db
|
||||
job.call
|
||||
rescue => ex
|
||||
Discourse.handle_exception(ex, {message: "Running deferred code '#{desc}'"})
|
||||
|
|
|
@ -16,21 +16,6 @@ describe Middleware::AnonymousCache::Helper do
|
|||
Middleware::AnonymousCache::Helper.new(env(opts))
|
||||
end
|
||||
|
||||
context "log_request" do
|
||||
it "can log requests correctly" do
|
||||
freeze_time Time.now
|
||||
|
||||
ApplicationRequest.clear_cache!
|
||||
|
||||
Middleware::AnonymousCache.log_request(env "HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)")
|
||||
Middleware::AnonymousCache.log_request(env)
|
||||
|
||||
ApplicationRequest.write_cache!
|
||||
|
||||
ApplicationRequest.crawler.first.count.should == 1
|
||||
ApplicationRequest.anon.first.count.should == 1
|
||||
end
|
||||
end
|
||||
|
||||
context "cachable?" do
|
||||
it "true by default" do
|
||||
|
|
|
@ -0,0 +1,38 @@
|
|||
require "spec_helper"
|
||||
require_dependency "middleware/request_tracker"
|
||||
|
||||
describe Middleware::RequestTracker do
|
||||
|
||||
def env(opts={})
|
||||
{
|
||||
"HTTP_HOST" => "http://test.com",
|
||||
"REQUEST_URI" => "/path?bla=1",
|
||||
"REQUEST_METHOD" => "GET",
|
||||
"rack.input" => ""
|
||||
}.merge(opts)
|
||||
end
|
||||
|
||||
context "log_request" do
|
||||
it "can log requests correctly" do
|
||||
freeze_time Time.now
|
||||
|
||||
ApplicationRequest.clear_cache!
|
||||
|
||||
Middleware::RequestTracker.log_request(["200"], env(
|
||||
"HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)",
|
||||
"action_dispatch.request.path_parameters" => {controller: "topics", action: "show"}
|
||||
))
|
||||
Middleware::RequestTracker.log_request(["200"], env(
|
||||
"action_dispatch.request.path_parameters" => {controller: "topics", action: "show"}
|
||||
))
|
||||
|
||||
ApplicationRequest.write_cache!
|
||||
|
||||
ApplicationRequest.total.first.count.should == 2
|
||||
ApplicationRequest.success.first.count.should == 2
|
||||
|
||||
ApplicationRequest.topic_anon.first.count.should == 1
|
||||
ApplicationRequest.topic_crawler.first.count.should == 1
|
||||
end
|
||||
end
|
||||
end
|
|
@ -18,9 +18,9 @@ describe ApplicationRequest do
|
|||
it 'can automatically flush' do
|
||||
t1 = Time.now.utc.at_midnight
|
||||
freeze_time(t1)
|
||||
inc(:anon)
|
||||
inc(:anon)
|
||||
inc(:anon, autoflush: 3)
|
||||
inc(:total)
|
||||
inc(:total)
|
||||
inc(:total, autoflush: 3)
|
||||
|
||||
ApplicationRequest.first.count.should == 3
|
||||
end
|
||||
|
@ -28,9 +28,9 @@ describe ApplicationRequest do
|
|||
it 'flushes yesterdays results' do
|
||||
t1 = Time.now.utc.at_midnight
|
||||
freeze_time(t1)
|
||||
inc(:anon)
|
||||
inc(:total)
|
||||
freeze_time(t1.tomorrow)
|
||||
inc(:anon)
|
||||
inc(:total)
|
||||
|
||||
ApplicationRequest.write_cache!
|
||||
ApplicationRequest.count.should == 2
|
||||
|
@ -49,15 +49,15 @@ describe ApplicationRequest do
|
|||
time = Time.now.at_midnight
|
||||
freeze_time(time)
|
||||
|
||||
3.times { inc(:anon) }
|
||||
2.times { inc(:logged_in) }
|
||||
4.times { inc(:crawler) }
|
||||
3.times { inc(:total) }
|
||||
2.times { inc(:success) }
|
||||
4.times { inc(:redirect) }
|
||||
|
||||
ApplicationRequest.write_cache!
|
||||
|
||||
ApplicationRequest.anon.first.count.should == 3
|
||||
ApplicationRequest.logged_in.first.count.should == 2
|
||||
ApplicationRequest.crawler.first.count.should == 4
|
||||
ApplicationRequest.total.first.count.should == 3
|
||||
ApplicationRequest.success.first.count.should == 2
|
||||
ApplicationRequest.redirect.first.count.should == 4
|
||||
|
||||
end
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue