diff --git a/app/assets/javascripts/admin/routes/admin-dashboard.js.es6 b/app/assets/javascripts/admin/routes/admin-dashboard.js.es6 index a54079349e0..bbe4c798453 100644 --- a/app/assets/javascripts/admin/routes/admin-dashboard.js.es6 +++ b/app/assets/javascripts/admin/routes/admin-dashboard.js.es6 @@ -24,7 +24,8 @@ export default Discourse.Route.extend({ c.set('top_referrers', topReferrers); } - ['disk_space','admins', 'moderators', 'blocked', 'suspended', 'top_traffic_sources', 'top_referred_topics', 'updated_at'].forEach(function(x) { + [ 'disk_space','admins', 'moderators', 'blocked', 'suspended', + 'top_traffic_sources', 'top_referred_topics', 'updated_at'].forEach(function(x) { c.set(x, d[x]); }); diff --git a/app/assets/javascripts/admin/templates/dashboard.hbs b/app/assets/javascripts/admin/templates/dashboard.hbs index 706af40595d..5feb34ce2b5 100644 --- a/app/assets/javascripts/admin/templates/dashboard.hbs +++ b/app/assets/javascripts/admin/templates/dashboard.hbs @@ -75,10 +75,15 @@ {{#unless loading}} - {{ render 'admin_report_counts' anon_reqs }} - {{ render 'admin_report_counts' logged_in_reqs }} - {{ render 'admin_report_counts' crawler_reqs}} - {{ render 'admin_report_counts' total_reqs}} + {{ render 'admin_report_counts' topic_anon_reqs }} + {{ render 'admin_report_counts' topic_logged_in_reqs }} + {{ render 'admin_report_counts' topic_crawler_reqs }} + {{ render 'admin_report_counts' background_reqs }} + {{ render 'admin_report_counts' success_reqs }} + {{ render 'admin_report_counts' redirect_reqs }} + {{ render 'admin_report_counts' server_error_reqs }} + {{ render 'admin_report_counts' client_error_reqs }} + {{ render 'admin_report_counts' total_reqs }} {{/unless}} diff --git a/app/models/admin_dashboard_data.rb b/app/models/admin_dashboard_data.rb index 8da27511536..aec71b03ae2 100644 --- a/app/models/admin_dashboard_data.rb +++ b/app/models/admin_dashboard_data.rb @@ -16,12 +16,8 @@ class AdminDashboardData 'system_private_messages', 'moderator_warning_private_messages', 'notify_moderators_private_messages', - 'notify_user_private_messages', - 'anon_reqs', - 'crawler_reqs', - 'logged_in_reqs', - 'total_reqs' - ] + 'notify_user_private_messages' + ] + ApplicationRequest.req_types.keys.map{|r| r + "_reqs"} def problems [ rails_env_check, diff --git a/app/models/application_request.rb b/app/models/application_request.rb index 24fa623c51a..dd7d0a18835 100644 --- a/app/models/application_request.rb +++ b/app/models/application_request.rb @@ -1,9 +1,9 @@ class ApplicationRequest < ActiveRecord::Base - enum req_type: %i(anon logged_in crawler) + enum req_type: %i(total success background topic_anon topic_logged_in topic_crawler server_error client_error redirect) cattr_accessor :autoflush # auto flush if backlog is larger than this - self.autoflush = 100 + self.autoflush = 200 def self.increment!(type, opts=nil) key = redis_key(type) diff --git a/app/models/report.rb b/app/models/report.rb index 7ce3e7fcbb7..848644c9f1e 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -33,28 +33,27 @@ class Report def self.find(type, opts=nil) opts ||= {} - report_method = :"report_#{type}" - return nil unless respond_to?(report_method) - # Load the report report = Report.new(type) - report.start_date = opts[:start_date] if opts[:start_date] report.end_date = opts[:end_date] if opts[:end_date] - send(report_method, report) + report_method = :"report_#{type}" + + if respond_to?(report_method) + send(report_method, report) + elsif type =~ /_reqs$/ + req_report(report, type.split(/_reqs$/)[0].to_sym) + else + return nil + end report end def self.req_report(report, filter=nil) - data = ApplicationRequest.all - - if mapped = ApplicationRequest.req_types[filter] - data = data.where(req_type: mapped) - end + data = ApplicationRequest.where(req_type: ApplicationRequest.req_types[filter]) filtered_results = data.where('date >= ? AND date <= ?', report.start_date.to_date, report.end_date.to_date) - report.data = [] filtered_results.group(:date) .sum(:count) @@ -67,21 +66,6 @@ class Report report.prev30Days = filtered_results.sum(:count) end - def self.report_anon_reqs(report) - req_report(report, :anon) - end - - def self.report_crawler_reqs(report) - req_report(report, :crawler) - end - - def self.report_logged_in_reqs(report) - req_report(report, :logged_in) - end - - def self.report_total_reqs(report) - req_report(report) - end def self.report_visits(report) basic_report_about report, UserVisit, :by_day, report.start_date, report.end_date diff --git a/config/initializers/99-request_tracker.rb b/config/initializers/99-request_tracker.rb new file mode 100644 index 00000000000..00674863395 --- /dev/null +++ b/config/initializers/99-request_tracker.rb @@ -0,0 +1,6 @@ +# no reason to track this in development, that is 300+ redis calls saved per +# page view (we serve all assets out of thin in development) +if Rails.env != 'development' + require 'middleware/request_tracker' + Rails.configuration.middleware.unshift Middleware::RequestTracker +end diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index eb4809cede2..ebc41aad8f1 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -588,18 +588,38 @@ en: title: "Top Referred Topics" xaxis: "Topic" num_clicks: "Clicks" - anon_reqs: - title: "Anonymous" + topic_anon_reqs: + title: "Anonymous Topics Views" xaxis: "Day" - yaxis: "Anonymous application requests" - logged_in_reqs: - title: "Logged In" + yaxis: "Anonymous Topic Views" + topic_logged_in_reqs: + title: "Logged In Topic Views" xaxis: "Day" - yaxis: "Logged In application requests" - crawler_reqs: - title: "Crawler" + yaxis: "Logged In Topic Views" + topic_crawler_reqs: + title: "Crawler Topic Views" xaxis: "Day" - yaxis: "Crawler application requests" + yaxis: "Crawler Topic Views" + background_reqs: + title: "Background requests" + xaxis: "Day" + yaxis: "Requests used for live update and tracking" + success_reqs: + title: "Successful requests" + xaxis: "Day" + yaxis: "Successful requests (Status 2xx)" + redirect_reqs: + title: "Redirect requests" + xaxis: "Day" + yaxis: "Redirect requests (Status 3xx)" + server_error_reqs: + title: "Server Errors" + xaxis: "Day" + yaxis: "Server Errors (Status 5xx)" + client_error_reqs: + title: "Bad requests" + xaxis: "Day" + yaxis: "Client Errors (Status 4xx)" total_reqs: title: "Total" xaxis: "Day" diff --git a/db/migrate/20150205032808_reset_application_requests.rb b/db/migrate/20150205032808_reset_application_requests.rb new file mode 100644 index 00000000000..a9c8f039f3e --- /dev/null +++ b/db/migrate/20150205032808_reset_application_requests.rb @@ -0,0 +1,10 @@ +class ResetApplicationRequests < ActiveRecord::Migration + + def up + # enum changed we need to clear the data + execute 'TRUNCATE TABLE application_requests' + end + + def down + end +end diff --git a/lib/middleware/anonymous_cache.rb b/lib/middleware/anonymous_cache.rb index 92a9143a073..d315c53a50b 100644 --- a/lib/middleware/anonymous_cache.rb +++ b/lib/middleware/anonymous_cache.rb @@ -114,29 +114,6 @@ module Middleware @app = app end - def self.log_request_on_site(env, helper=nil) - host = RailsMultisite::ConnectionManagement.host(env) - RailsMultisite::ConnectionManagement.with_hostname(host) do - log_request(env,helper) - end - end - - def self.log_request(env,helper=nil) - - helper ||= Helper.new(env) - - type = - if helper.is_crawler? - :crawler - elsif helper.has_auth_cookie? - :logged_in - else - :anon - end - - ApplicationRequest.increment!(type) - end - def call(env) helper = Helper.new(env) @@ -146,10 +123,6 @@ module Middleware @app.call(env) end - ensure - Scheduler::Defer.later "Track view" do - self.class.log_request_on_site(env,helper) - end end end diff --git a/lib/middleware/request_tracker.rb b/lib/middleware/request_tracker.rb new file mode 100644 index 00000000000..b300bb20c4d --- /dev/null +++ b/lib/middleware/request_tracker.rb @@ -0,0 +1,66 @@ +require_dependency 'middleware/anonymous_cache' + +class Middleware::RequestTracker + + def initialize(app, settings={}) + @app = app + end + + def self.log_request_on_site(result, env, helper=nil) + host = RailsMultisite::ConnectionManagement.host(env) + RailsMultisite::ConnectionManagement.with_hostname(host) do + log_request(result,env,helper) + end + end + + PATH_PARAMS = "action_dispatch.request.path_parameters".freeze + + def self.log_request(result,env,helper=nil) + + helper ||= Middleware::AnonymousCache::Helper.new(env) + params = env[PATH_PARAMS] + request = Rack::Request.new(env) + + ApplicationRequest.increment!(:total) + + status,_ = result + status = status.to_i + + if status >= 500 + ApplicationRequest.increment!(:server_error) + elsif status >= 400 + ApplicationRequest.increment!(:client_error) + elsif status >= 300 + ApplicationRequest.increment!(:redirect) + end + + if request.path =~ /^\/message-bus\// || request.path == /\/topics\/timings/ + ApplicationRequest.increment!(:background) + elsif status >= 200 && status < 300 + ApplicationRequest.increment!(:success) + end + + if params && params[:controller] == "topics" && params[:action] == "show" + if helper.is_crawler? + ApplicationRequest.increment!(:topic_crawler) + elsif helper.has_auth_cookie? + ApplicationRequest.increment!(:topic_logged_in) + else + ApplicationRequest.increment!(:topic_anon) + end + end + + rescue => ex + Discourse.handle_exception(ex, {message: "Failed to log request"}) + end + + + def call(env) + result = @app.call(env) + ensure + Scheduler::Defer.later("Track view", _db=nil) do + self.class.log_request_on_site(result,env) + end + end + +end diff --git a/lib/scheduler/defer.rb b/lib/scheduler/defer.rb index 89522f08abc..804deb62aa6 100644 --- a/lib/scheduler/defer.rb +++ b/lib/scheduler/defer.rb @@ -14,10 +14,10 @@ module Scheduler @async = val end - def later(desc = nil, &blk) + def later(desc = nil, db=RailsMultisite::ConnectionManagement.current_db, &blk) if @async start_thread unless @thread.alive? - @queue << [RailsMultisite::ConnectionManagement.current_db, blk, desc] + @queue << [db, blk, desc] else blk.call end @@ -48,7 +48,7 @@ module Scheduler def do_work db, job, desc = @queue.deq begin - RailsMultisite::ConnectionManagement.establish_connection(db: db) + RailsMultisite::ConnectionManagement.establish_connection(db: db) if db job.call rescue => ex Discourse.handle_exception(ex, {message: "Running deferred code '#{desc}'"}) diff --git a/spec/components/middleware/anonymous_cache_spec.rb b/spec/components/middleware/anonymous_cache_spec.rb index 8a9a77d96a1..b562f344b7f 100644 --- a/spec/components/middleware/anonymous_cache_spec.rb +++ b/spec/components/middleware/anonymous_cache_spec.rb @@ -16,21 +16,6 @@ describe Middleware::AnonymousCache::Helper do Middleware::AnonymousCache::Helper.new(env(opts)) end - context "log_request" do - it "can log requests correctly" do - freeze_time Time.now - - ApplicationRequest.clear_cache! - - Middleware::AnonymousCache.log_request(env "HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)") - Middleware::AnonymousCache.log_request(env) - - ApplicationRequest.write_cache! - - ApplicationRequest.crawler.first.count.should == 1 - ApplicationRequest.anon.first.count.should == 1 - end - end context "cachable?" do it "true by default" do diff --git a/spec/components/middleware/request_tracker_spec.rb b/spec/components/middleware/request_tracker_spec.rb new file mode 100644 index 00000000000..bbf0e817ddc --- /dev/null +++ b/spec/components/middleware/request_tracker_spec.rb @@ -0,0 +1,38 @@ +require "spec_helper" +require_dependency "middleware/request_tracker" + +describe Middleware::RequestTracker do + + def env(opts={}) + { + "HTTP_HOST" => "http://test.com", + "REQUEST_URI" => "/path?bla=1", + "REQUEST_METHOD" => "GET", + "rack.input" => "" + }.merge(opts) + end + + context "log_request" do + it "can log requests correctly" do + freeze_time Time.now + + ApplicationRequest.clear_cache! + + Middleware::RequestTracker.log_request(["200"], env( + "HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)", + "action_dispatch.request.path_parameters" => {controller: "topics", action: "show"} + )) + Middleware::RequestTracker.log_request(["200"], env( + "action_dispatch.request.path_parameters" => {controller: "topics", action: "show"} + )) + + ApplicationRequest.write_cache! + + ApplicationRequest.total.first.count.should == 2 + ApplicationRequest.success.first.count.should == 2 + + ApplicationRequest.topic_anon.first.count.should == 1 + ApplicationRequest.topic_crawler.first.count.should == 1 + end + end +end diff --git a/spec/models/application_request_spec.rb b/spec/models/application_request_spec.rb index 67bc4247e90..9f823fe89a2 100644 --- a/spec/models/application_request_spec.rb +++ b/spec/models/application_request_spec.rb @@ -18,9 +18,9 @@ describe ApplicationRequest do it 'can automatically flush' do t1 = Time.now.utc.at_midnight freeze_time(t1) - inc(:anon) - inc(:anon) - inc(:anon, autoflush: 3) + inc(:total) + inc(:total) + inc(:total, autoflush: 3) ApplicationRequest.first.count.should == 3 end @@ -28,9 +28,9 @@ describe ApplicationRequest do it 'flushes yesterdays results' do t1 = Time.now.utc.at_midnight freeze_time(t1) - inc(:anon) + inc(:total) freeze_time(t1.tomorrow) - inc(:anon) + inc(:total) ApplicationRequest.write_cache! ApplicationRequest.count.should == 2 @@ -49,15 +49,15 @@ describe ApplicationRequest do time = Time.now.at_midnight freeze_time(time) - 3.times { inc(:anon) } - 2.times { inc(:logged_in) } - 4.times { inc(:crawler) } + 3.times { inc(:total) } + 2.times { inc(:success) } + 4.times { inc(:redirect) } ApplicationRequest.write_cache! - ApplicationRequest.anon.first.count.should == 3 - ApplicationRequest.logged_in.first.count.should == 2 - ApplicationRequest.crawler.first.count.should == 4 + ApplicationRequest.total.first.count.should == 3 + ApplicationRequest.success.first.count.should == 2 + ApplicationRequest.redirect.first.count.should == 4 end end