improve metrics gathered using in our traffic section

this also pulls out the middleware into its own home and inserts in front

app/assets/javascripts/admin/routes/admin-dashboard.js.es6

@@ -24,7 +24,8 @@ export default Discourse.Route.extend({
           c.set('top_referrers', topReferrers);
         }
 
-        ['disk_space','admins', 'moderators', 'blocked', 'suspended', 'top_traffic_sources', 'top_referred_topics', 'updated_at'].forEach(function(x) {
+        [ 'disk_space','admins', 'moderators', 'blocked', 'suspended',
+          'top_traffic_sources', 'top_referred_topics', 'updated_at'].forEach(function(x) {
           c.set(x, d[x]);
         });
 

app/assets/javascripts/admin/templates/dashboard.hbs

@@ -75,10 +75,15 @@
         </tr>
       </thead>
       {{#unless loading}}
-        {{ render 'admin_report_counts' anon_reqs }}
+        {{ render 'admin_report_counts' topic_anon_reqs }}
-        {{ render 'admin_report_counts' logged_in_reqs }}
+        {{ render 'admin_report_counts' topic_logged_in_reqs }}
-        {{ render 'admin_report_counts' crawler_reqs}}
+        {{ render 'admin_report_counts' topic_crawler_reqs }}
-        {{ render 'admin_report_counts' total_reqs}}
+        {{ render 'admin_report_counts' background_reqs }}
+        {{ render 'admin_report_counts' success_reqs }}
+        {{ render 'admin_report_counts' redirect_reqs }}
+        {{ render 'admin_report_counts' server_error_reqs }}
+        {{ render 'admin_report_counts' client_error_reqs }}
+        {{ render 'admin_report_counts' total_reqs }}
       {{/unless}}
     </table>
   </div>

app/models/admin_dashboard_data.rb

@@ -16,12 +16,8 @@ class AdminDashboardData
     'system_private_messages',
     'moderator_warning_private_messages',
     'notify_moderators_private_messages',
-    'notify_user_private_messages',
+    'notify_user_private_messages'
-    'anon_reqs',
+  ] + ApplicationRequest.req_types.keys.map{|r| r + "_reqs"}
-    'crawler_reqs',
-    'logged_in_reqs',
-    'total_reqs'
-  ]
 
   def problems
     [ rails_env_check,

app/models/application_request.rb

@@ -1,9 +1,9 @@
 class ApplicationRequest < ActiveRecord::Base
-  enum req_type: %i(anon logged_in crawler)
+  enum req_type: %i(total success background topic_anon topic_logged_in topic_crawler server_error client_error redirect)
 
   cattr_accessor :autoflush
   # auto flush if backlog is larger than this
-  self.autoflush = 100
+  self.autoflush = 200
 
   def self.increment!(type, opts=nil)
     key = redis_key(type)

app/models/report.rb

@@ -33,28 +33,27 @@ class Report
 
   def self.find(type, opts=nil)
     opts ||= {}
-    report_method = :"report_#{type}"
-    return nil unless respond_to?(report_method)
-
     # Load the report
     report = Report.new(type)
-
     report.start_date = opts[:start_date] if opts[:start_date]
     report.end_date = opts[:end_date] if opts[:end_date]
-    send(report_method, report)
+    report_method = :"report_#{type}"
+
+    if respond_to?(report_method)
+      send(report_method, report)
+    elsif type =~ /_reqs$/
+      req_report(report, type.split(/_reqs$/)[0].to_sym)
+    else
+      return nil
+    end
     report
   end
 
   def self.req_report(report, filter=nil)
-    data = ApplicationRequest.all
+    data = ApplicationRequest.where(req_type:  ApplicationRequest.req_types[filter])
-
-    if mapped = ApplicationRequest.req_types[filter]
-      data = data.where(req_type: mapped)
-    end
 
     filtered_results = data.where('date >= ? AND date <= ?', report.start_date.to_date, report.end_date.to_date)
 
-
     report.data = []
     filtered_results.group(:date)
                     .sum(:count)
@@ -67,21 +66,6 @@ class Report
     report.prev30Days = filtered_results.sum(:count)
   end
 
-  def self.report_anon_reqs(report)
-    req_report(report, :anon)
-  end
-
-  def self.report_crawler_reqs(report)
-    req_report(report, :crawler)
-  end
-
-  def self.report_logged_in_reqs(report)
-    req_report(report, :logged_in)
-  end
-
-  def self.report_total_reqs(report)
-    req_report(report)
-  end
 
   def self.report_visits(report)
     basic_report_about report, UserVisit, :by_day, report.start_date, report.end_date

config/initializers/99-request_tracker.rb

@@ -0,0 +1,6 @@
+# no reason to track this in development, that is 300+ redis calls saved per
+# page view (we serve all assets out of thin in development)
+if Rails.env != 'development'
+  require 'middleware/request_tracker'
+  Rails.configuration.middleware.unshift Middleware::RequestTracker
+end

config/locales/server.en.yml

@@ -588,18 +588,38 @@ en:
       title: "Top Referred Topics"
       xaxis: "Topic"
       num_clicks: "Clicks"
-    anon_reqs:
+    topic_anon_reqs:
-      title: "Anonymous"
+      title: "Anonymous Topics Views"
       xaxis: "Day"
-      yaxis: "Anonymous application requests"
+      yaxis: "Anonymous Topic Views"
-    logged_in_reqs:
+    topic_logged_in_reqs:
-      title: "Logged In"
+      title: "Logged In Topic Views"
       xaxis: "Day"
-      yaxis: "Logged In application requests"
+      yaxis: "Logged In Topic Views"
-    crawler_reqs:
+    topic_crawler_reqs:
-      title: "Crawler"
+      title: "Crawler Topic Views"
       xaxis: "Day"
-      yaxis: "Crawler application requests"
+      yaxis: "Crawler Topic Views"
+    background_reqs:
+      title: "Background requests"
+      xaxis: "Day"
+      yaxis: "Requests used for live update and tracking"
+    success_reqs:
+      title: "Successful requests"
+      xaxis: "Day"
+      yaxis: "Successful requests (Status 2xx)"
+    redirect_reqs:
+      title: "Redirect requests"
+      xaxis: "Day"
+      yaxis: "Redirect requests (Status 3xx)"
+    server_error_reqs:
+      title: "Server Errors"
+      xaxis: "Day"
+      yaxis: "Server Errors (Status 5xx)"
+    client_error_reqs:
+      title: "Bad requests"
+      xaxis: "Day"
+      yaxis: "Client Errors (Status 4xx)"
     total_reqs:
       title: "Total"
       xaxis: "Day"

db/migrate/20150205032808_reset_application_requests.rb

@@ -0,0 +1,10 @@
+class ResetApplicationRequests < ActiveRecord::Migration
+
+  def up
+    # enum changed we need to clear the data
+    execute 'TRUNCATE TABLE application_requests'
+  end
+
+  def down
+  end
+end

lib/middleware/anonymous_cache.rb

@@ -114,29 +114,6 @@ module Middleware
       @app = app
     end
 
-    def self.log_request_on_site(env, helper=nil)
-      host = RailsMultisite::ConnectionManagement.host(env)
-      RailsMultisite::ConnectionManagement.with_hostname(host) do
-        log_request(env,helper)
-      end
-    end
-
-    def self.log_request(env,helper=nil)
-
-      helper ||= Helper.new(env)
-
-      type =
-        if helper.is_crawler?
-          :crawler
-        elsif helper.has_auth_cookie?
-          :logged_in
-        else
-          :anon
-        end
-
-      ApplicationRequest.increment!(type)
-    end
-
     def call(env)
       helper = Helper.new(env)
 
@@ -146,10 +123,6 @@ module Middleware
         @app.call(env)
       end
 
-    ensure
-      Scheduler::Defer.later "Track view" do
-        self.class.log_request_on_site(env,helper)
-      end
     end
 
   end

lib/middleware/request_tracker.rb

@@ -0,0 +1,66 @@
+require_dependency 'middleware/anonymous_cache'
+
+class Middleware::RequestTracker
+
+  def initialize(app, settings={})
+    @app = app
+  end
+
+  def self.log_request_on_site(result, env, helper=nil)
+    host = RailsMultisite::ConnectionManagement.host(env)
+    RailsMultisite::ConnectionManagement.with_hostname(host) do
+      log_request(result,env,helper)
+    end
+  end
+
+  PATH_PARAMS = "action_dispatch.request.path_parameters".freeze
+
+  def self.log_request(result,env,helper=nil)
+
+    helper ||= Middleware::AnonymousCache::Helper.new(env)
+    params = env[PATH_PARAMS]
+    request = Rack::Request.new(env)
+
+    ApplicationRequest.increment!(:total)
+
+    status,_ = result
+    status = status.to_i
+
+    if status >= 500
+      ApplicationRequest.increment!(:server_error)
+    elsif status >= 400
+      ApplicationRequest.increment!(:client_error)
+    elsif status >= 300
+      ApplicationRequest.increment!(:redirect)
+    end
+
+    if request.path =~ /^\/message-bus\// || request.path == /\/topics\/timings/
+      ApplicationRequest.increment!(:background)
+    elsif status >= 200 && status < 300
+      ApplicationRequest.increment!(:success)
+    end
+
+    if params && params[:controller] == "topics" && params[:action] == "show"
+      if helper.is_crawler?
+        ApplicationRequest.increment!(:topic_crawler)
+      elsif helper.has_auth_cookie?
+        ApplicationRequest.increment!(:topic_logged_in)
+      else
+        ApplicationRequest.increment!(:topic_anon)
+      end
+    end
+
+  rescue => ex
+    Discourse.handle_exception(ex, {message: "Failed to log request"})
+  end
+
+
+  def call(env)
+    result = @app.call(env)
+  ensure
+    Scheduler::Defer.later("Track view", _db=nil) do
+      self.class.log_request_on_site(result,env)
+    end
+  end
+
+end

lib/scheduler/defer.rb

@@ -14,10 +14,10 @@ module Scheduler
       @async = val
     end
 
-    def later(desc = nil, &blk)
+    def later(desc = nil, db=RailsMultisite::ConnectionManagement.current_db, &blk)
       if @async
         start_thread unless @thread.alive?
-        @queue << [RailsMultisite::ConnectionManagement.current_db, blk, desc]
+        @queue << [db, blk, desc]
       else
         blk.call
       end
@@ -48,7 +48,7 @@ module Scheduler
     def do_work
       db, job, desc = @queue.deq
       begin
-        RailsMultisite::ConnectionManagement.establish_connection(db: db)
+        RailsMultisite::ConnectionManagement.establish_connection(db: db) if db
         job.call
       rescue => ex
         Discourse.handle_exception(ex, {message: "Running deferred code '#{desc}'"})

spec/components/middleware/anonymous_cache_spec.rb

@@ -16,21 +16,6 @@ describe Middleware::AnonymousCache::Helper do
     Middleware::AnonymousCache::Helper.new(env(opts))
   end
 
-  context "log_request" do
-    it "can log requests correctly" do
-      freeze_time Time.now
-
-      ApplicationRequest.clear_cache!
-
-      Middleware::AnonymousCache.log_request(env "HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)")
-      Middleware::AnonymousCache.log_request(env)
-
-      ApplicationRequest.write_cache!
-
-      ApplicationRequest.crawler.first.count.should == 1
-      ApplicationRequest.anon.first.count.should == 1
-    end
-  end
 
   context "cachable?" do
     it "true by default" do

spec/components/middleware/request_tracker_spec.rb

@@ -0,0 +1,38 @@
+require "spec_helper"
+require_dependency "middleware/request_tracker"
+
+describe Middleware::RequestTracker do
+
+  def env(opts={})
+    {
+      "HTTP_HOST" => "http://test.com",
+      "REQUEST_URI" => "/path?bla=1",
+      "REQUEST_METHOD" => "GET",
+      "rack.input" => ""
+    }.merge(opts)
+  end
+
+  context "log_request" do
+    it "can log requests correctly" do
+      freeze_time Time.now
+
+      ApplicationRequest.clear_cache!
+
+      Middleware::RequestTracker.log_request(["200"], env(
+        "HTTP_USER_AGENT" => "AdsBot-Google (+http://www.google.com/adsbot.html)",
+        "action_dispatch.request.path_parameters" => {controller: "topics", action: "show"}
+      ))
+      Middleware::RequestTracker.log_request(["200"], env(
+        "action_dispatch.request.path_parameters" => {controller: "topics", action: "show"}
+      ))
+
+      ApplicationRequest.write_cache!
+
+      ApplicationRequest.total.first.count.should == 2
+      ApplicationRequest.success.first.count.should == 2
+
+      ApplicationRequest.topic_anon.first.count.should == 1
+      ApplicationRequest.topic_crawler.first.count.should == 1
+    end
+  end
+end

spec/models/application_request_spec.rb

@@ -18,9 +18,9 @@ describe ApplicationRequest do
   it 'can automatically flush' do
     t1 = Time.now.utc.at_midnight
     freeze_time(t1)
-    inc(:anon)
+    inc(:total)
-    inc(:anon)
+    inc(:total)
-    inc(:anon, autoflush: 3)
+    inc(:total, autoflush: 3)
 
     ApplicationRequest.first.count.should == 3
   end
@@ -28,9 +28,9 @@ describe ApplicationRequest do
   it 'flushes yesterdays results' do
     t1 = Time.now.utc.at_midnight
     freeze_time(t1)
-    inc(:anon)
+    inc(:total)
     freeze_time(t1.tomorrow)
-    inc(:anon)
+    inc(:total)
 
     ApplicationRequest.write_cache!
     ApplicationRequest.count.should == 2
@@ -49,15 +49,15 @@ describe ApplicationRequest do
     time = Time.now.at_midnight
     freeze_time(time)
 
-    3.times { inc(:anon) }
+    3.times { inc(:total) }
-    2.times { inc(:logged_in) }
+    2.times { inc(:success) }
-    4.times { inc(:crawler) }
+    4.times { inc(:redirect) }
 
     ApplicationRequest.write_cache!
 
-    ApplicationRequest.anon.first.count.should == 3
+    ApplicationRequest.total.first.count.should == 3
-    ApplicationRequest.logged_in.first.count.should == 2
+    ApplicationRequest.success.first.count.should == 2
-    ApplicationRequest.crawler.first.count.should == 4
+    ApplicationRequest.redirect.first.count.should == 4
 
   end
 end