infrustructure for tracking application web requests

app/models/application_request.rb

@@ -0,0 +1,86 @@
+class ApplicationRequest < ActiveRecord::Base
+  enum req_type: %i(anon logged_in crawler)
+
+  cattr_accessor :autoflush
+  # auto flush if backlog is larger than this
+  self.autoflush = 100
+
+  def self.increment!(type, opts=nil)
+    key = redis_key(type)
+    val = $redis.incr(key).to_i
+    $redis.expire key, 3.days
+
+    autoflush = (opts && opts[:autoflush]) || self.autoflush
+    if autoflush > 0 && val >= autoflush
+      write_cache!
+    end
+  end
+
+  def self.write_cache!(date=nil)
+    if date.nil?
+      write_cache!(Time.now.utc)
+      write_cache!(Time.now.utc.yesterday)
+      return
+    end
+
+    date = date.to_date
+
+    # this may seem a bit fancy but in so it allows
+    # for concurrent calls without double counting
+    req_types.each do |req_type,_|
+      key = redis_key(req_type,date)
+      val = $redis.get(key).to_i
+
+      next if val == 0
+
+      new_val = $redis.incrby(key, -val).to_i
+
+      if new_val < 0
+        # undo and flush next time
+        $redis.incrby(key, val)
+        next
+      end
+
+      id = req_id(date,req_type)
+
+      where(id: id).update_all(["count = count + ?", val])
+    end
+  end
+
+  def self.clear_cache!(date=nil)
+    if date.nil?
+      clear_cache!(Time.now.utc)
+      clear_cache!(Time.now.utc.yesterday)
+      return
+    end
+
+    req_types.each do |req_type,_|
+      key = redis_key(req_type,date)
+      $redis.del key
+    end
+  end
+
+  protected
+
+  def self.req_id(date,req_type,retries=0)
+
+    req_type_id = req_types[req_type]
+
+    # a poor man's upsert
+    id = where(date: date, req_type: req_type_id).pluck(:id).first
+    id ||= create!(date: date, req_type: req_type_id, count: 0).id
+
+  rescue # primary key violation
+    if retries == 0
+      req_id(date,req_type,1)
+    else
+      raise
+    end
+  end
+
+  def self.redis_key(req_type, time=Time.now.utc)
+    "app_req_#{req_type}#{time.strftime('%Y%m%d')}"
+  end
+
+end
+

db/migrate/20150203041207_add_application_requests.rb

@@ -0,0 +1,11 @@
+class AddApplicationRequests < ActiveRecord::Migration
+  def change
+    create_table :application_requests do |t|
+      t.date :date, null: false
+      t.integer :req_type, null: false
+      t.integer :count, null: false, default: 0
+    end
+
+    add_index :application_requests, [:date, :req_type], unique: true
+  end
+end

spec/models/application_request_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+
+describe ApplicationRequest do
+
+  before do
+    ApplicationRequest.clear_cache!
+  end
+
+  def inc(key,opts=nil)
+    ApplicationRequest.increment!(key,opts)
+  end
+
+  it 'logs nothing for an unflushed increment' do
+    ApplicationRequest.increment!(:anon)
+    ApplicationRequest.count.should == 0
+  end
+
+  it 'can automatically flush' do
+    t1 = Time.now.utc.at_midnight
+    freeze_time(t1)
+    inc(:anon)
+    inc(:anon)
+    inc(:anon, autoflush: 3)
+
+    ApplicationRequest.first.count.should == 3
+  end
+
+  it 'flushes yesterdays results' do
+    t1 = Time.now.utc.at_midnight
+    freeze_time(t1)
+    inc(:anon)
+    freeze_time(t1.tomorrow)
+    inc(:anon)
+
+    ApplicationRequest.write_cache!
+    ApplicationRequest.count.should == 2
+  end
+
+  it 'clears cache correctly' do
+    # otherwise we have test pollution
+    inc(:anon)
+    ApplicationRequest.clear_cache!
+    ApplicationRequest.write_cache!
+
+    ApplicationRequest.count.should == 0
+  end
+
+  it 'logs a few counts once flushed' do
+    time = Time.now.at_midnight
+    freeze_time(time)
+
+    3.times { inc(:anon) }
+    2.times { inc(:logged_in) }
+    4.times { inc(:crawler) }
+
+    ApplicationRequest.write_cache!
+
+    ApplicationRequest.anon.first.count.should == 3
+    ApplicationRequest.logged_in.first.count.should == 2
+    ApplicationRequest.crawler.first.count.should == 4
+
+  end
+end