289 lines
7.3 KiB
Ruby
289 lines
7.3 KiB
Ruby
require 'date'
|
|
|
|
class LogAnalyzer
|
|
|
|
class LineParser
|
|
|
|
|
|
# log_format log_discourse '[$time_local] $remote_addr "$request" "$http_user_agent" "$sent_http_x_discourse_route" $status $bytes_sent "$http_referer" $upstream_response_time $request_time "$sent_http_x_discourse_username"';
|
|
|
|
attr_accessor :time, :ip_address, :url, :route, :user_agent, :rails_duration, :total_duration,
|
|
:username, :status, :bytes_sent, :referer
|
|
|
|
PATTERN = /\[(.*)\] (\S+) \"(.*)\" \"(.*)\" \"(.*)\" ([0-9]+) ([0-9]+) \"(.*)\" ([0-9.]+) ([0-9.]+) "(.*)"/
|
|
|
|
TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %Z"
|
|
|
|
def self.parse(line)
|
|
|
|
result = new
|
|
_, result.time, result.ip_address, result.url, result.user_agent,
|
|
result.route, result.status, result.bytes_sent, result.referer,
|
|
result.rails_duration, result.total_duration, result.username = line.match(PATTERN).to_a
|
|
|
|
result.rails_duration = result.rails_duration.to_f
|
|
result.total_duration = result.total_duration.to_f
|
|
|
|
verb = result.url[0..3] if result.url
|
|
if verb && verb == "POST"
|
|
result.route += " (POST)"
|
|
end
|
|
|
|
if verb && verb == "PUT"
|
|
result.route += " (PUT)"
|
|
end
|
|
|
|
result
|
|
end
|
|
|
|
def is_mobile?
|
|
user_agent =~ /Mobile|Android|webOS/ && !(user_agent =~ /iPad|Nexus (7|10)/)
|
|
end
|
|
|
|
def parsed_time
|
|
DateTime.strptime(time, TIME_FORMAT) if time
|
|
end
|
|
end
|
|
|
|
attr_reader :total_requests, :message_bus_requests, :filenames,
|
|
:ip_to_rails_duration, :username_to_rails_duration,
|
|
:route_to_rails_duration, :url_to_rails_duration,
|
|
:status_404_to_count, :from_time, :to_time
|
|
|
|
def self.analyze(filenames, args)
|
|
new(filenames, args).analyze
|
|
end
|
|
|
|
class Aggeregator
|
|
|
|
attr_accessor :aggregate_type
|
|
|
|
def initialize
|
|
@data = {}
|
|
@aggregate_type = :duration
|
|
end
|
|
|
|
def add(id, duration, aggregate=nil)
|
|
ary = (@data[id] ||= [0,0])
|
|
ary[0] += duration
|
|
ary[1] += 1
|
|
unless aggregate.nil?
|
|
ary[2] ||= Hash.new(0)
|
|
if @aggregate_type == :duration
|
|
ary[2][aggregate] += duration
|
|
elsif @aggregate_type == :count
|
|
ary[2][aggregate] += 1
|
|
end
|
|
end
|
|
end
|
|
|
|
def top(n, aggregator_formatter=nil)
|
|
@data.sort{|a,b| b[1][0] <=> a[1][0]}.first(n).map do |metric, ary|
|
|
metric = metric.to_s
|
|
metric = "[empty]" if metric.length == 0
|
|
result = [metric, ary[0], ary[1]]
|
|
# handle aggregate
|
|
if ary[2]
|
|
if aggregator_formatter
|
|
result.push aggregator_formatter.call(ary[2], ary[0], ary[1])
|
|
else
|
|
result.push ary[2].sort{|a,b| b[1] <=> a[1]}.first(5).map{|k,v|
|
|
v = "%.2f" % v if Float === v
|
|
"#{k}(#{v})"}.join(" ")
|
|
end
|
|
end
|
|
|
|
result
|
|
end
|
|
end
|
|
end
|
|
|
|
def initialize(filenames, args={})
|
|
@filenames = filenames
|
|
@ip_to_rails_duration = Aggeregator.new
|
|
@username_to_rails_duration = Aggeregator.new
|
|
|
|
@route_to_rails_duration = Aggeregator.new
|
|
@route_to_rails_duration.aggregate_type = :count
|
|
|
|
@url_to_rails_duration = Aggeregator.new
|
|
@status_404_to_count = Aggeregator.new
|
|
|
|
@total_requests = 0
|
|
@message_bus_requests = 0
|
|
@limit = args[:limit]
|
|
end
|
|
|
|
def analyze
|
|
now = DateTime.now
|
|
|
|
@filenames.each do |filename|
|
|
File.open(filename).each_line do |line|
|
|
@total_requests += 1
|
|
parsed = LineParser.parse(line)
|
|
|
|
next unless parsed.time
|
|
next if @limit && ((now - parsed.parsed_time) * 24 * 60).to_i > @limit
|
|
|
|
@from_time ||= parsed.time
|
|
@to_time = parsed.time
|
|
|
|
if parsed.url =~ /(POST|GET) \/message-bus/
|
|
@message_bus_requests += 1
|
|
next
|
|
end
|
|
|
|
@ip_to_rails_duration.add(parsed.ip_address, parsed.rails_duration)
|
|
|
|
username = parsed.username == "-" ? "[Anonymous]" : parsed.username
|
|
@username_to_rails_duration.add(username, parsed.rails_duration, parsed.route)
|
|
|
|
@route_to_rails_duration.add(parsed.route, parsed.rails_duration, parsed.is_mobile? ? "mobile" : "desktop")
|
|
|
|
@url_to_rails_duration.add(parsed.url, parsed.rails_duration)
|
|
|
|
@status_404_to_count.add(parsed.url,1) if parsed.status == "404"
|
|
end
|
|
end
|
|
self
|
|
end
|
|
|
|
end
|
|
|
|
args = ARGV.dup
|
|
|
|
if args[0] == "--limit"
|
|
args.shift
|
|
limit = args.shift.to_i
|
|
end
|
|
|
|
filenames = args if args[0]
|
|
filenames ||= ["/var/log/nginx/access.log", "/var/log/nginx/access.log.1"]
|
|
|
|
analyzer = LogAnalyzer.analyze(filenames, limit: limit)
|
|
|
|
SPACER = "-" * 100
|
|
|
|
# don't feel like pulling in active support
|
|
def map_with_index(ary, &block)
|
|
idx = 0
|
|
ary.map do |item|
|
|
v = block.call(item, idx)
|
|
idx += 1
|
|
v
|
|
end
|
|
end
|
|
|
|
def top(cols, aggregator, count, aggregator_formatter = nil)
|
|
sorted = aggregator.top(count, aggregator_formatter)
|
|
|
|
col_just = []
|
|
|
|
col_widths = map_with_index(cols) do |name,idx|
|
|
max_width = name.length
|
|
|
|
if cols[idx].respond_to? :align
|
|
col_just[idx] = cols[idx].align
|
|
skip_just_detection = true
|
|
else
|
|
col_just[idx] = :ljust
|
|
end
|
|
|
|
sorted.each do |row|
|
|
col_just[idx] = :rjust unless (String === row[idx] || row[idx].nil?) && !skip_just_detection
|
|
row[idx] = '%.2f' % row[idx] if Float === row[idx]
|
|
row[idx] = row[idx].to_s
|
|
max_width = row[idx].length if row[idx].length > max_width
|
|
end
|
|
[max_width,80].min
|
|
end
|
|
|
|
puts(map_with_index(cols) do |name,idx|
|
|
name.ljust(col_widths[idx])
|
|
end.join(" "))
|
|
|
|
puts(map_with_index(cols) do |name,idx|
|
|
("-" * name.length).ljust(col_widths[idx])
|
|
end.join(" "))
|
|
|
|
sorted.each do |raw_row|
|
|
|
|
rows = []
|
|
idx = 0
|
|
raw_row.each do |col|
|
|
j = 0
|
|
col.to_s.scan(/(.{1,80}($|\s)|.{1,80})/).each do |r|
|
|
rows[j] ||= []
|
|
rows[j][idx] = r[0]
|
|
j += 1
|
|
end
|
|
idx += 1
|
|
end
|
|
|
|
if rows.length > 1
|
|
puts
|
|
end
|
|
|
|
rows.each do |row|
|
|
cols.length.times do |i|
|
|
print row[i].to_s.send(col_just[i], col_widths[i])
|
|
print " "
|
|
end
|
|
puts
|
|
end
|
|
|
|
if rows.length > 1
|
|
puts
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
class Column < String
|
|
attr_accessor :align
|
|
|
|
def initialize(val, align)
|
|
super(val)
|
|
@align = align
|
|
end
|
|
end
|
|
|
|
puts
|
|
puts "Analyzed: #{analyzer.filenames.join(",")} on #{`hostname`}"
|
|
if limit
|
|
puts "Limited to #{DateTime.now - (limit.to_f / (60*24.0))} - #{DateTime.now}"
|
|
end
|
|
puts SPACER
|
|
puts "#{analyzer.from_time} - #{analyzer.to_time}"
|
|
puts SPACER
|
|
puts "Total Requests: #{analyzer.total_requests} ( MessageBus: #{analyzer.message_bus_requests} )"
|
|
puts SPACER
|
|
puts "Top 30 IPs by Server Load"
|
|
puts
|
|
top(["IP Address", "Duration", "Reqs"], analyzer.ip_to_rails_duration, 30)
|
|
puts SPACER
|
|
puts
|
|
puts "Top 30 users by Server Load"
|
|
puts
|
|
top(["Username", "Duration", "Reqs", "Routes"], analyzer.username_to_rails_duration, 30)
|
|
puts SPACER
|
|
puts
|
|
puts "Top 100 routes by Server Load"
|
|
puts
|
|
top(["Route", "Duration", "Reqs", Column.new("Mobile", :rjust)], analyzer.route_to_rails_duration, 100, lambda{
|
|
|hash, name, total|
|
|
"#{hash["mobile"] || 0} (#{"%.2f" % (((hash["mobile"] || 0) / (total + 0.0)) * 100)})%"}
|
|
)
|
|
puts SPACER
|
|
puts
|
|
puts "Top 30 urls by Server Load"
|
|
puts
|
|
top(["Url", "Duration", "Reqs"], analyzer.url_to_rails_duration, 30)
|
|
|
|
puts "(all durations in seconds)"
|
|
puts SPACER
|
|
puts
|
|
puts "Top 30 not found urls (404s)"
|
|
puts
|
|
top(["Url", "Count"], analyzer.status_404_to_count, 30)
|