2014-02-05 18:14:41 -05:00
|
|
|
require 'scheduler/scheduler'
|
|
|
|
|
2013-02-05 14:16:51 -05:00
|
|
|
module Jobs
|
|
|
|
|
2013-03-22 11:35:32 -04:00
|
|
|
def self.queued
|
|
|
|
Sidekiq::Stats.new.enqueued
|
|
|
|
end
|
|
|
|
|
|
|
|
def self.last_job_performed_at
|
|
|
|
Sidekiq.redis do |r|
|
|
|
|
int = r.get('last_job_perform_at')
|
|
|
|
int ? Time.at(int.to_i) : nil
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2013-04-22 13:37:16 -04:00
|
|
|
def self.num_email_retry_jobs
|
|
|
|
Sidekiq::RetrySet.new.select { |job| job.klass =~ /Email$/ }.size
|
|
|
|
end
|
|
|
|
|
2013-02-05 14:16:51 -05:00
|
|
|
class Base
|
2013-09-29 23:22:19 -04:00
|
|
|
|
|
|
|
class Instrumenter
|
|
|
|
|
|
|
|
def self.stats
|
|
|
|
Thread.current[:db_stats] ||= Stats.new
|
|
|
|
end
|
|
|
|
|
|
|
|
class Stats
|
|
|
|
attr_accessor :query_count, :duration_ms
|
|
|
|
|
|
|
|
def initialize
|
|
|
|
@query_count = 0
|
|
|
|
@duration_ms = 0
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def call(name, start, finish, message_id, values)
|
|
|
|
stats = Instrumenter.stats
|
|
|
|
stats.query_count += 1
|
|
|
|
stats.duration_ms += (((finish - start).to_f) * 1000).to_i
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2013-02-05 14:16:51 -05:00
|
|
|
include Sidekiq::Worker
|
|
|
|
|
2013-09-29 23:22:19 -04:00
|
|
|
def initialize
|
|
|
|
@db_duration = 0
|
|
|
|
end
|
|
|
|
|
2013-06-30 14:00:23 -04:00
|
|
|
def log(*args)
|
|
|
|
puts args
|
|
|
|
args.each do |arg|
|
|
|
|
Rails.logger.info "#{Time.now.to_formatted_s(:db)}: [#{self.class.name.upcase}] #{arg}"
|
|
|
|
end
|
|
|
|
true
|
|
|
|
end
|
|
|
|
|
2013-02-05 14:16:51 -05:00
|
|
|
def self.delayed_perform(opts={})
|
|
|
|
self.new.perform(opts)
|
|
|
|
end
|
|
|
|
|
|
|
|
def execute(opts={})
|
|
|
|
raise "Overwrite me!"
|
|
|
|
end
|
|
|
|
|
2013-09-29 23:22:19 -04:00
|
|
|
def last_db_duration
|
|
|
|
@db_duration || 0
|
|
|
|
end
|
|
|
|
|
|
|
|
def ensure_db_instrumented
|
|
|
|
@@instrumented ||= begin
|
|
|
|
ActiveSupport::Notifications.subscribe('sql.active_record', Instrumenter.new)
|
|
|
|
true
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2013-08-07 13:25:05 -04:00
|
|
|
def perform(*args)
|
2013-09-29 23:22:19 -04:00
|
|
|
ensure_db_instrumented
|
2013-08-07 13:25:05 -04:00
|
|
|
opts = args.extract_options!.with_indifferent_access
|
2013-02-25 11:42:20 -05:00
|
|
|
|
2013-03-22 11:35:32 -04:00
|
|
|
if SiteSetting.queue_jobs?
|
|
|
|
Sidekiq.redis do |r|
|
|
|
|
r.set('last_job_perform_at', Time.now.to_i)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2013-02-05 14:16:51 -05:00
|
|
|
if opts.delete(:sync_exec)
|
2013-03-04 19:42:44 -05:00
|
|
|
if opts.has_key?(:current_site_id) && opts[:current_site_id] != RailsMultisite::ConnectionManagement.current_db
|
2013-02-05 14:16:51 -05:00
|
|
|
raise ArgumentError.new("You can't connect to another database when executing a job synchronously.")
|
|
|
|
else
|
|
|
|
return execute(opts)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
dbs =
|
2013-02-25 11:42:20 -05:00
|
|
|
if opts[:current_site_id]
|
2013-02-05 14:16:51 -05:00
|
|
|
[opts[:current_site_id]]
|
2013-02-25 11:42:20 -05:00
|
|
|
else
|
2013-02-05 14:16:51 -05:00
|
|
|
RailsMultisite::ConnectionManagement.all_dbs
|
|
|
|
end
|
|
|
|
|
2013-09-29 23:22:19 -04:00
|
|
|
total_db_time = 0
|
2014-02-20 23:31:15 -05:00
|
|
|
exceptions = []
|
2013-02-05 14:16:51 -05:00
|
|
|
dbs.each do |db|
|
|
|
|
begin
|
2013-08-15 23:08:23 -04:00
|
|
|
thread_exception = nil
|
|
|
|
# NOTE: This looks odd, in fact it looks crazy but there is a reason
|
|
|
|
# A bug in therubyracer means that under certain conditions running in a fiber
|
|
|
|
# can cause the whole v8 context to corrupt so much that it will hang sidekiq
|
|
|
|
#
|
|
|
|
# If you are brave and want to try to fix this either in celluloid or therubyracer, the repro is:
|
|
|
|
#
|
|
|
|
# 1. Create a big Discourse db: (you can start from script/profile_db_generator.rb)
|
|
|
|
# 2. Queue a ton of jobs, eg: User.pluck(:id).each{|id| Jobs.enqueue(:user_email, type: :digest, user_id: id)};
|
|
|
|
# 3. Run sidekiq
|
|
|
|
#
|
|
|
|
# The issue only happens in Ruby 2.0 for some reason, you start getting V8::Error with no context
|
|
|
|
#
|
|
|
|
# See: https://github.com/cowboyd/therubyracer/issues/206
|
|
|
|
#
|
|
|
|
# The restricted stack space of fibers opens a bunch of risks up, by avoiding them altogether
|
|
|
|
# we can mitigate giving up a very marginal amount of throughput
|
|
|
|
#
|
|
|
|
# Ideally we could just tell sidekiq to avoid fibers
|
|
|
|
|
|
|
|
t = Thread.new do
|
|
|
|
begin
|
|
|
|
RailsMultisite::ConnectionManagement.establish_connection(db: db)
|
|
|
|
I18n.locale = SiteSetting.default_locale
|
|
|
|
execute(opts)
|
|
|
|
rescue => e
|
|
|
|
thread_exception = e
|
|
|
|
ensure
|
|
|
|
ActiveRecord::Base.connection_handler.clear_active_connections!
|
2013-09-29 23:22:19 -04:00
|
|
|
total_db_time += Instrumenter.stats.duration_ms
|
2013-08-15 23:08:23 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
t.join
|
|
|
|
|
2014-02-21 00:05:19 -05:00
|
|
|
exceptions << thread_exception if thread_exception
|
2013-02-05 14:16:51 -05:00
|
|
|
end
|
|
|
|
end
|
2013-05-19 10:22:31 -04:00
|
|
|
|
2014-02-20 23:31:15 -05:00
|
|
|
if exceptions.length > 0
|
|
|
|
exceptions[1..-1].each do |exception|
|
|
|
|
Discourse.handle_exception(exception, opts)
|
|
|
|
end
|
|
|
|
raise exceptions[0]
|
|
|
|
end
|
|
|
|
|
2013-05-19 10:22:31 -04:00
|
|
|
ensure
|
|
|
|
ActiveRecord::Base.connection_handler.clear_active_connections!
|
2013-09-29 23:22:19 -04:00
|
|
|
@db_duration = total_db_time
|
2013-02-05 14:16:51 -05:00
|
|
|
end
|
2013-05-19 10:22:31 -04:00
|
|
|
|
2013-02-05 14:16:51 -05:00
|
|
|
end
|
|
|
|
|
2013-08-07 13:25:05 -04:00
|
|
|
class Scheduled < Base
|
2014-02-05 18:14:41 -05:00
|
|
|
extend Scheduler::Schedule
|
2013-08-07 13:25:05 -04:00
|
|
|
end
|
2013-02-05 14:16:51 -05:00
|
|
|
|
2013-08-07 13:25:05 -04:00
|
|
|
def self.enqueue(job_name, opts={})
|
|
|
|
klass = "Jobs::#{job_name.to_s.camelcase}".constantize
|
2013-02-05 14:16:51 -05:00
|
|
|
|
|
|
|
# Unless we want to work on all sites
|
|
|
|
unless opts.delete(:all_sites)
|
|
|
|
opts[:current_site_id] ||= RailsMultisite::ConnectionManagement.current_db
|
|
|
|
end
|
|
|
|
|
|
|
|
# If we are able to queue a job, do it
|
|
|
|
if SiteSetting.queue_jobs?
|
|
|
|
if opts[:delay_for].present?
|
|
|
|
klass.delay_for(opts.delete(:delay_for)).delayed_perform(opts)
|
|
|
|
else
|
2013-08-07 13:25:05 -04:00
|
|
|
Sidekiq::Client.enqueue(klass, opts)
|
2013-02-05 14:16:51 -05:00
|
|
|
end
|
|
|
|
else
|
|
|
|
# Otherwise execute the job right away
|
|
|
|
opts.delete(:delay_for)
|
|
|
|
opts[:sync_exec] = true
|
|
|
|
klass.new.perform(opts)
|
|
|
|
end
|
|
|
|
|
|
|
|
end
|
|
|
|
|
|
|
|
def self.enqueue_in(secs, job_name, opts={})
|
|
|
|
enqueue(job_name, opts.merge!(delay_for: secs))
|
|
|
|
end
|
|
|
|
|
2013-05-07 14:25:41 -04:00
|
|
|
def self.enqueue_at(datetime, job_name, opts={})
|
|
|
|
enqueue_in( [(datetime - Time.zone.now).to_i, 0].max, job_name, opts )
|
|
|
|
end
|
|
|
|
|
|
|
|
def self.cancel_scheduled_job(job_name, params={})
|
2013-05-15 15:19:41 -04:00
|
|
|
jobs = scheduled_for(job_name, params)
|
|
|
|
return false if jobs.empty?
|
|
|
|
jobs.each { |job| job.delete }
|
|
|
|
true
|
|
|
|
end
|
|
|
|
|
|
|
|
def self.scheduled_for(job_name, params={})
|
2013-05-07 14:25:41 -04:00
|
|
|
job_class = "Jobs::#{job_name.to_s.camelcase}"
|
2013-05-15 15:19:41 -04:00
|
|
|
Sidekiq::ScheduledSet.new.select do |scheduled_job|
|
2013-05-07 14:25:41 -04:00
|
|
|
if scheduled_job.klass == 'Sidekiq::Extensions::DelayedClass'
|
|
|
|
job_args = YAML.load(scheduled_job.args[0])
|
2013-06-07 14:59:08 -04:00
|
|
|
job_args_class, _, (job_args_params, *) = job_args
|
|
|
|
if job_args_class.to_s == job_class && job_args_params
|
2013-05-07 14:25:41 -04:00
|
|
|
matched = true
|
|
|
|
params.each do |key, value|
|
2013-06-07 14:59:08 -04:00
|
|
|
unless job_args_params[key] == value
|
2013-05-07 14:25:41 -04:00
|
|
|
matched = false
|
|
|
|
break
|
|
|
|
end
|
|
|
|
end
|
2013-05-15 15:19:41 -04:00
|
|
|
matched
|
|
|
|
else
|
|
|
|
false
|
2013-05-07 14:25:41 -04:00
|
|
|
end
|
2013-05-15 15:19:41 -04:00
|
|
|
else
|
|
|
|
false
|
2013-05-07 14:25:41 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
2013-02-05 14:16:51 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
# Require all jobs
|
2013-10-01 03:04:02 -04:00
|
|
|
Dir["#{Rails.root}/app/jobs/regular/*.rb"].each {|file| require_dependency file }
|
|
|
|
Dir["#{Rails.root}/app/jobs/scheduled/*.rb"].each {|file| require_dependency file }
|