diff --git a/Gemfile b/Gemfile index 0345d7ee1eb..7b0d05e1766 100644 --- a/Gemfile +++ b/Gemfile @@ -140,7 +140,10 @@ gem 'rinku' gem 'sanitize' gem 'sass' gem 'sidekiq' -gem 'sidekiq-failures' + +# https://github.com/mhfs/sidekiq-failures/issues/72 +gem 'sidekiq-failures-discourse', require: 'sidekiq-failures' + gem 'sinatra', require: nil gem 'slim' # required for sidekiq-web diff --git a/Gemfile.lock b/Gemfile.lock index 736bbf99633..02a51596b6d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -62,7 +62,7 @@ GEM timers (~> 1.1.0) certified (0.1.1) coderay (1.1.0) - connection_pool (1.2.0) + connection_pool (2.0.0) daemons (1.1.9) debug_inspector (0.0.2) diff-lcs (1.2.5) @@ -324,7 +324,7 @@ GEM json redis (>= 3.0.6) redis-namespace (>= 1.3.1) - sidekiq-failures (0.3.0) + sidekiq-failures-discourse (0.3.0) sidekiq (>= 2.14.0) simple-rss (1.3.1) simplecov (0.8.2) @@ -464,7 +464,7 @@ DEPENDENCIES seed-fu-discourse shoulda sidekiq - sidekiq-failures + sidekiq-failures-discourse simple-rss simplecov sinatra diff --git a/lib/demon/base.rb b/lib/demon/base.rb index d9b2f093c0f..4867d89958d 100644 --- a/lib/demon/base.rb +++ b/lib/demon/base.rb @@ -3,7 +3,7 @@ module Demon; end # intelligent fork based demonizer class Demon::Base - def self.start(count) + def self.start(count=1) @demons ||= {} count.times do |i| (@demons["#{prefix}_#{i}"] ||= new(i)).start @@ -17,11 +17,17 @@ class Demon::Base end end + def self.ensure_running + @demons.values.each do |demon| + demon.ensure_running + end + end + def initialize(index) @index = index @pid = nil @parent_pid = Process.pid - @monitor = nil + @started = false end def pid_file @@ -29,29 +35,47 @@ class Demon::Base end def stop - if @monitor - @monitor.kill - @monitor.join - @monitor = nil - end - + @started = false if @pid Process.kill("HUP",@pid) @pid = nil end end + def ensure_running + return unless @started + + if !@pid + @started = false + start + return + end + + dead = Process.waitpid(@pid, Process::WNOHANG) rescue -1 + if dead + STDERR.puts "Detected dead worker #{@pid}, restarting..." + @pid = nil + @started = false + start + end + end + def start + return if @pid || @started + if existing = already_running? # should not happen ... so kill violently + STDERR.puts "Attempting to kill pid #{existing}" Process.kill("TERM",existing) end - return if @pid + @started = true + run + end + def run if @pid = fork write_pid_file - monitor_child return end @@ -73,19 +97,6 @@ class Demon::Base private - def monitor_child - @monitor ||= Thread.new do - while true - sleep 5 - unless alive?(@pid) - STDERR.puts "#{@pid} died, restarting the process" - @pid = nil - start - end - end - end - end - def write_pid_file FileUtils.mkdir_p(Rails.root + "tmp/pids") File.open(pid_file,'w') do |f| @@ -110,13 +121,21 @@ class Demon::Base def alive?(pid) begin - Process.getpgid(pid) + Process.kill(0, pid) true - rescue Errno::ESRCH + rescue false end end + def suppress_stdout + true + end + + def suppress_stderr + true + end + def establish_app Discourse.after_fork @@ -129,8 +148,8 @@ class Demon::Base end # keep stuff simple for now - $stdout.reopen("/dev/null", "w") - $stderr.reopen("/dev/null", "w") + $stdout.reopen("/dev/null", "w") if suppress_stdout + $stderr.reopen("/dev/null", "w") if suppress_stderr end def after_fork diff --git a/lib/demon/sidekiq.rb b/lib/demon/sidekiq.rb index 392128e1edf..b2d414ac050 100644 --- a/lib/demon/sidekiq.rb +++ b/lib/demon/sidekiq.rb @@ -8,12 +8,21 @@ class Demon::Sidekiq < Demon::Base private + def suppress_stdout + false + end + + def suppress_stderr + false + end + def after_fork + STDERR.puts "Loading Sidekiq in process id #{Process.pid}" require 'sidekiq/cli' - # Reload initializer cause it needs to run after sidekiq/cli was required - load Rails.root + "config/initializers/sidekiq.rb" cli = Sidekiq::CLI.instance cli.parse([]) + + load Rails.root + "config/initializers/sidekiq.rb" cli.run rescue => e STDERR.puts e.message diff --git a/lib/discourse.rb b/lib/discourse.rb index 3f9235ae908..2d652cc0538 100644 --- a/lib/discourse.rb +++ b/lib/discourse.rb @@ -4,6 +4,7 @@ require_dependency 'auth/default_current_user_provider' module Discourse + require 'sidekiq/exception_handler' class SidekiqExceptionHandler extend Sidekiq::ExceptionHandler end diff --git a/lib/scheduler/manager.rb b/lib/scheduler/manager.rb index 9529e7c6bac..3d48eb101ec 100644 --- a/lib/scheduler/manager.rb +++ b/lib/scheduler/manager.rb @@ -229,9 +229,17 @@ module Scheduler def self.discover_schedules + # hack for developemnt reloader is crazytown + # multiple classes with same name can be in + # object space + unique = Set.new schedules = [] ObjectSpace.each_object(Scheduler::Schedule) do |schedule| - schedules << schedule if schedule.scheduled? + if schedule.scheduled? + next if unique.include?(schedule.to_s) + schedules << schedule + unique << schedule.to_s + end end schedules end diff --git a/lib/scheduler/schedule_info.rb b/lib/scheduler/schedule_info.rb index 7a79463a02c..7be6da36fe1 100644 --- a/lib/scheduler/schedule_info.rb +++ b/lib/scheduler/schedule_info.rb @@ -10,7 +10,6 @@ module Scheduler @klass = klass @manager = manager - key = Manager.schedule_key(klass) data = nil if data = $redis.get(key) @@ -85,7 +84,6 @@ module Scheduler end def write! - key = Manager.schedule_key(@klass) clear! redis.set key, { next_run: @next_run, @@ -94,6 +92,7 @@ module Scheduler prev_result: @prev_result, current_owner: @current_owner }.to_json + redis.zadd Manager.queue_key, @next_run , @klass end @@ -112,9 +111,8 @@ module Scheduler private def clear! - key = Manager.schedule_key(@klass) redis.del key - redis.zrem Manager.queue_key, key + redis.zrem Manager.queue_key, @klass end end