diff --git a/app/jobs/regular/run_heartbeat.rb b/app/jobs/regular/run_heartbeat.rb new file mode 100644 index 00000000000..5cf0072588b --- /dev/null +++ b/app/jobs/regular/run_heartbeat.rb @@ -0,0 +1,16 @@ +module Jobs + class RunHeartbeat < Jobs::Base + + def self.heartbeat_key + 'heartbeat_last_run' + end + + def execute(args) + $redis.set(self.class.heartbeat_key, Time.new.to_i.to_s) + end + + def self.last_heartbeat + $redis.get(heartbeat_key).to_i + end + end +end diff --git a/app/jobs/scheduled/heartbeat.rb b/app/jobs/scheduled/heartbeat.rb new file mode 100644 index 00000000000..101a082ef77 --- /dev/null +++ b/app/jobs/scheduled/heartbeat.rb @@ -0,0 +1,11 @@ +module Jobs + + # used to ensure at least 1 sidekiq is running correctly + class Heartbeat < Jobs::Scheduled + every 3.minute + + def execute(args) + Jobs.enqueue(:run_heartbeat, {}) + end + end +end diff --git a/config/unicorn.conf.rb b/config/unicorn.conf.rb index 6b28c2dc1d9..3f97c4742a5 100644 --- a/config/unicorn.conf.rb +++ b/config/unicorn.conf.rb @@ -72,6 +72,7 @@ before_fork do |server, worker| sidekiqs = ENV['UNICORN_SIDEKIQS'].to_i if sidekiqs > 0 puts "Starting up #{sidekiqs} supervised sidekiqs" + require 'demon/sidekiq' Demon::Sidekiq.start(sidekiqs) @@ -79,8 +80,26 @@ before_fork do |server, worker| class ::Unicorn::HttpServer alias :master_sleep_orig :master_sleep + def check_sidekiq_heartbeat + @sidekiq_heartbeat_interval ||= 30.minutes + @sidekiq_next_heartbeat_check ||= Time.new.to_i + @sidekiq_heartbeat_interval + + if @sidekiq_next_heartbeat_check < Time.new.to_i + last_heartbeat = Jobs::RunHeartbeat.last_heartbeat + if last_heartbeat < Time.new.to_i - @sidekiq_heartbeat_interval + STDERR.puts "Sidekiq heartbeat test failed, restarting" + puts "Sidekiq heartbeat test failed, restarting" + + Demon::Sidekiq.restart + end + @sidekiq_next_heartbeat_check = Time.new.to_i + @sidekiq_heartbeat_interval + end + end + def master_sleep(sec) Demon::Sidekiq.ensure_running + check_sidekiq_heartbeat + master_sleep_orig(sec) end end diff --git a/lib/demon/base.rb b/lib/demon/base.rb index 4867d89958d..c8cb0b0116e 100644 --- a/lib/demon/base.rb +++ b/lib/demon/base.rb @@ -17,6 +17,14 @@ class Demon::Base end end + def self.restart + return unless @demons + @demons.values.each do |demon| + demon.stop + demon.start + end + end + def self.ensure_running @demons.values.each do |demon| demon.ensure_running @@ -39,6 +47,7 @@ class Demon::Base if @pid Process.kill("HUP",@pid) @pid = nil + @started = false end end