FEATURE: unicorn sidekiq will restart sidekiq on complete failure.

(checks every 30 minutes for complete failure)

app/jobs/regular/run_heartbeat.rb

@@ -0,0 +1,16 @@
+module Jobs
+  class RunHeartbeat < Jobs::Base
+
+    def self.heartbeat_key
+      'heartbeat_last_run'
+    end
+
+    def execute(args)
+      $redis.set(self.class.heartbeat_key, Time.new.to_i.to_s)
+    end
+
+    def self.last_heartbeat
+      $redis.get(heartbeat_key).to_i
+    end
+  end
+end

app/jobs/scheduled/heartbeat.rb

@@ -0,0 +1,11 @@
+module Jobs
+
+  # used to ensure at least 1 sidekiq is running correctly
+  class Heartbeat < Jobs::Scheduled
+    every 3.minute
+
+    def execute(args)
+      Jobs.enqueue(:run_heartbeat, {})
+    end
+  end
+end

config/unicorn.conf.rb

@@ -72,6 +72,7 @@ before_fork do |server, worker|
     sidekiqs = ENV['UNICORN_SIDEKIQS'].to_i
     if sidekiqs > 0
       puts "Starting up #{sidekiqs} supervised sidekiqs"
+
       require 'demon/sidekiq'
 
       Demon::Sidekiq.start(sidekiqs)
@@ -79,8 +80,26 @@ before_fork do |server, worker|
       class ::Unicorn::HttpServer
         alias :master_sleep_orig :master_sleep
 
+        def check_sidekiq_heartbeat
+          @sidekiq_heartbeat_interval ||= 30.minutes
+          @sidekiq_next_heartbeat_check ||= Time.new.to_i + @sidekiq_heartbeat_interval
+
+          if @sidekiq_next_heartbeat_check < Time.new.to_i
+            last_heartbeat = Jobs::RunHeartbeat.last_heartbeat
+            if last_heartbeat < Time.new.to_i - @sidekiq_heartbeat_interval
+              STDERR.puts "Sidekiq heartbeat test failed, restarting"
+              puts "Sidekiq heartbeat test failed, restarting"
+
+              Demon::Sidekiq.restart
+            end
+            @sidekiq_next_heartbeat_check = Time.new.to_i + @sidekiq_heartbeat_interval
+          end
+        end
+
         def master_sleep(sec)
           Demon::Sidekiq.ensure_running
+          check_sidekiq_heartbeat
+
           master_sleep_orig(sec)
         end
       end

lib/demon/base.rb

@@ -17,6 +17,14 @@ class Demon::Base
     end
   end
 
+  def self.restart
+    return unless @demons
+    @demons.values.each do |demon|
+      demon.stop
+      demon.start
+    end
+  end
+
   def self.ensure_running
     @demons.values.each do |demon|
       demon.ensure_running
@@ -39,6 +47,7 @@ class Demon::Base
     if @pid
       Process.kill("HUP",@pid)
       @pid = nil
+      @started = false
     end
   end