From a3885a18f8db2f32d6556bae377a16c3a9e94130 Mon Sep 17 00:00:00 2001
From: Sam <sam.saffron@gmail.com>
Date: Thu, 18 Jun 2015 15:32:04 +1000
Subject: [PATCH] extra logic to force kill orphan sidekiqs

---
 config/unicorn.conf.rb | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/config/unicorn.conf.rb b/config/unicorn.conf.rb
index 647b1f17351..1e3752218da 100644
--- a/config/unicorn.conf.rb
+++ b/config/unicorn.conf.rb
@@ -91,11 +91,25 @@ before_fork do |server, worker|
           rss * 1024
         end
 
+        def max_allowed_size
+          [ENV['UNICORN_SIDEKIQ_MAX_RSS'].to_i, 500].max.megabytes
+        end
+
         def out_of_memory?
-          max_allowed_size = [ENV['UNICORN_SIDEKIQ_MAX_RSS'].to_i, 500].max.megabytes;
           max_rss > max_allowed_size
         end
 
+        def force_kill_rogue_sidekiq
+          info = `ps -eo pid,rss,args | grep sidekiq | grep -v grep | awk '{print $1,$2}'`
+          info.split("\n").each do |row|
+            pid,mem = row.split(" ").map(&:to_i)
+            if pid > 0 && (mem*1024) > max_allowed_size
+              Rails.logger.warn "Detected rogue Sidekiq pid #{pid} mem #{mem*1024}, killing"
+              Process.kill("KILL", pid) rescue nil
+            end
+          end
+        end
+
         def check_sidekiq_heartbeat
           @sidekiq_heartbeat_interval ||= 30.minutes
           @sidekiq_next_heartbeat_check ||= Time.new.to_i + @sidekiq_heartbeat_interval
@@ -118,7 +132,11 @@ before_fork do |server, worker|
             end
             @sidekiq_next_heartbeat_check = Time.new.to_i + @sidekiq_heartbeat_interval
 
-            Demon::Sidekiq.restart if restart
+            if restart
+              Demon::Sidekiq.restart
+              sleep 10
+              force_kill_rogue_sidekiq
+            end
             $redis.client.disconnect
           end
         end