PERF: restart sidekiq if it consumes more than 500MB
configurable with UNICORN_SIDEKIQ_MAX_RSS
This commit is contained in:
parent
48c58601a6
commit
43eb030402
|
@ -80,19 +80,39 @@ before_fork do |server, worker|
|
||||||
class ::Unicorn::HttpServer
|
class ::Unicorn::HttpServer
|
||||||
alias :master_sleep_orig :master_sleep
|
alias :master_sleep_orig :master_sleep
|
||||||
|
|
||||||
|
def max_rss
|
||||||
|
`ps -eo rss,args | grep sidekiq | grep -v grep | awk '{print $1}'`
|
||||||
|
.split("\n")
|
||||||
|
.map(&:to_i)
|
||||||
|
.max * 1024
|
||||||
|
end
|
||||||
|
|
||||||
|
def out_of_memory?
|
||||||
|
max_allowed_size = [ENV['UNICORN_SIDEKIQ_MAX_RSS'].to_i, 500].min;
|
||||||
|
max_rss > max_allowed_size
|
||||||
|
end
|
||||||
|
|
||||||
def check_sidekiq_heartbeat
|
def check_sidekiq_heartbeat
|
||||||
@sidekiq_heartbeat_interval ||= 30.minutes
|
@sidekiq_heartbeat_interval ||= 30.minutes
|
||||||
@sidekiq_next_heartbeat_check ||= Time.new.to_i + @sidekiq_heartbeat_interval
|
@sidekiq_next_heartbeat_check ||= Time.new.to_i + @sidekiq_heartbeat_interval
|
||||||
|
|
||||||
if @sidekiq_next_heartbeat_check < Time.new.to_i
|
if @sidekiq_next_heartbeat_check < Time.new.to_i
|
||||||
|
|
||||||
last_heartbeat = Jobs::RunHeartbeat.last_heartbeat
|
last_heartbeat = Jobs::RunHeartbeat.last_heartbeat
|
||||||
|
if out_of_memory?
|
||||||
|
Rails.logger.warn("Sidekiq is consuming too much memory (using: %0.2fM), restarting" % (max_rss.to_f / 1.megabyte))
|
||||||
|
Demon::Sidekiq.restart
|
||||||
|
end
|
||||||
|
|
||||||
if last_heartbeat < Time.new.to_i - @sidekiq_heartbeat_interval
|
if last_heartbeat < Time.new.to_i - @sidekiq_heartbeat_interval
|
||||||
STDERR.puts "Sidekiq heartbeat test failed, restarting"
|
STDERR.puts "Sidekiq heartbeat test failed, restarting"
|
||||||
puts "Sidekiq heartbeat test failed, restarting"
|
Rails.logger.warn "Sidekiq heartbeat test failed, restarting"
|
||||||
|
|
||||||
Demon::Sidekiq.restart
|
Demon::Sidekiq.restart
|
||||||
end
|
end
|
||||||
@sidekiq_next_heartbeat_check = Time.new.to_i + @sidekiq_heartbeat_interval
|
@sidekiq_next_heartbeat_check = Time.new.to_i + @sidekiq_heartbeat_interval
|
||||||
|
|
||||||
|
$redis.client.disconnect
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue