IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Revert "FIX: Heartbeat check per sidekiq process (#7873)""

Browse Source 
This reverts commit e805d44965.
We now have mechanisms in place to ensure heartbeat will always
be scheduled even if the scheduler is overloaded per: 098f938b
This commit is contained in:
Sam Saffron
2019-08-30 10:12:10 +10:00
parent ff8cc244d8
commit c3497559be
6 changed files with 76 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
config/unicorn.conf.rb
View File

@@ -144,25 +144,32 @@ before_fork do |server, worker|
@sidekiq_next_heartbeat_check ||= Time.new.to_i + @sidekiq_heartbeat_interval
if @sidekiq_next_heartbeat_check < Time.new.to_i
last_heartbeat = Jobs::RunHeartbeat.last_heartbeat
restart = false
@sidekiq_next_heartbeat_check = Time.new.to_i + @sidekiq_heartbeat_interval
restarted = false
if out_of_memory?
Rails.logger.warn("Sidekiq is consuming too much memory (using: %0.2fM) for '%s', restarting" % [(max_rss.to_f / 1.megabyte), ENV["DISCOURSE_HOSTNAME"]])
restart = true
end
if last_heartbeat < Time.new.to_i - @sidekiq_heartbeat_interval
STDERR.puts "Sidekiq heartbeat test failed, restarting"
Rails.logger.warn "Sidekiq heartbeat test failed, restarting"
restart = true
end
@sidekiq_next_heartbeat_check = Time.new.to_i + @sidekiq_heartbeat_interval
if restart
Demon::Sidekiq.restart
restarted = true
end
if !restarted
Demon::Sidekiq::QUEUE_IDS.each do |identifier|
last_heartbeat = Demon::Sidekiq.get_queue_last_heartbeat(identifier)
if last_heartbeat < Time.new.to_i - @sidekiq_heartbeat_interval
if demon = Demon::Sidekiq.demons.values.find { |d| d.identifier == identifier }
STDERR.puts "Sidekiq heartbeat test for worker #{demon.pid} failed, restarting"
Rails.logger.warn "Sidekiq heartbeat test for worker #{demon.pid} failed, restarting"
demon.stop
demon.start
restarted = true
end
end
end
end
if restarted
sleep 10
force_kill_rogue_sidekiq
end