mirror of
https://github.com/discourse/discourse.git
synced 2025-02-25 18:55:32 -06:00
DEV: Log Unicorn worker timeout backtraces to Rails.logger (#27257)
This commit introduces the following changes: 1. Introduce the `SignalTrapLogger` singleton which starts a single thread that polls a queue to log messages with the specified logger. This thread is necessary becasue most loggers cannot be used inside the `Signal.trap` context as they rely on mutexes which are not allowed within the context. 2. Moves the monkey patch in `freedom_patches/unicorn_http_server_patch.rb` to `config/unicorn.config.rb` which is already monkey patching `Unicorn::HttpServer`. 3. `Unicorn::HttpServer` will now automatically send a `USR2` signal to a unicorn worker 2 seconds before the worker is timed out by the Unicorn master. 4. When a Unicorn worker receives a `USR2` signal, it will now log only the main thread's backtraces to `Rails.logger`. Previously, it was `put`ing the backtraces to `STDOUT` which most people wouldn't read. Logging it via `Rails.logger` will make the backtraces easily accessible via `/logs`.
This commit is contained in:
committed by
GitHub
parent
4b2bd4d682
commit
23c38cbf11
@@ -9,9 +9,9 @@ class Demon::Base
|
||||
@demons
|
||||
end
|
||||
|
||||
def self.start(count = 1, verbose: false)
|
||||
def self.start(count = 1, verbose: false, logger: nil)
|
||||
@demons ||= {}
|
||||
count.times { |i| (@demons["#{prefix}_#{i}"] ||= new(i, verbose: verbose)).start }
|
||||
count.times { |i| (@demons["#{prefix}_#{i}"] ||= new(i, verbose:, logger:)).start }
|
||||
end
|
||||
|
||||
def self.stop
|
||||
@@ -39,7 +39,7 @@ class Demon::Base
|
||||
attr_reader :pid, :parent_pid, :started, :index
|
||||
attr_accessor :stop_timeout
|
||||
|
||||
def initialize(index, rails_root: nil, parent_pid: nil, verbose: false)
|
||||
def initialize(index, rails_root: nil, parent_pid: nil, verbose: false, logger: nil)
|
||||
@index = index
|
||||
@pid = nil
|
||||
@parent_pid = parent_pid || Process.pid
|
||||
@@ -47,6 +47,11 @@ class Demon::Base
|
||||
@stop_timeout = 10
|
||||
@rails_root = rails_root || Rails.root
|
||||
@verbose = verbose
|
||||
@logger = logger || Logger.new(STDERR)
|
||||
end
|
||||
|
||||
def log(message, level: :info)
|
||||
@logger.public_send(level, message)
|
||||
end
|
||||
|
||||
def pid_file
|
||||
@@ -72,6 +77,7 @@ class Demon::Base
|
||||
|
||||
def stop
|
||||
@started = false
|
||||
|
||||
if @pid
|
||||
Process.kill(stop_signal, @pid)
|
||||
|
||||
@@ -99,7 +105,7 @@ class Demon::Base
|
||||
wait_for_stop.call
|
||||
|
||||
if alive?
|
||||
STDERR.puts "Process would not terminate cleanly, force quitting. pid: #{@pid} #{self.class}"
|
||||
log("Process would not terminate cleanly, force quitting. pid: #{@pid} #{self.class}")
|
||||
Process.kill("KILL", @pid)
|
||||
end
|
||||
|
||||
@@ -125,8 +131,9 @@ class Demon::Base
|
||||
rescue StandardError
|
||||
-1
|
||||
end
|
||||
|
||||
if dead
|
||||
STDERR.puts "Detected dead worker #{@pid}, restarting..."
|
||||
log("Detected dead worker #{@pid}, restarting...")
|
||||
@pid = nil
|
||||
@started = false
|
||||
start
|
||||
@@ -138,7 +145,7 @@ class Demon::Base
|
||||
|
||||
if existing = already_running?
|
||||
# should not happen ... so kill violently
|
||||
STDERR.puts "Attempting to kill pid #{existing}"
|
||||
log("Attempting to kill pid #{existing}")
|
||||
Process.kill("TERM", existing)
|
||||
end
|
||||
|
||||
@@ -199,7 +206,7 @@ class Demon::Base
|
||||
Process.kill "KILL", Process.pid
|
||||
end
|
||||
rescue => e
|
||||
STDERR.puts "URGENT monitoring thread had an exception #{e}"
|
||||
log("URGENT monitoring thread had an exception #{e}")
|
||||
end
|
||||
sleep 1
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user