DEV: Log Unicorn worker timeout backtraces to Rails.logger (#27257)

This commit introduces the following changes:

1. Introduce the `SignalTrapLogger` singleton which starts a single
   thread that polls a queue to log messages with the specified logger.
   This thread is necessary becasue most loggers cannot be used inside
   the `Signal.trap` context as they rely on mutexes which are not
   allowed within the context.

2. Moves the monkey patch in `freedom_patches/unicorn_http_server_patch.rb` to
   `config/unicorn.config.rb` which is already monkey patching
   `Unicorn::HttpServer`.

3. `Unicorn::HttpServer` will now automatically send a `USR2` signal to
   a unicorn worker 2 seconds before the worker is timed out by the
   Unicorn master.

4. When a Unicorn worker receives a `USR2` signal, it will now log only
   the main thread's backtraces to `Rails.logger`. Previously, it was
   `put`ing the backtraces to `STDOUT` which most people wouldn't read.
   Logging it via `Rails.logger` will make the backtraces easily
   accessible via `/logs`.
This commit is contained in:
Alan Guo Xiang Tan
2024-06-03 12:51:12 +08:00
committed by GitHub
parent 4b2bd4d682
commit 23c38cbf11
6 changed files with 118 additions and 105 deletions

View File

@@ -9,9 +9,9 @@ class Demon::Base
@demons
end
def self.start(count = 1, verbose: false)
def self.start(count = 1, verbose: false, logger: nil)
@demons ||= {}
count.times { |i| (@demons["#{prefix}_#{i}"] ||= new(i, verbose: verbose)).start }
count.times { |i| (@demons["#{prefix}_#{i}"] ||= new(i, verbose:, logger:)).start }
end
def self.stop
@@ -39,7 +39,7 @@ class Demon::Base
attr_reader :pid, :parent_pid, :started, :index
attr_accessor :stop_timeout
def initialize(index, rails_root: nil, parent_pid: nil, verbose: false)
def initialize(index, rails_root: nil, parent_pid: nil, verbose: false, logger: nil)
@index = index
@pid = nil
@parent_pid = parent_pid || Process.pid
@@ -47,6 +47,11 @@ class Demon::Base
@stop_timeout = 10
@rails_root = rails_root || Rails.root
@verbose = verbose
@logger = logger || Logger.new(STDERR)
end
def log(message, level: :info)
@logger.public_send(level, message)
end
def pid_file
@@ -72,6 +77,7 @@ class Demon::Base
def stop
@started = false
if @pid
Process.kill(stop_signal, @pid)
@@ -99,7 +105,7 @@ class Demon::Base
wait_for_stop.call
if alive?
STDERR.puts "Process would not terminate cleanly, force quitting. pid: #{@pid} #{self.class}"
log("Process would not terminate cleanly, force quitting. pid: #{@pid} #{self.class}")
Process.kill("KILL", @pid)
end
@@ -125,8 +131,9 @@ class Demon::Base
rescue StandardError
-1
end
if dead
STDERR.puts "Detected dead worker #{@pid}, restarting..."
log("Detected dead worker #{@pid}, restarting...")
@pid = nil
@started = false
start
@@ -138,7 +145,7 @@ class Demon::Base
if existing = already_running?
# should not happen ... so kill violently
STDERR.puts "Attempting to kill pid #{existing}"
log("Attempting to kill pid #{existing}")
Process.kill("TERM", existing)
end
@@ -199,7 +206,7 @@ class Demon::Base
Process.kill "KILL", Process.pid
end
rescue => e
STDERR.puts "URGENT monitoring thread had an exception #{e}"
log("URGENT monitoring thread had an exception #{e}")
end
sleep 1
end