mirror of
https://github.com/discourse/discourse.git
synced 2024-11-28 03:33:58 -06:00
4c0af24173
This commit adds a `MiniSchedulerLongRunningJobLogger` class which will poll every 60 seconds for mini_scheduler jobs which are stuck. When it detects that a job is stuck, it will log a warning message with the current backtrace of the thread that is executing the job. Note that for scheduled jobs which are executed at a frequency of less than 30 minutes, we will log when the job has been executing for 30 minutes. For scheduled jobs executed at a frequency of less than 2 hours, we will log when the job has been executing for a duration greater than its specified frequency. For scheduled jobs executed at a frequency greater than 2 hours, we will log as long as the job has been executing for more than 2 hours.
102 lines
3.0 KiB
Ruby
102 lines
3.0 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require "sidekiq/pausable"
|
|
require "sidekiq_logster_reporter"
|
|
require "sidekiq_long_running_job_logger"
|
|
require "mini_scheduler_long_running_job_logger"
|
|
|
|
Sidekiq.configure_client { |config| config.redis = Discourse.sidekiq_redis_config }
|
|
|
|
Sidekiq.configure_server do |config|
|
|
config.redis = Discourse.sidekiq_redis_config
|
|
|
|
config.server_middleware { |chain| chain.add Sidekiq::Pausable }
|
|
|
|
if stuck_sidekiq_job_minutes = GlobalSetting.sidekiq_report_long_running_jobs_minutes
|
|
config.on(:startup) { SidekiqLongRunningJobLogger.new(stuck_sidekiq_job_minutes:).start }
|
|
end
|
|
end
|
|
|
|
if Sidekiq.server?
|
|
module Sidekiq
|
|
class CLI
|
|
private
|
|
|
|
def print_banner
|
|
# banner takes up too much space
|
|
end
|
|
end
|
|
end
|
|
|
|
Rails.application.config.after_initialize do
|
|
# defer queue should simply run in sidekiq
|
|
Scheduler::Defer.async = false
|
|
|
|
# warm up AR
|
|
RailsMultisite::ConnectionManagement.safe_each_connection do
|
|
(ActiveRecord::Base.connection.tables - %w[schema_migrations versions]).each do |table|
|
|
begin
|
|
table.classify.constantize.first
|
|
rescue StandardError
|
|
nil
|
|
end
|
|
end
|
|
end
|
|
|
|
scheduler_hostname = ENV["UNICORN_SCHEDULER_HOSTNAME"]
|
|
|
|
if !scheduler_hostname || scheduler_hostname.split(",").include?(Discourse.os_hostname)
|
|
begin
|
|
MiniScheduler.start(workers: GlobalSetting.mini_scheduler_workers)
|
|
|
|
MiniSchedulerLongRunningJobLogger.new(
|
|
poll_interval_seconds:
|
|
ENV["DISCOURSE_MINI_SCHEDULER_LONG_RUNNING_JOB_LOGGER_POLL_INTERVAL_SECONDS"],
|
|
).start
|
|
rescue MiniScheduler::DistributedMutex::Timeout
|
|
sleep 5
|
|
retry
|
|
end
|
|
end
|
|
end
|
|
else
|
|
# Sidekiq#logger= applies patches to whichever logger we pass it.
|
|
# Therefore something like Sidekiq.logger = Rails.logger will break
|
|
# all logging in the application.
|
|
#
|
|
# Instead, this patch adds a dedicated logger instance and patches
|
|
# the #add method to forward messages to Rails.logger.
|
|
Sidekiq.logger = Logger.new(nil)
|
|
Sidekiq
|
|
.logger
|
|
.define_singleton_method(:add) do |severity, message = nil, progname = nil, &blk|
|
|
Rails.logger.add(severity, message, progname, &blk)
|
|
end
|
|
end
|
|
|
|
Sidekiq.error_handlers.clear
|
|
Sidekiq.error_handlers << SidekiqLogsterReporter.new
|
|
|
|
Sidekiq.strict_args!
|
|
|
|
Rails.application.config.to_prepare do
|
|
# Ensure that scheduled jobs are loaded before mini_scheduler is configured.
|
|
Dir.glob("#{Rails.root}/app/jobs/scheduled/*.rb") { |f| require(f) } if Rails.env.development?
|
|
|
|
MiniScheduler.configure do |config|
|
|
config.redis = Discourse.redis
|
|
|
|
config.job_exception_handler { |ex, context| Discourse.handle_job_exception(ex, context) }
|
|
|
|
config.job_ran { |stat| DiscourseEvent.trigger(:scheduled_job_ran, stat) }
|
|
|
|
config.skip_schedule { Sidekiq.paused? }
|
|
|
|
config.before_sidekiq_web_request do
|
|
RailsMultisite::ConnectionManagement.establish_connection(
|
|
db: RailsMultisite::ConnectionManagement::DEFAULT,
|
|
)
|
|
end
|
|
end
|
|
end
|