DEV: Redo DiscourseLogstashLogger to not rely on logstash-logger (#27663)

This commit rewrites `DiscourseLogstashLogger` to not be an instance
of `LogstashLogger`. The reason we don't want it to be an instance of
`LogstashLogger` is because we want the new logger to be chained to
Logster's logger which can then pass down useful information like the
request's env and error backtraces which Logster has already gathered.

Note that this commit does not bother to maintain backwards
compatibility and drops the `LOGSTASH_URI` and `UNICORN_LOGSTASH_URI`
ENV variables which were previously used to configure the destination in
which `logstash-logger` would send the logs to. Instead, we introduce
the `ENABLE_LOGSTASH_LOGGER` ENV variable to replace both ENV and remove
the need for the log paths to be specified. Note that the previous
feature was considered experimental as stated in d888d3c54c
and the new feature should be considered experimental as well. The code
may be moved into a plugin in the future.

config/environments/development.rb

@@ -44,6 +44,7 @@ Discourse::Application.configure do
 
   config.log_level = ENV["DISCOURSE_DEV_LOG_LEVEL"] if ENV["DISCOURSE_DEV_LOG_LEVEL"]
 
+  config.active_record.logger = nil if ENV["RAILS_DISABLE_ACTIVERECORD_LOGS"] == "1"
   config.active_record.verbose_query_logs = true if ENV["RAILS_VERBOSE_QUERY_LOGS"] == "1"
 
   if defined?(BetterErrors)
@@ -90,8 +91,6 @@ Discourse::Application.configure do
       end
     end
 
-    ActiveRecord::Base.logger = nil if ENV["RAILS_DISABLE_ACTIVERECORD_LOGS"] == "1"
-
     if ENV["BULLET"]
       Bullet.enable = true
       Bullet.rails_logger = true

config/initializers/101-lograge.rb

@@ -102,18 +102,29 @@ Rails.application.config.to_prepare do
           end
         end
 
-      if ENV["LOGSTASH_URI"]
+      if ENV["ENABLE_LOGSTASH_LOGGER"] == "1"
         config.lograge.formatter = Lograge::Formatters::Logstash.new
 
         require "discourse_logstash_logger"
 
         config.lograge.logger =
-          DiscourseLogstashLogger.logger(uri: ENV["LOGSTASH_URI"], type: :rails)
+          DiscourseLogstashLogger.logger(
+            logdev: Rails.root.join("log", "#{Rails.env}.log"),
+            type: :rails,
+            customize_event:
+              lambda do |event|
+                event["database"] = RailsMultisite::ConnectionManagement.current_db
+              end,
+          )
 
-        # Remove ActiveSupport::Logger from the chain and replace with Lograge's
-        # logger
-        Rails.logger.stop_broadcasting_to(Rails.logger.broadcasts.first)
-        Rails.logger.broadcast_to(config.lograge.logger)
+        # Stop broadcasting to Rails' default logger
+        Rails.logger.stop_broadcasting_to(
+          Rails.logger.broadcasts.find { |logger| logger.is_a?(ActiveSupport::Logger) },
+        )
+
+        Logster.logger.subscribe do |severity, message, progname, opts, &block|
+          config.lograge.logger.add_with_opts(severity, message, progname, opts, &block)
+        end
       end
     end
   end

config/unicorn.conf.rb

@@ -1,13 +1,6 @@
 # frozen_string_literal: true
 
 # See http://unicorn.bogomips.org/Unicorn/Configurator.html
-
-if (ENV["LOGSTASH_UNICORN_URI"] || "").length > 0
-  require_relative "../lib/discourse_logstash_logger"
-  require_relative "../lib/unicorn_logstash_patch"
-  logger DiscourseLogstashLogger.logger(uri: ENV["LOGSTASH_UNICORN_URI"], type: :unicorn)
-end
-
 discourse_path = File.expand_path(File.expand_path(File.dirname(__FILE__)) + "/../")
 
 # tune down if not enough ram
@@ -25,18 +18,31 @@ FileUtils.mkdir_p("#{discourse_path}/tmp/pids") if !File.exist?("#{discourse_pat
 # feel free to point this anywhere accessible on the filesystem
 pid(ENV["UNICORN_PID_PATH"] || "#{discourse_path}/tmp/pids/unicorn.pid")
 
-if ENV["RAILS_ENV"] != "production"
-  logger Logger.new(STDOUT)
-  # we want a longer timeout in dev cause first request can be really slow
-  timeout(ENV["UNICORN_TIMEOUT"] && ENV["UNICORN_TIMEOUT"].to_i || 60)
-else
+if ENV["RAILS_ENV"] == "production"
   # By default, the Unicorn logger will write to stderr.
   # Additionally, some applications/frameworks log to stderr or stdout,
   # so prevent them from going to /dev/null when daemonized here:
   stderr_path "#{discourse_path}/log/unicorn.stderr.log"
   stdout_path "#{discourse_path}/log/unicorn.stdout.log"
+
   # nuke workers after 30 seconds instead of 60 seconds (the default)
   timeout 30
+else
+  # we want a longer timeout in dev cause first request can be really slow
+  timeout(ENV["UNICORN_TIMEOUT"] && ENV["UNICORN_TIMEOUT"].to_i || 60)
+end
+
+enable_logstash_logger = ENV["ENABLE_LOGSTASH_LOGGER"] == "1"
+
+if enable_logstash_logger
+  require_relative "../lib/discourse_logstash_logger"
+  require_relative "../lib/unicorn_logstash_patch"
+  logger DiscourseLogstashLogger.logger(
+           logdev: "#{discourse_path}/log/unicorn.stderr.log",
+           type: :unicorn,
+         )
+else
+  logger Logger.new(STDOUT)
 end
 
 # important for Ruby 2.0