Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "DEV: Redo `DiscourseLogstashLogger` to not rely on `logstash-logger` (#27663)" (#27733) 

This reverts commit 8e10878e1a.

Something is broken on a friday so reverting first before I pick this up
again next Monday.
This commit is contained in:
Alan Guo Xiang Tan 2024-07-05 17:26:58 +08:00 committed by GitHub
parent 8b963986b3
commit 92d7d24d0f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 65 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Gemfile
View File

@ -212,6 +212,7 @@ gem "cppjieba_rb", require: false
gem "lograge", require: false
gem "logstash-event", require: false
gem "logstash-logger", require: false
gem "logster"
# A fork of sassc with dart-sass support

5
Gemfile.lock
View File

@ -218,7 +218,9 @@ GEM
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
logster (2.20.0)
logstash-logger (0.26.1)
logstash-event (~> 1.2)
logster (2.19.1)
loofah (2.22.0)
crass (~> 1.0.2)
nokogiri (>= 1.12.0)
@ -638,6 +640,7 @@ DEPENDENCIES
listen
lograge
logstash-event
logstash-logger
logster
loofah
lru_redux

3
config/environments/development.rb
View File

@ -44,7 +44,6 @@ Discourse::Application.configure do
config.log_level = ENV["DISCOURSE_DEV_LOG_LEVEL"] if ENV["DISCOURSE_DEV_LOG_LEVEL"]
config.active_record.logger = nil if ENV["RAILS_DISABLE_ACTIVERECORD_LOGS"] == "1"
config.active_record.verbose_query_logs = true if ENV["RAILS_VERBOSE_QUERY_LOGS"] == "1"
if defined?(BetterErrors)
@ -91,6 +90,8 @@ Discourse::Application.configure do
end
end
ActiveRecord::Base.logger = nil if ENV["RAILS_DISABLE_ACTIVERECORD_LOGS"] == "1"
if ENV["BULLET"]
Bullet.enable = true
Bullet.rails_logger = true

23
config/initializers/101-lograge.rb
View File

@ -102,29 +102,18 @@ Rails.application.config.to_prepare do
end
end
if ENV["ENABLE_LOGSTASH_LOGGER"] == "1"
if ENV["LOGSTASH_URI"]
config.lograge.formatter = Lograge::Formatters::Logstash.new
require "discourse_logstash_logger"
config.lograge.logger =
DiscourseLogstashLogger.logger(
logdev: Rails.root.join("log", "#{Rails.env}.log"),
type: :rails,
customize_event:
lambda do |event|
event["database"] = RailsMultisite::ConnectionManagement.current_db
end,
)
DiscourseLogstashLogger.logger(uri: ENV["LOGSTASH_URI"], type: :rails)
# Stop broadcasting to Rails' default logger
Rails.logger.stop_broadcasting_to(
Rails.logger.broadcasts.find { |logger| logger.is_a?(ActiveSupport::Logger) },
)
Logster.logger.subscribe do |severity, message, progname, opts, &block|
config.lograge.logger.add_with_opts(severity, message, progname, opts, &block)
end
# Remove ActiveSupport::Logger from the chain and replace with Lograge's
# logger
Rails.logger.stop_broadcasting_to(Rails.logger.broadcasts.first)
Rails.logger.broadcast_to(config.lograge.logger)
end
end
end

30
config/unicorn.conf.rb
View File

@ -1,6 +1,13 @@
# frozen_string_literal: true
# See http://unicorn.bogomips.org/Unicorn/Configurator.html
if (ENV["LOGSTASH_UNICORN_URI"] || "").length > 0
require_relative "../lib/discourse_logstash_logger"
require_relative "../lib/unicorn_logstash_patch"
logger DiscourseLogstashLogger.logger(uri: ENV["LOGSTASH_UNICORN_URI"], type: :unicorn)
end
discourse_path = File.expand_path(File.expand_path(File.dirname(__FILE__)) + "/../")
# tune down if not enough ram
@ -18,31 +25,18 @@ FileUtils.mkdir_p("#{discourse_path}/tmp/pids") if !File.exist?("#{discourse_pat
# feel free to point this anywhere accessible on the filesystem
pid(ENV["UNICORN_PID_PATH"] || "#{discourse_path}/tmp/pids/unicorn.pid")
if ENV["RAILS_ENV"] == "production"
if ENV["RAILS_ENV"] != "production"
logger Logger.new(STDOUT)
# we want a longer timeout in dev cause first request can be really slow
timeout(ENV["UNICORN_TIMEOUT"] && ENV["UNICORN_TIMEOUT"].to_i || 60)
else
# By default, the Unicorn logger will write to stderr.
# Additionally, some applications/frameworks log to stderr or stdout,
# so prevent them from going to /dev/null when daemonized here:
stderr_path "#{discourse_path}/log/unicorn.stderr.log"
stdout_path "#{discourse_path}/log/unicorn.stdout.log"
# nuke workers after 30 seconds instead of 60 seconds (the default)
timeout 30
else
# we want a longer timeout in dev cause first request can be really slow
timeout(ENV["UNICORN_TIMEOUT"] && ENV["UNICORN_TIMEOUT"].to_i || 60)
end
enable_logstash_logger = ENV["ENABLE_LOGSTASH_LOGGER"] == "1"
if enable_logstash_logger
require_relative "../lib/discourse_logstash_logger"
require_relative "../lib/unicorn_logstash_patch"
logger DiscourseLogstashLogger.logger(
logdev: "#{discourse_path}/log/unicorn.stderr.log",
type: :unicorn,
)
else
logger Logger.new(STDOUT)
end
# important for Ruby 2.0

30
lib/discourse.rb
View File

@ -4,7 +4,6 @@ require "cache"
require "open3"
require "plugin/instance"
require "version"
require "git_utils"
module Discourse
DB_POST_MIGRATE_PATH ||= "db/post_migrate"
@ -830,23 +829,42 @@ module Discourse
end
def self.git_version
@git_version ||= GitUtils.git_version
@git_version ||=
begin
git_cmd = "git rev-parse HEAD"
self.try_git(git_cmd, Discourse::VERSION::STRING)
end
end
def self.git_branch
@git_branch ||= GitUtils.git_branch
@git_branch ||=
self.try_git("git branch --show-current", nil) ||
self.try_git("git config user.discourse-version", "unknown")
end
def self.full_version
@full_version ||= GitUtils.full_version
@full_version ||=
begin
git_cmd = 'git describe --dirty --match "v[0-9]*" 2> /dev/null'
self.try_git(git_cmd, "unknown")
end
end
def self.last_commit_date
@last_commit_date ||= GitUtils.last_commit_date
@last_commit_date ||=
begin
git_cmd = 'git log -1 --format="%ct"'
seconds = self.try_git(git_cmd, nil)
seconds.nil? ? nil : DateTime.strptime(seconds, "%s")
end
end
def self.try_git(git_cmd, default_value)
GitUtils.try_git(git_cmd, default_value)
begin
`#{git_cmd}`.strip
rescue StandardError
default_value
end.presence || default_value
end
# Either returns the site_contact_username user or the first admin.

109
lib/discourse_logstash_logger.rb
View File

@ -1,101 +1,18 @@
# frozen_string_literal: true
require "json"
require "socket"
require "logstash-logger"
class DiscourseLogstashLogger < Logger
PROCESS_PID = Process.pid
HOST = Socket.gethostname
attr_accessor :customize_event, :type
# Creates a new logger instance.
#
# @param logdev [String, IO, nil] The log device. This can be one of:
# - A string filepath: entries are written to the file at that path. If the file exists, new entries are appended.
# - An IO stream (typically +$stdout+, +$stderr+, or an open file): entries are written to the given stream.
# - nil or File::NULL: no entries are written.
# @param type [String] The type of log messages. This will add a `type` field to all log messages.
# @param customize_event [Proc, nil] A proc that customizes the log event before it is written to the log device.
# The proc is called with a hash of log event data and can be modified in place.
#
# @return [Logger] A new logger instance with the specified log device and type.
def self.logger(logdev:, type:, customize_event: nil)
logger = self.new(logdev)
logger.type = type
logger.customize_event = customize_event if customize_event
logger
end
# :nodoc:
def add(*args, &block)
add_with_opts(*args, &block)
end
ALLOWED_HEADERS_FROM_ENV = %w[
REQUEST_URI
REQUEST_METHOD
HTTP_HOST
HTTP_USER_AGENT
HTTP_ACCEPT
HTTP_REFERER
HTTP_X_FORWARDED_FOR
HTTP_X_REAL_IP
]
# :nodoc:
def add_with_opts(severity, message = nil, progname = nil, opts = {}, &block)
return true if @logdev.nil? || severity < @level
progname = @progname if progname.nil?
if message.nil?
if block_given?
message = yield
else
message = progname
progname = @progname
end
end
event = {
"message" => message,
"severity" => severity,
"severity_name" => Logger::SEV_LABEL[severity],
"pid" => PROCESS_PID,
"type" => @type.to_s,
"host" => HOST,
}
# Only log backtrace and env for Logger::WARN and above.
# Backtrace is just noise for anything below that.
if severity >= Logger::WARN
if (backtrace = opts&.dig(:backtrace)).present?
event["backtrace"] = backtrace
end
if (env = opts&.dig(:env)).present?
ALLOWED_HEADERS_FROM_ENV.each do |header|
event["request.headers.#{header.downcase}"] = opts[:env][header]
end
end
end
if message.start_with?("{") && message.end_with?("}")
begin
parsed = JSON.parse(message)
event["message"] = parsed.delete("message") if parsed["message"]
event.merge!(parsed)
event
rescue JSON::ParserError
# Do nothing
end
end
@customize_event.call(event) if @customize_event
@logdev.write("#{event.to_json}\n")
rescue Exception => e
STDERR.puts "Error logging message in DiscourseLogstashLogger: #{message} (#{e.message})\n#{e.backtrace.join("\n")}"
class DiscourseLogstashLogger
def self.logger(uri:, type:)
LogStashLogger.new(
uri: uri,
sync: true,
customize_event: ->(event) do
event["severity_name"] = event["severity"]
event["severity"] = Object.const_get("Logger::Severity::#{event["severity"]}")
event["type"] = type
event["pid"] = Process.pid
end,
)
end
end

5
lib/version.rb
View File

@ -3,6 +3,7 @@
module Discourse
VERSION_REGEXP ||= /\A\d+\.\d+\.\d+(\.beta\d+)?\z/
VERSION_COMPATIBILITY_FILENAME ||= ".discourse-compatibility"
# work around reloader
unless defined?(::Discourse::VERSION)
module VERSION #:nodoc:
@ -15,8 +16,8 @@ module Discourse
MAJOR = PARTS[0].to_i
MINOR = PARTS[1].to_i
TINY = PARTS[2].to_i
PRE = PARTS[3]&.split("-", 2)&.[](0)
DEV = PARTS[3]&.split("-", 2)&.[](1)
PRE = PARTS[3]&.split("-", 2)&.first
DEV = PARTS[3]&.split("-", 2)&.second
end
end