From b89c9f65129bc57bae9ce4428bf6e13e6200961c Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Fri, 23 Apr 2021 15:27:48 +0900 Subject: [PATCH 1/4] Enable system.log.rotate_... system configuration In the previous versions, log rotation options were supported as --log-rotate-age or --log-rotate-size via command line options. On Windows, as fluentd is launched as a windows service, it is required to configure again via --reg-winsvc-fluentdopt or edit fluentdopt registry key for customization. This approach is not convenient for Windows users, so it may be better to support more comprehensive solution - customize via configuration file. This commit introduces such a configurable parameter in .conf rotate_age 5 rotate_size 1048576 Signed-off-by: Kentaro Hayashi --- lib/fluent/supervisor.rb | 15 +++++++++++++++ lib/fluent/system_config.rb | 14 ++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index 86cb9f3556..8cc9df021d 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -518,6 +518,8 @@ def init(process_type, worker_id) dl_opts = {} # subtract 1 to match serverengine daemon logger side logging severity. dl_opts[:log_level] = @level - 1 + dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age + dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts) $log = Fluent::Log.new(logger, @opts) $log.enable_color(false) if @path @@ -606,6 +608,19 @@ def initialize(opt) @cl_opt = opt @conf = nil + # parse configuration immediately to initialize logger in early stage + if @config_path and File.exist?(@config_path) + @conf = Fluent::Config.build(config_path: @config_path, + encoding: @conf_encoding ? @conf_encoding : 'utf-8', + additional_config: @inline_config ? @inline_config : nil, + use_v1_config: !!@use_v1_config) + @system_config = build_system_config(@conf) + if @system_config.log + @log_rotate_age ||= @system_config.log.rotate_age + @log_rotate_size ||= @system_config.log.rotate_size + end + @conf = nil + end log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval], ignore_same_log_interval: opt[:ignore_same_log_interval]} diff --git a/lib/fluent/system_config.rb b/lib/fluent/system_config.rb index 041ecaeea9..746268854c 100644 --- a/lib/fluent/system_config.rb +++ b/lib/fluent/system_config.rb @@ -55,6 +55,20 @@ class SystemConfig config_section :log, required: false, init: true, multi: false do config_param :format, :enum, list: [:text, :json], default: :text config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z' + config_param :rotate_age, default: nil do |v| + if %w(daily weekly monthly).include?(v) + v.to_sym + else + begin + Integer(v) + rescue ArgumentError => e + raise Fluent::ConfigError, e.message + else + v.to_i + end + end + end + config_param :rotate_size, :size, default: nil end config_section :counter_server, multi: false do From 01648de274d881e773d4bfb02a00110fc7f2b1e4 Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Thu, 6 May 2021 17:24:14 +0900 Subject: [PATCH 2/4] test: add rotation system config test case Signed-off-by: Kentaro Hayashi --- test/config/test_system_config.rb | 46 +++++++++++++++++++++++++++++++ test/test_supervisor.rb | 35 +++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/test/config/test_system_config.rb b/test/config/test_system_config.rb index 675311ab3b..0e992fa310 100644 --- a/test/config/test_system_config.rb +++ b/test/config/test_system_config.rb @@ -143,5 +143,51 @@ def parse_text(text) sc.overwrite_variables(**s.for_system_config) assert_equal(level, sc.log_level) end + + sub_test_case "log rotation" do + data('daily' => "daily", + 'weekly' => 'weekly', + 'monthly' => 'monthly') + test "symbols for rotate_age" do |age| + conf = parse_text(<<-EOS) + + + rotate_age #{age} + + + EOS + sc = Fluent::SystemConfig.new(conf) + assert_equal(age.to_sym, sc.log.rotate_age) + end + + test "numeric number for rotate age" do + conf = parse_text(<<-EOS) + + + rotate_age 3 + + + EOS + s = FakeSupervisor.new + sc = Fluent::SystemConfig.new(conf) + assert_equal(3, sc.log.rotate_age) + end + + data(h: ['100', 100], + k: ['1k', 1024], + m: ['1m', 1024 * 1024], + g: ['1g', 1024 * 1024 * 1024]) + test "numeric and SI prefix for rotate_size" do |(label, size)| + conf = parse_text(<<-EOS) + + + rotate_size #{label} + + + EOS + sc = Fluent::SystemConfig.new(conf) + assert_equal(size, sc.log.rotate_size) + end + end end end diff --git a/test/test_supervisor.rb b/test/test_supervisor.rb index 26025f5810..cea498e2b4 100644 --- a/test/test_supervisor.rb +++ b/test/test_supervisor.rb @@ -7,6 +7,7 @@ require 'net/http' require 'uri' require 'fileutils' +require 'tempfile' if Fluent.windows? require 'win32/event' @@ -489,6 +490,40 @@ def test_logger_with_rotate_age_and_rotate_size(rotate_age) assert_equal 10, $log.out.instance_variable_get(:@shift_size) end + sub_test_case "system log rotation" do + def parse_text(text) + basepath = File.expand_path(File.dirname(__FILE__) + '/../../') + Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' } + end + + def test_override_default_log_rotate + Tempfile.open do |file| + config = parse_text(<<-EOS) + + + rotate_age 3 + rotate_size 300 + + + EOS + file.puts(config) + file.flush + opts = Fluent::Supervisor.default_options.merge( + log_path: "#{TMP_DIR}/test.log", config_path: file.path + ) + sv = Fluent::Supervisor.new(opts) + + log = sv.instance_variable_get(:@log) + log.init(:standalone, 0) + logger = $log.instance_variable_get(:@logger) + + assert_equal([3, 300], + [logger.instance_variable_get(:@rotate_age), + logger.instance_variable_get(:@rotate_size)]) + end + end + end + def test_inline_config omit 'this feature is deprecated. see https://github.com/fluent/fluentd/issues/2711' From 9a10164bdd03fe848138cb3774f609d4c66ec12b Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Mon, 10 May 2021 16:45:18 +0900 Subject: [PATCH 3/4] test: declare that test case use v0.10 explicitly In the previous versions, Fluent::Supervisor.new(opts) doesn't parse content actually at that time, so it doesn't care whether v0 or v1 parser is used. But, by introducing support for log rotation configuration, it was changed to fetch rotate_age and rotate_size in early stage because these parameters must be passed to LogInitializer during initialization steps, so configuration file is changed to parse in Fluent::Supervisor.new(opts). As a result, we need to pass additional parameter for this test case. Without this fix, log rotate configuration in support raises the following exception: Failure: test_inline(ConfigTest): Exception raised: Fluent::ConfigParseError(' but got end of file at config_test_1.conf line 11,10 10: port 2222 11: ----------^ >) Signed-off-by: Kentaro Hayashi --- test/test_config.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_config.rb b/test/test_config.rb index 1efb79edd2..e9d1005647 100644 --- a/test/test_config.rb +++ b/test/test_config.rb @@ -160,7 +160,8 @@ def test_inline prepare_config opts = { :config_path => "#{TMP_DIR}/config_test_1.conf", - :inline_config => "\n type http\n port 2222\n " + :inline_config => "\n type http\n port 2222\n ", + :use_v1_config => false } assert_nothing_raised do Fluent::Supervisor.new(opts) From 627c841d671366822fffac24bf3fcde092afbaed Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Thu, 13 May 2021 16:03:39 +0900 Subject: [PATCH 4/4] unify Fluent::Log::LOG_ROTATE_AGE for consistency Signed-off-by: Kentaro Hayashi --- lib/fluent/command/fluentd.rb | 3 +-- lib/fluent/log.rb | 1 + lib/fluent/system_config.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/fluent/command/fluentd.rb b/lib/fluent/command/fluentd.rb index e66285bf81..2f08689b07 100644 --- a/lib/fluent/command/fluentd.rb +++ b/lib/fluent/command/fluentd.rb @@ -85,9 +85,8 @@ opts[:log_path] = s } -ROTATE_AGE = %w(daily weekly monthly) op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age| - if ROTATE_AGE.include?(age) + if Fluent::Log::LOG_ROTATE_AGE.include?(age) opts[:log_rotate_age] = age else begin diff --git a/lib/fluent/log.rb b/lib/fluent/log.rb index 4c096a615e..201b2d88a6 100644 --- a/lib/fluent/log.rb +++ b/lib/fluent/log.rb @@ -49,6 +49,7 @@ module TTYColor LOG_TYPE_DEFAULT = :default # show logs in all supervisor/workers, with worker id in workers (default) LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze + LOG_ROTATE_AGE = %w(daily weekly monthly) def self.str_to_level(log_level_str) case log_level_str.downcase diff --git a/lib/fluent/system_config.rb b/lib/fluent/system_config.rb index 746268854c..b322b4bb60 100644 --- a/lib/fluent/system_config.rb +++ b/lib/fluent/system_config.rb @@ -56,7 +56,7 @@ class SystemConfig config_param :format, :enum, list: [:text, :json], default: :text config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z' config_param :rotate_age, default: nil do |v| - if %w(daily weekly monthly).include?(v) + if Fluent::Log::LOG_ROTATE_AGE.include?(v) v.to_sym else begin