From 86062182a47baadb9a45a1f0c6695a3c836f99ec Mon Sep 17 00:00:00 2001 From: Yuta Iwama Date: Wed, 5 Feb 2020 18:38:36 +0900 Subject: [PATCH] yaml config file Signed-off-by: Yuta Iwama --- lib/fluent/command/fluentd.rb | 8 ++ lib/fluent/config.rb | 8 +- lib/fluent/config/yaml_parser.rb | 44 ++++++ lib/fluent/config/yaml_parser/fluent_value.rb | 47 +++++++ lib/fluent/config/yaml_parser/loader.rb | 91 +++++++++++++ lib/fluent/config/yaml_parser/parser.rb | 126 ++++++++++++++++++ .../config/yaml_parser/section_builder.rb | 107 +++++++++++++++ lib/fluent/supervisor.rb | 9 +- 8 files changed, 438 insertions(+), 2 deletions(-) create mode 100644 lib/fluent/config/yaml_parser.rb create mode 100644 lib/fluent/config/yaml_parser/fluent_value.rb create mode 100644 lib/fluent/config/yaml_parser/loader.rb create mode 100644 lib/fluent/config/yaml_parser/parser.rb create mode 100644 lib/fluent/config/yaml_parser/section_builder.rb diff --git a/lib/fluent/command/fluentd.rb b/lib/fluent/command/fluentd.rb index d14d8fece6..8943dda672 100644 --- a/lib/fluent/command/fluentd.rb +++ b/lib/fluent/command/fluentd.rb @@ -123,6 +123,14 @@ opts[:without_source] = b } +op.on('--config-file-type VALU', 'file type of fluentd. yaml or yml') { |s| + if (s == 'yaml') || (s == 'yml') + opts[:config_file_type] = s.to_sym + else + usage '--config-file-type accepts yaml or yml' + end +} + op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b| opts[:use_v1_config] = b } diff --git a/lib/fluent/config.rb b/lib/fluent/config.rb index b2411853e1..60fd90b997 100644 --- a/lib/fluent/config.rb +++ b/lib/fluent/config.rb @@ -17,6 +17,7 @@ require 'fluent/config/error' require 'fluent/config/element' require 'fluent/configurable' +require 'fluent/config/yaml_parser' module Fluent module Config @@ -25,7 +26,11 @@ module Config # @param additional_config [String] config which is added to last of config body # @param use_v1_config [Bool] config is formatted with v1 or not # @return [Fluent::Config] - def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true) + def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil) + if type == :yaml + return Fluent::Config::YamlParser.parse(config_path) + end + config_fname = File.basename(config_path) config_basedir = File.dirname(config_path) config_data = File.open(config_path, "r:#{encoding}:utf-8") do |f| @@ -36,6 +41,7 @@ def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_c end s end + Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config) end diff --git a/lib/fluent/config/yaml_parser.rb b/lib/fluent/config/yaml_parser.rb new file mode 100644 index 0000000000..07ef8a414c --- /dev/null +++ b/lib/fluent/config/yaml_parser.rb @@ -0,0 +1,44 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/config/yaml_parser/loader' +require 'fluent/config/yaml_parser/parser' +require 'pathname' + +module Fluent + module Config + module YamlParser + def self.parse(path) + context = Kernel.binding + + unless context.respond_to?(:use_nil) + context.define_singleton_method(:use_nil) do + raise Fluent::SetNil + end + end + + unless context.respond_to?(:use_default) + context.define_singleton_method(:use_default) do + raise Fluent::SetDefault + end + end + + s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path)) + Fluent::Config::YamlParser::Parser.new(s).build.to_element + end + end + end +end diff --git a/lib/fluent/config/yaml_parser/fluent_value.rb b/lib/fluent/config/yaml_parser/fluent_value.rb new file mode 100644 index 0000000000..69108d8bd1 --- /dev/null +++ b/lib/fluent/config/yaml_parser/fluent_value.rb @@ -0,0 +1,47 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Fluent + module Config + module YamlParser + module FluentValue + JsonValue = Struct.new(:val) do + def to_s + val.to_json + end + + def to_element + to_s + end + end + + StringValue = Struct.new(:val, :context) do + def to_s + context.instance_eval("\"#{val}\"") + rescue Fluent::SetNil => _ + '' + rescue Fluent::SetDefault => _ + ':default' + end + + def to_element + to_s + end + end + end + end + end +end diff --git a/lib/fluent/config/yaml_parser/loader.rb b/lib/fluent/config/yaml_parser/loader.rb new file mode 100644 index 0000000000..22f32885e4 --- /dev/null +++ b/lib/fluent/config/yaml_parser/loader.rb @@ -0,0 +1,91 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'psych' +require 'json' +require 'fluent/config/error' +require 'fluent/config/yaml_parser/fluent_value' + +# Based on https://github.com/eagletmt/hako/blob/34cdde06fe8f3aeafd794be830180c3cedfbb4dc/lib/hako/yaml_loader.rb + +module Fluent + module Config + module YamlParser + class Loader + INCLUDE_TAG = 'tag:include'.freeze + FLUENT_JSON_TAG = 'tag:fluent/json'.freeze + FLUENT_STR_TAG = 'tag:fluent/s'.freeze + SHOVEL = '<<'.freeze + + def initialize(context = Kernel.binding) + @context = context + @current_path = nil + end + + # @param [String] path + # @return [Hash] + def load(path) + class_loader = Psych::ClassLoader.new + scanner = Psych::ScalarScanner.new(class_loader) + + visitor = Visitor.new(scanner, class_loader) + + visitor._register_domain(INCLUDE_TAG) do |_, val| + load(path.parent.join(val)) + end + + visitor._register_domain(FLUENT_JSON_TAG) do |_, val| + Fluent::Config::YamlParser::FluentValue::JsonValue.new(val) + end + + visitor._register_domain(FLUENT_STR_TAG) do |_, val| + Fluent::Config::YamlParser::FluentValue::StringValue.new(val, @context) + end + + path.open do |f| + visitor.accept(Psych.parse(f)) + end + end + + class Visitor < Psych::Visitors::ToRuby + def initialize(scanner, class_loader) + super(scanner, class_loader) + end + + def _register_domain(name, &block) + @domain_types.merge!({ name => [name, block] }) + end + + def revive_hash(hash, o) + super(hash, o).tap do |r| + if r[SHOVEL].is_a?(Hash) + h2 = {} + r.each do |k, v| + if k == SHOVEL + h2.merge!(v) + else + h2[k] = v + end + end + r.replace(h2) + end + end + end + end + end + end + end +end diff --git a/lib/fluent/config/yaml_parser/parser.rb b/lib/fluent/config/yaml_parser/parser.rb new file mode 100644 index 0000000000..939798690a --- /dev/null +++ b/lib/fluent/config/yaml_parser/parser.rb @@ -0,0 +1,126 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'fluent/config/yaml_parser/section_builder' + +module Fluent + module Config + module YamlParser + class Parser + def initialize(config, indent: 2) + @base_indent = indent + @config = config + end + + def build + s = @config['system'] && system_config_build(@config['system']) + c = @config['config'] && config_build(@config['config'], root: true) + RootBuilder.new(s, c) + end + + private + + def system_config_build(config) + section_build('system', config) + end + + def config_build(config, indent: 0, root: false) + sb = SectionBodyBuilder.new(indent, root: root) + config.each do |c| + if (lc = c.delete('label')) + sb.add_section(label_build(lc, indent: indent)) + end + + if (sc = c.delete('source')) + sb.add_section(source_build(sc, indent: indent)) + end + + if (fc = c.delete('filter')) + sb.add_section(filter_build(fc, indent: indent)) + end + + if (mc = c.delete('match')) + sb.add_section(match_build(mc, indent: indent)) + end + + if (wc = c.delete('worker')) + sb.add_section(worker_build(wc, indent: indent)) + end + end + + sb + end + + def label_build(config, indent: 0) + config = config.dup + name = config.delete('$name') + c = config.delete('config') + SectionBuilder.new('label', config_build(c, indent: indent + @base_indent), indent, name) + end + + def worker_build(config, indent: 0) + config = config.dup + num = config.delete('$arg') + c = config.delete('config') + SectionBuilder.new('worker', config_build(c, indent: indent + @base_indent), indent, num) + end + + def source_build(config, indent: 0) + section_build('source', config, indent: indent) + end + + def filter_build(config, indent: 0) + config = config.dup + tag = config.delete('$tag') + section_build('filter', config, indent: indent, arg: tag) + end + + def match_build(config, indent: 0) + config = config.dup + tag = config.delete('$tag') + section_build('match', config, indent: indent, arg: tag) + end + + def section_build(name, config, indent: 0, arg: nil) + sb = SectionBodyBuilder.new(indent + @base_indent) + + if (v = config.delete('$type')) + sb.add_line('@type', v) + end + + if (v = config.delete('$label')) + sb.add_line('@label', v) + end + + config.each do |key, val| + if val.is_a?(Array) + val.each do |v| + sb.add_section(section_build(key, v, indent: indent + @base_indent)) + end + elsif val.is_a?(Hash) + harg = val.delete('$arg') + sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg)) + else + sb.add_line(key, val) + end + end + + SectionBuilder.new(name, sb, indent, arg) + end + end + end + end +end diff --git a/lib/fluent/config/yaml_parser/section_builder.rb b/lib/fluent/config/yaml_parser/section_builder.rb new file mode 100644 index 0000000000..58a89e6cd9 --- /dev/null +++ b/lib/fluent/config/yaml_parser/section_builder.rb @@ -0,0 +1,107 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Fluent + module Config + module YamlParser + SectionBuilder = Struct.new(:name, :body, :indent_size, :arg) do + def to_s + indent = ' ' * indent_size + + if arg && !arg.to_s.empty? + "#{indent}<#{name} #{arg}>\n#{body}\n#{indent}" + else + "#{indent}<#{name}>\n#{body}\n#{indent}" + end + end + + def to_element + elem = body.to_element + elem.name = name + elem.arg = arg.to_s if arg + elem.v1_config = true + elem + end + end + + class RootBuilder + def initialize(system, conf) + @system = system + @conf = conf + end + + attr_reader :system, :conf + + def to_element + Fluent::Config::Element.new('ROOT', '', {}, [@system, @conf].compact.map(&:to_element).flatten) + end + + def to_s + s = StringIO.new(+'') + s.puts(@system.to_s) if @system + s.puts(@conf.to_s) if @conf + + s.string + end + end + + class SectionBodyBuilder + Row = Struct.new(:key, :value, :indent) do + def to_s + "#{indent}#{key} #{value}" + end + end + + def initialize(indent, root: false) + @indent = ' ' * indent + @bodies = [] + @root = root + end + + def add_line(k, v) + @bodies << Row.new(k, v, @indent) + end + + def add_section(section) + @bodies << section + end + + def to_element + if @root + return @bodies.map(&:to_element) + end + + not_section, section = @bodies.partition { |e| e.is_a?(Row) } + r = {} + not_section.each do |e| + v = e.value + r[e.key] = v.respond_to?(:to_element) ? v.to_element : v + end + + if @root + section.map(&:to_element) + else + Fluent::Config::Element.new('', '', r, section.map(&:to_element)) + end + end + + def to_s + @bodies.map(&:to_s).join("\n") + end + end + end + end +end diff --git a/lib/fluent/supervisor.rb b/lib/fluent/supervisor.rb index b02b5f623f..c7ad1884a6 100644 --- a/lib/fluent/supervisor.rb +++ b/lib/fluent/supervisor.rb @@ -487,6 +487,7 @@ def self.cleanup_resources end def initialize(opt) + @config_file_type = opt[:config_file_type] @daemonize = opt[:daemonize] @standalone_worker= opt[:standalone_worker] @config_path = opt[:config_path] @@ -620,7 +621,13 @@ def configure(supervisor: false) $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711') @inline_config = STDIN.read end - @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config) + @conf = Fluent::Config.build( + config_path: @config_path, + encoding: @conf_encoding, + additional_config: @inline_config, + use_v1_config: @use_v1_config, + type: @config_file_type, + ) @system_config = build_system_config(@conf) @log.level = @system_config.log_level