config_format: new config reader with YAML support (#4331)

CMakeLists.txt

@@ -630,6 +630,17 @@ if(FLB_HAVE_UNIX_SOCKET)
   FLB_DEFINITION(FLB_HAVE_UNIX_SOCKET)
 endif()
 
+# libyaml support
+check_c_source_compiles("
+  #include <yaml.h>
+  int main() {
+    yaml_parser_t parser;
+    return 0;
+  }" FLB_HAVE_LIBYAML)
+if(FLB_HAVE_LIBYAML)
+  FLB_DEFINITION(FLB_HAVE_LIBYAML)
+endif()
+
 # check attribute alloc_size
 check_c_source_compiles("
 #include <stdlib.h>

include/fluent-bit/config_format/flb_cf.h

@@ -0,0 +1,146 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_CONFIG_FORMAT_MAIN_H
+#define FLB_CONFIG_FORMAT_MAIN_H
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_sds.h>
+#include <monkey/mk_core.h>
+
+#define FLB_CF_ERROR_SERVICE_EXISTS "SERVICE definition already exists"
+#define FLB_CF_ERROR_META_CHAR      "invalid first meta character: '@' expected"
+#define FLB_CF_ERROR_KV_INVALID_KEY "invalid key content"
+#define FLB_CF_ERROR_KV_INVALID_VAL "invalid value content"
+
+/* manipulate error state for the context */
+#define flb_cf_error_set(cf, err)  cf->error_str = err
+#define flb_cf_error_get(cf)       cf->error_str
+#define flb_cr_error_reset(cf)     cf->error_str = ""
+
+/* meta commands: handled as key value pairs */
+#define flb_cf_meta flb_kv
+
+enum cf_file_format {
+    FLB_CF_FLUENTBIT,
+#ifdef FLB_HAVE_LIBYAML
+    FLB_CF_YAML
+#endif
+};
+
+enum section_type {
+    FLB_CF_SERVICE = 0,           /* [SERVICE]           */
+    FLB_CF_PARSER,                /* [PARSER]            */
+    FLB_CF_MULTILINE_PARSER,      /* [MULTILINE_PARSER]  */
+    FLB_CF_CUSTOM,                /* [CUSTOM]            */
+    FLB_CF_INPUT,                 /* [INPUT]             */
+    FLB_CF_FILTER,                /* [FILTER]            */
+    FLB_CF_OUTPUT,                /* [OUTPUT]            */
+    FLB_CF_OTHER,                 /* any other section.. */
+};
+
+struct flb_cf_group {
+    flb_sds_t name;               /* group name */
+    struct mk_list properties;    /* key value properties */
+    struct mk_list _head;         /* link to struct flb_cf_section->groups */
+};
+
+struct flb_cf_section {
+    int type;
+    flb_sds_t name;               /* name (used for FLB_CF_OTHER type) */
+    struct mk_list properties;    /* key value properties              */
+
+    struct mk_list groups;        /* list of groups */
+
+    struct mk_list _head;         /* link to struct flb_cf->sections */
+    struct mk_list _head_section; /* link to section type, e.g: inputs, filters.. */
+};
+
+struct flb_cf {
+    /* global service */
+    struct flb_cf_section *service;
+
+    /* config environment variables (env section, availble on YAML) */
+    struct mk_list env;
+
+    /* meta commands (used by fluentbit classic mode) */
+    struct mk_list metas;
+
+    /* parsers */
+    struct mk_list parsers;
+    struct mk_list multiline_parsers;
+
+    /* custom plugins */
+    struct mk_list customs;
+
+    /* pipeline */
+    struct mk_list inputs;
+    struct mk_list filters;
+    struct mk_list outputs;
+
+    /* others */
+    struct mk_list others;
+
+    /* list head for all sections */
+    struct mk_list sections;
+
+    /* set the last error found */
+    char *error_str;
+};
+
+
+struct flb_cf *flb_cf_create();
+struct flb_cf *flb_cf_create_from_file(struct flb_cf *cf, char *file);
+
+void flb_cf_destroy(struct flb_cf *cf);
+
+void flb_cf_dump(struct flb_cf *cf);
+
+/* metas */
+struct flb_kv *flb_cf_meta_create(struct flb_cf *cf, char *meta, int len);
+
+#define flb_cf_foreach_meta(cf) \
+
+
+void flb_cf_meta_destroy(struct flb_cf *cf, struct flb_cf_meta *meta);
+void flb_cf_meta_destroy_all(struct flb_cf *cf);
+
+/* groups */
+struct flb_cf_group *flb_cf_group_create(struct flb_cf *cf, struct flb_cf_section *s,
+                                         char *name, int len);
+void flb_cf_group_destroy(struct flb_cf_group *g);
+
+/* sections */
+struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int len);
+struct flb_cf_section *flb_cf_section_get_by_name(struct flb_cf *cf, char *name);
+void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s);
+void flb_cf_section_destroy_all(struct flb_cf *cf);
+
+/* properties */
+struct flb_kv *flb_cf_property_add(struct flb_cf *cf,
+                                   struct mk_list *kv_list,
+                                   char *k_buf, size_t k_len,
+                                   char *v_buf, size_t v_len);
+
+
+char *flb_cf_section_property_get(struct flb_cf *cf, struct flb_cf_section *s,
+                                  char *key);
+
+#endif

include/fluent-bit/config_format/flb_cf_fluentbit.h

@@ -0,0 +1,27 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_CONFIG_FORMAT_FLUENTBIT_H
+#define FLB_CONFIG_FORMAT_FLUENTBIT_H
+
+struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf,
+                                       char *file_path, char *buf, size_t size);
+
+#endif

include/fluent-bit/config_format/flb_cf_yaml.h

@@ -0,0 +1,26 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_CONFIG_FORMAT_YAML_H
+#define FLB_CONFIG_FORMAT_YAML_H
+
+struct flb_cf *flb_cf_yaml_create(struct flb_cf *cf, char *file_path,
+                                  char *buf, size_t size);
+#endif

include/fluent-bit/flb_config.h

@@ -69,6 +69,10 @@ struct flb_config {
 
     struct mk_rconf *file;
 
+    /* main configuration */
+    struct flb_cf *cf_main;
+    struct flb_cf *cf_parsers;
+
     flb_sds_t program_name;      /* argv[0] */
 
     /*
@@ -255,7 +259,7 @@ int flb_config_set_program_name(struct flb_config *config, char *name);
 
 int set_log_level_from_env(struct flb_config *config);
 #ifdef FLB_HAVE_STATIC_CONF
-struct mk_rconf *flb_config_static_open(const char *file);
+struct flb_cf *flb_config_static_open(const char *file);
 #endif
 
 struct flb_service_config {

include/fluent-bit/flb_config_format.h

@@ -0,0 +1,33 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_CONFIG_FORMAT_H
+#define FLB_CONFIG_FORMAT_H
+
+#include <fluent-bit/flb_info.h>
+
+#include "config_format/flb_cf.h"
+#include "config_format/flb_cf_fluentbit.h"
+
+#ifdef FLB_HAVE_LIBYAML
+#include "config_format/flb_cf_yaml.h"
+#endif
+
+#endif

include/fluent-bit/flb_parser_decoder.h

@@ -23,7 +23,7 @@
 
 #include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_sds.h>
-#include <monkey/mk_core.h>
+#include <fluent-bit/flb_config_format.h>
 
 /* Decoder behavior */
 #define FLB_PARSER_DEC_DEFAULT  0  /* results place as separate keys    */
@@ -59,7 +59,8 @@ struct flb_parser_dec {
     struct mk_list _head;    /* link to parser->decoders */
 };
 
-struct mk_list *flb_parser_decoder_list_create(struct mk_rconf_section *section);
+struct mk_list *flb_parser_decoder_list_create(struct flb_cf_section *section);
+
 int flb_parser_decoder_list_destroy(struct mk_list *list);
 int flb_parser_decoder_do(struct mk_list *decoders,
                           const char *in_buf, size_t in_size,

include/fluent-bit/flb_sds.h

@@ -99,6 +99,7 @@ static inline int flb_sds_casecmp(flb_sds_t s, const char *str, int len)
 flb_sds_t flb_sds_create(const char *str);
 flb_sds_t flb_sds_create_len(const char *str, int len);
 flb_sds_t flb_sds_create_size(size_t size);
+int flb_sds_trim(flb_sds_t s);
 flb_sds_t flb_sds_cat(flb_sds_t s, const char *str, int len);
 flb_sds_t flb_sds_cat_esc(flb_sds_t s, const char *str, int len,
                                        char *esc, size_t esc_size);

src/CMakeLists.txt

@@ -61,6 +61,19 @@ set(src
   flb_event.c
   )
 
+# Config format
+set(src
+  ${src}
+  config_format/flb_config_format.c
+  config_format/flb_cf_fluentbit.c
+)
+if(FLB_HAVE_LIBYAML)
+  set(src
+    ${src}
+    config_format/flb_cf_yaml.c
+    )
+endif()
+
 # Multiline subsystem
 add_subdirectory(multiline)
 set(src
@@ -329,6 +342,14 @@ if(OPENSSL_FOUND)
     )
 endif()
 
+# libyaml
+if(FLB_HAVE_LIBYAML)
+set(FLB_DEPS
+  ${FLB_DEPS}
+  yaml
+  )
+endif()
+
 # UTF8 Encoding
 if(FLB_UTF8_ENCODER)
 set(FLB_DEPS