From 2cc5214d8f26a4c15397771e7fcc8d42fc2f4510 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 16 Jan 2022 21:29:10 -0600 Subject: [PATCH 01/51] build: detect 'libyaml' Signed-off-by: Eduardo Silva --- CMakeLists.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5dfdff1b99d..73e85da8388 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -630,6 +630,17 @@ if(FLB_HAVE_UNIX_SOCKET) FLB_DEFINITION(FLB_HAVE_UNIX_SOCKET) endif() +# libyaml support +check_c_source_compiles(" + #include + int main() { + yaml_parser_t parser; + return 0; + }" FLB_HAVE_LIBYAML) +if(FLB_HAVE_LIBYAML) + FLB_DEFINITION(FLB_HAVE_LIBYAML) +endif() + # check attribute alloc_size check_c_source_compiles(" #include From 7ed1548ef6a5b360a0e0e5e9b6a1a3b9db5d4e0d Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 16 Jan 2022 21:30:24 -0600 Subject: [PATCH 02/51] sds: new flb_sds_trim() function Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_sds.h | 1 + src/flb_sds.c | 63 ++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/include/fluent-bit/flb_sds.h b/include/fluent-bit/flb_sds.h index 5665aff6612..4d303f1e06a 100644 --- a/include/fluent-bit/flb_sds.h +++ b/include/fluent-bit/flb_sds.h @@ -99,6 +99,7 @@ static inline int flb_sds_casecmp(flb_sds_t s, const char *str, int len) flb_sds_t flb_sds_create(const char *str); flb_sds_t flb_sds_create_len(const char *str, int len); flb_sds_t flb_sds_create_size(size_t size); +int flb_sds_trim(flb_sds_t s); flb_sds_t flb_sds_cat(flb_sds_t s, const char *str, int len); flb_sds_t flb_sds_cat_esc(flb_sds_t s, const char *str, int len, char *esc, size_t esc_size); diff --git a/src/flb_sds.c b/src/flb_sds.c index fb99a63a6ef..9953b8adacc 100644 --- a/src/flb_sds.c +++ b/src/flb_sds.c @@ -29,7 +29,9 @@ #include #include #include + #include +#include static flb_sds_t sds_alloc(size_t size) { @@ -138,6 +140,67 @@ flb_sds_t flb_sds_cat(flb_sds_t s, const char *str, int len) return s; } + +/* + * remove empty spaces on left/right from sds buffer 's' and return the new length + * of the content. + */ +int flb_sds_trim(flb_sds_t s) +{ + unsigned int i; + unsigned int len; + char *left = 0, *right = 0; + char *buf; + + if (!s) { + return -1; + } + + len = flb_sds_len(s); + if (len == 0) { + return 0; + } + + buf = s; + left = buf; + + /* left spaces */ + while (left) { + if (isspace(*left)) { + left++; + } + else { + break; + } + } + + right = buf + (len - 1); + /* Validate right v/s left */ + if (right < left) { + buf[0] = '\0'; + return -1; + } + + /* Move back */ + while (right != buf){ + if (isspace(*right)) { + right--; + } + else { + break; + } + } + + len = (right - left) + 1; + for (i=0; i Date: Sun, 16 Jan 2022 22:02:27 -0600 Subject: [PATCH 03/51] config_format: new config reader for Fluent Bit and YAML formats (#4331) The following patch introduces a new interface called 'config_format'. This new interface implements a new layer to read and compose a configuration 'context' for Fluent Bit. The idea started on discussion #4331 where initially looking for feedback to extend the format and support 'groups' (or sub section) many needs were expressed like other formats. In the actual implementation which comes from lib/monkey, is not flexible enough and adds complexity when trying to extend it. The new 'config_format' is a 'layer' that generate configuration contexts: +-------------+ +---------------+ +----------------------------+ | Config File | <-- | Config Format | --> | Fluent Bit Readers & Setup | +-------------+ +---------------+ +----------------------------+ The advantage of this is that 'config_format' supports different 'backends' to read data from: this implementation supports 'fluentbit' (current) format but also 'YAML' format. +---------------+ | Config Format | +------+-+------+ | | +------------------+ +-------------+ | fluentbit format | | YAML format | +------------------+ +-------------+ 'fluentbit' format is a backport format from lib/monkey with API changes, the YAML format is parsed by using libyaml when available. The concepts of configuration keeps being the same: - sections: define a block - properties: key/value pairs that belongs to a section The following examples are identical configurations in different formats: --- fluentbit format --- [SERVICE] flush 1 log_level info [INPUT] name tail path /var/log/containers/*.log parser docker [INPUT] name forward listen 0.0.0.0 [OUTPUT] name stdout match * --- eof --- Now the identical representation in YAML: --- YAML format --- service: flush: 1 log_level: info inputs: tail: path: "/var/log/containers/*.log" parser: docker forward: listen: 0.0.0.0 outputs: stdout: match: "*" --- eof --- == Notes == - Fluent Bit config reader is still using the old mechanism, the next patches will migrate to this interface. - 'Groups' are not implemented/supported yet. Signed-off-by: Eduardo Silva --- include/fluent-bit/config_format/flb_cf.h | 125 ++++ .../config_format/flb_cf_fluentbit.h | 26 + .../fluent-bit/config_format/flb_cf_yaml.h | 26 + include/fluent-bit/flb_config_format.h | 33 + src/CMakeLists.txt | 21 + src/config_format/flb_cf_fluentbit.c | 600 ++++++++++++++++++ src/config_format/flb_cf_yaml.c | 497 +++++++++++++++ src/config_format/flb_config_format.c | 394 ++++++++++++ 8 files changed, 1722 insertions(+) create mode 100644 include/fluent-bit/config_format/flb_cf.h create mode 100644 include/fluent-bit/config_format/flb_cf_fluentbit.h create mode 100644 include/fluent-bit/config_format/flb_cf_yaml.h create mode 100644 include/fluent-bit/flb_config_format.h create mode 100644 src/config_format/flb_cf_fluentbit.c create mode 100644 src/config_format/flb_cf_yaml.c create mode 100644 src/config_format/flb_config_format.c diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h new file mode 100644 index 00000000000..9e11afff6ca --- /dev/null +++ b/include/fluent-bit/config_format/flb_cf.h @@ -0,0 +1,125 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_MAIN_H +#define FLB_CONFIG_FORMAT_MAIN_H + +#include +#include +#include + +#define FLB_CF_ERROR_SERVICE_EXISTS "SERVICE definition already exists" +#define FLB_CF_ERROR_META_CHAR "invalid first meta character: '@' expected" +#define FLB_CF_ERROR_KV_INVALID_KEY "invalid key content" +#define FLB_CF_ERROR_KV_INVALID_VAL "invalid value content" + +/* manipulate error state for the context */ +#define flb_cf_error_set(cf, err) cf->error_str = err +#define flb_cf_error_get(cf) cf->error_str +#define flb_cr_error_reset(cf) cf->error_str = "" + +/* meta commands: handled as key value pairs */ +#define flb_cf_meta flb_kv + +enum section_type { + FLB_CF_SERVICE, /* [SERVICE] */ + FLB_CF_PARSER, /* [PARSER] */ + FLB_CF_MULTILINE_PARSER, /* [MULTILINE_PARSER] */ + FLB_CF_CUSTOM, /* [CUSTOM] */ + FLB_CF_INPUT, /* [INPUT] */ + FLB_CF_FILTER, /* [FILTER] */ + FLB_CF_OUTPUT, /* [OUTPUT] */ + FLB_CF_OTHER, /* any other section.. */ +}; + +struct flb_cf_group { + flb_sds_t name; /* group name */ + struct mk_list properties; /* key value properties */ + struct mk_list _head; /* link to struct flb_cf_section->groups */ +}; + +struct flb_cf_section { + int type; + flb_sds_t name; /* name (used for FLB_CF_OTHER type) */ + struct mk_list properties; /* key value properties */ + + struct mk_list groups; /* list of groups */ + + struct mk_list _head; /* link to struct flb_cf->sections */ + struct mk_list _head_section; /* link to section type, e.g: inputs, filters.. */ +}; + +struct flb_cf { + /* global service */ + struct flb_cf_section *service; + + /* meta commands */ + struct mk_list metas; + + /* parsers */ + struct mk_list parsers; + struct mk_list multiline_parsers; + + /* custom plugins */ + struct mk_list customs; + + /* pipeline */ + struct mk_list inputs; + struct mk_list filters; + struct mk_list outputs; + + /* others */ + struct mk_list others; + + /* list head for all sections */ + struct mk_list sections; + + /* set the last error found */ + char *error_str; +}; + + +struct flb_cf *flb_cf_create(); +void flb_cf_destroy(struct flb_cf *cf); + +void flb_cf_dump(struct flb_cf *cf); + +/* metas */ +struct flb_kv *flb_cf_meta_create(struct flb_cf *cf, char *meta, int len); + +void flb_cf_meta_destroy(struct flb_cf *cf, struct flb_cf_meta *meta); +void flb_cf_meta_destroy_all(struct flb_cf *cf); + +/* groups */ +struct flb_cf_group *flb_cf_group_create(struct flb_cf *cf, struct flb_cf_section *s, + char *name, int len); +void flb_cf_group_destroy(struct flb_cf_group *g); + +/* sections */ +struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int len); +void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s); +void flb_cf_section_destroy_all(struct flb_cf *cf); + +/* properties */ +struct flb_kv *flb_cf_property_add(struct flb_cf *cf, + struct mk_list *kv_list, + char *k_buf, size_t k_len, + char *v_buf, size_t v_len); +#endif diff --git a/include/fluent-bit/config_format/flb_cf_fluentbit.h b/include/fluent-bit/config_format/flb_cf_fluentbit.h new file mode 100644 index 00000000000..99fbc392c9c --- /dev/null +++ b/include/fluent-bit/config_format/flb_cf_fluentbit.h @@ -0,0 +1,26 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_FLUENTBIT_H +#define FLB_CONFIG_FORMAT_FLUENTBIT_H + +struct flb_cf *flb_cf_fluentbit_create(char *file_path, char *buf, size_t size); + +#endif \ No newline at end of file diff --git a/include/fluent-bit/config_format/flb_cf_yaml.h b/include/fluent-bit/config_format/flb_cf_yaml.h new file mode 100644 index 00000000000..b0fba5cae99 --- /dev/null +++ b/include/fluent-bit/config_format/flb_cf_yaml.h @@ -0,0 +1,26 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_YAML_H +#define FLB_CONFIG_FORMAT_YAML_H + +struct flb_cf *flb_cf_yaml_create(char *file_path, char *buf, size_t size); + +#endif diff --git a/include/fluent-bit/flb_config_format.h b/include/fluent-bit/flb_config_format.h new file mode 100644 index 00000000000..27f11b3c322 --- /dev/null +++ b/include/fluent-bit/flb_config_format.h @@ -0,0 +1,33 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_H +#define FLB_CONFIG_FORMAT_H + +#include + +#include "config_format/flb_cf.h" +#include "config_format/flb_cf_fluentbit.h" + +#ifdef FLB_HAVE_LIBYAML +#include "config_format/flb_cf_yaml.h" +#endif + +#endif \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 37fb5e7eac0..977898f728a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -61,6 +61,19 @@ set(src flb_event.c ) +# Config format +set(src + ${src} + config_format/flb_config_format.c + config_format/flb_cf_fluentbit.c +) +if(FLB_HAVE_LIBYAML) + set(src + ${src} + config_format/flb_cf_yaml.c + ) +endif() + # Multiline subsystem add_subdirectory(multiline) set(src @@ -329,6 +342,14 @@ if(OPENSSL_FOUND) ) endif() +# libyaml +if(FLB_HAVE_LIBYAML) +set(FLB_DEPS + ${FLB_DEPS} + yaml + ) +endif() + # UTF8 Encoding if(FLB_UTF8_ENCODER) set(FLB_DEPS diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c new file mode 100644 index 00000000000..9b30ee9b7ac --- /dev/null +++ b/src/config_format/flb_cf_fluentbit.c @@ -0,0 +1,600 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifndef _MSC_VER +#include +#endif + +#ifdef _WIN32 +#include +#include +#define PATH_MAX MAX_PATH +#endif + +#define FLB_CF_BUF_SIZE 4096 + +/* Included file by configuration */ +struct local_file { + flb_sds_t path; + struct mk_list _head; +}; + +/* Local context to keep state of variables and general */ +struct local_ctx { + int level; + char *file; + flb_sds_t root_path; + + /* included files */ + struct mk_list includes; + + /* meta instructions */ + struct mk_list metas; + + /* list of sections */ + struct mk_list sections; +}; + +static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file); + +/* Raise a configuration schema error */ +static void config_error(const char *path, int line, const char *msg) +{ + flb_error("[config] error in %s:%i: %s", path, line, msg); +} + +/* Raise a warning */ +static void config_warn(const char *path, int line, const char *msg) +{ + mk_warn("Config file warning '%s':\n" + "\t\t\t\tat line %i: %s", + path, line, msg); +} + +static int char_search(const char *string, int c, int len) +{ + char *p; + + if (len < 0) { + len = strlen(string); + } + + p = memchr(string, c, len); + if (p) { + return (p - string); + } + + return -1; +} + +#ifndef _WIN32 +static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char * path) +{ + int ret = -1; + glob_t glb; + char tmp[PATH_MAX]; + + const char *glb_path; + size_t i; + int ret_glb = -1; + + if (ctx->root_path && path[0] != '/') { + snprintf(tmp, PATH_MAX, "%s/%s", ctx->root_path, path); + glb_path = tmp; + } + else { + glb_path = path; + } + + ret_glb = glob(glb_path, GLOB_NOSORT, NULL, &glb); + if (ret_glb != 0) { + switch(ret_glb){ + case GLOB_NOSPACE: + flb_warn("[%s] glob: [%s] no space", __FUNCTION__, glb_path); + break; + case GLOB_NOMATCH: + flb_warn("[%s] glob: [%s] no match", __FUNCTION__, glb_path); + break; + case GLOB_ABORTED: + flb_warn("[%s] glob: [%s] aborted", __FUNCTION__, glb_path); + break; + default: + flb_warn("[%s] glob: [%s] other error", __FUNCTION__, glb_path); + } + return ret; + } + + for (i = 0; i < glb.gl_pathc; i++) { + ret = read_config(cf, ctx, glb.gl_pathv[i]); + if (ret < 0) { + break; + } + } + + globfree(&glb); + return ret; +} +#else +static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char *path) +{ + char *star, *p0, *p1; + char pattern[MAX_PATH]; + char buf[MAX_PATH]; + int ret; + struct stat st; + HANDLE h; + WIN32_FIND_DATA data; + + if (strlen(path) > MAX_PATH - 1) { + return -1; + } + + star = strchr(path, '*'); + if (star == NULL) { + return -1; + } + + /* + * C:\data\tmp\input_*.conf + * 0<-----| + */ + p0 = star; + while (path <= p0 && *p0 != '\\') { + p0--; + } + + /* + * C:\data\tmp\input_*.conf + * |---->1 + */ + p1 = star; + while (*p1 && *p1 != '\\') { + p1++; + } + + memcpy(pattern, path, (p1 - path)); + pattern[p1 - path] = '\0'; + + h = FindFirstFileA(pattern, &data); + if (h == INVALID_HANDLE_VALUE) { + return 0; + } + + do { + /* Ignore the current and parent dirs */ + if (!strcmp(".", data.cFileName) || !strcmp("..", data.cFileName)) { + continue; + } + + /* Avoid an infinite loop */ + if (strchr(data.cFileName, '*')) { + continue; + } + + /* Create a path (prefix + filename + suffix) */ + memcpy(buf, path, p0 - path + 1); + buf[p0 - path + 1] = '\0'; + + if (FAILED(StringCchCatA(buf, MAX_PATH, data.cFileName))) { + continue; + } + if (FAILED(StringCchCatA(buf, MAX_PATH, p1))) { + continue; + } + + if (strchr(p1, '*')) { + read_glob(cf, ctx, buf); /* recursive */ + continue; + } + + ret = stat(buf, &st); + if (ret == 0 && (st.st_mode & S_IFMT) == S_IFREG) { + if (read_config(cf, ctx, buf) < 0) { + return -1; + } + } + } while (FindNextFileA(h, &data) != 0); + + FindClose(h); + return 0; +} +#endif + +static int local_init(struct local_ctx *ctx, char *file) +{ + char *p; + char *end; + char path[PATH_MAX + 1]; + + if (file) { +#ifdef _MSC_VER + p = _fullpath(path, file, PATH_MAX + 1); +#else + p = realpath(file, path); +#endif + if (!p) { + return -1; + } + } + + /* lookup path ending and truncate */ + end = strrchr(path, '/'); + if (!end) { + return -1; + } + end++; + *end = '\0'; + + if (file) { + ctx->file = flb_sds_create(file); + ctx->root_path = flb_sds_create(path); + } + else { + ctx->file = NULL; + ctx->root_path = NULL; + } + + ctx->level = 0; + mk_list_init(&ctx->metas); + mk_list_init(&ctx->sections); + mk_list_init(&ctx->includes); + + return 0; +} + +static void local_exit(struct local_ctx *ctx) +{ + struct mk_list *tmp; + struct mk_list *head; + struct local_file *f; + + mk_list_foreach_safe(head, tmp, &ctx->includes) { + f = mk_list_entry(head, struct local_file, _head); + flb_sds_destroy(f->path); + mk_list_del(&f->_head); + flb_free(f); + } + + if (ctx->file) { + flb_sds_destroy(ctx->file); + } + + if (ctx->root_path) { + flb_sds_destroy(ctx->root_path); + } +} + +static int is_file_included(struct local_ctx *ctx, const char *path) +{ + struct mk_list *head; + struct local_file *file; + + mk_list_foreach(head, &ctx->includes) { + file = mk_list_entry(head, struct local_file, _head); + if (strcmp(file->path, path) == 0) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + +static int check_indent(const char *line, const char *indent) +{ + while (*line == *indent && *indent) { + line++; + indent++; + } + + if (*indent != '\0') { + if (isblank(*line)) { + flb_error("[config] inconsistent use of tab and space"); + } + else { + flb_error("[config] indentation level is too low"); + } + return -1; + } + + if (isblank(*line)) { + flb_error("[config] Extra indentation level found"); + return -1; + } + + return 0; +} + +static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) +{ + int i; + int len; + int ret; + int line = 0; + int indent_len = -1; + int n_keys = 0; + char *key; + int key_len; + char *val; + int val_len; + char *buf; + char tmp[PATH_MAX]; + flb_sds_t section = NULL; + flb_sds_t indent = NULL; + struct stat st; + struct local_file *file; + struct flb_cf_meta *meta; + struct flb_cf_section *current = NULL; + struct flb_kv *kv; + FILE *f; + + /* Check if the path exists (relative cases for included files) */ + if (ctx->level >= 0) { + ret = stat(cfg_file, &st); + if (ret == -1 && errno == ENOENT) { + /* Try to resolve the real path (if exists) */ + if (cfg_file[0] == '/') { + return -1; + } + + if (ctx->root_path) { + snprintf(tmp, PATH_MAX, "%s/%s", ctx->root_path, cfg_file); + cfg_file = tmp; + } + } + } + + /* Check this file have not been included before */ + ret = is_file_included(ctx, cfg_file); + if (ret) { + flb_error("[config] file already included %s", cfg_file); + return -1; + } + ctx->level++; + + /* Open configuration file */ + if ((f = fopen(cfg_file, "r")) == NULL) { + flb_warn("[config] I cannot open %s file", cfg_file); + return -1; + } + + /* Allocate temporal buffer to read file content */ + buf = flb_malloc(FLB_CF_BUF_SIZE); + if (!buf) { + flb_errno(); + return -1; + } + + /* looking for configuration directives */ + while (fgets(buf, FLB_CF_BUF_SIZE, f)) { + len = strlen(buf); + if (len > 0 && buf[len - 1] == '\n') { + buf[--len] = 0; + if (len && buf[len - 1] == '\r') { + buf[--len] = 0; + } + } + else { + /* + * If we don't find a break line, validate if we got an EOF or not. No EOF + * means that the incoming string is not finished so we must raise an + * exception. + */ + if (!feof(f)) { + config_error(cfg_file, line, "length of content has exceeded limit"); + flb_free(buf); + return -1; + } + } + + /* Line number */ + line++; + + if (!buf[0]) { + continue; + } + + /* Skip commented lines */ + if (buf[0] == '#') { + continue; + } + + if (len > 9 && strncasecmp(buf, "@INCLUDE ", 9) == 0) { + if (strchr(buf + 9, '*') != NULL) { + ret = read_glob(cf, ctx, buf + 9); + } + else { + ret = read_config(cf, ctx, buf + 9); + } + if (ret == -1) { + ctx->level--; + fclose(f); + if (indent) { + flb_sds_destroy(indent); + } + flb_free(buf); + return -1; + } + continue; + } + else if (buf[0] == '@' && len > 3) { + meta = flb_cf_meta_create(cf, buf, len); + if (!meta) { + fclose(f); + if (indent) { + flb_sds_destroy(indent); + } + flb_free(buf); + return -1; + } + continue; + } + + /* Section definition */ + if (buf[0] == '[') { + int end = -1; + end = char_search(buf, ']', len); + if (end > 0) { + /* + * Before to add a new section, lets check the previous + * one have at least one key set + */ + if (current && n_keys == 0) { + config_warn(cfg_file, line, + "previous section did not have keys"); + } + + /* Create new section */ + current = flb_cf_section_create(cf, buf + 1, end - 1); + n_keys = 0; + continue; + } + else { + config_error(cfg_file, line, "bad header definition"); + flb_free(buf); + return -1; + } + } + + /* No separator defined */ + if (!indent) { + i = 0; + + do { i++; } while (i < len && isblank(buf[i])); + + indent = flb_sds_create_len(buf, i); + indent_len = flb_sds_len(indent); + + /* Blank indented line */ + if (i == len) { + continue; + } + } + + /* Validate indentation level */ + if (check_indent(buf, indent) < 0) { + config_error(cfg_file, line, "Invalid indentation level"); + flb_sds_destroy(key); + flb_sds_destroy(val); + return -1; + } + + if (buf[indent_len] == '#' || indent_len == len) { + continue; + } + + if (len - indent_len >= 3 && strncmp(buf + indent_len, "---", 3) == 0) { + continue; + } + + /* Get the separator */ + i = char_search(buf + indent_len, ' ', len - indent_len); + + /* key */ + key = buf + indent_len; + key_len = i; + + /* val */ + val = buf + indent_len + i + 1; + val_len = len - indent_len - i - 1; + + if (!key || !val || i < 0) { + config_error(cfg_file, line, "Each key must have a value"); + return -1; + } + + if (val_len == 0) { + config_error(cfg_file, line, "Key has an empty value"); + return -1; + } + + /* Register entry: key and val are copied as duplicated */ + kv = flb_cf_property_add(cf, ¤t->properties, + key, key_len, + val, val_len); + if (!kv) { + config_error(cfg_file, line, "could not allocate key value pair"); + return -1; + } + + /* Free temporary key and val */ + n_keys++; + } + + if (section && n_keys == 0) { + /* No key, no warning */ + } + + fclose(f); + if (indent) { + flb_sds_destroy(indent); + } + flb_free(buf); + + /* Append this file to the list */ + file = flb_malloc(sizeof(struct local_file)); + if (!file) { + flb_errno(); + ctx->level--; + return -1; + } + file->path = flb_sds_create(cfg_file); + mk_list_add(&file->_head, &ctx->includes); + ctx->level--; + + return 0; +} + +struct flb_cf *flb_cf_fluentbit_create(char *file_path, char *buf, size_t size) +{ + int ret; + struct flb_cf *cf; + struct local_ctx ctx; + + cf = flb_cf_create(); + if (!cf) { + return NULL; + } + + local_init(&ctx, file_path); + + ret = read_config(cf, &ctx, file_path); + + local_exit(&ctx); + + if (ret == -1) { + exit(28); + } + + + return cf; +} diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c new file mode 100644 index 00000000000..4baf1e06ed8 --- /dev/null +++ b/src/config_format/flb_cf_yaml.c @@ -0,0 +1,497 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +enum section { + SECTION_SERVICE, + SECTION_CUSTOM, + SECTION_INPUT, + SECTION_FILTER, + SECTION_OUTPUT, + SECTION_OTHER, +}; + +enum state { + STATE_START, /* start state */ + STATE_STREAM, /* start/end stream */ + STATE_DOCUMENT, /* start/end document */ + + STATE_SECTION, /* top level */ + STATE_SECTION_KEY, + STATE_SECTION_VAL, + + STATE_SERVICE, /* service section */ + STATE_OTHER, /* service section */ + + STATE_PLUGIN_CUSTOM, /* custom plugins section */ + STATE_PLUGIN_INPUT, /* input plugins section */ + STATE_PLUGIN_FILTER, /* filter plugins section */ + STATE_PLUGIN_OUTPUT, /* output plugins section */ + + STATE_PLUGIN_TYPE, + STATE_PLUGIN_KEY_VALUE_PAIR, + STATE_PLUGIN_KEY, + STATE_PLUGIN_VAL, + + STATE_STOP /* end state */ +}; + +struct parser_state { + /* file path */ + flb_sds_t file; + + /* tokens state */ + enum state state; + + /* active section (if any) */ + enum section section; + + /* temporary key value pair */ + flb_sds_t key; + flb_sds_t val; + + /* active section */ + struct flb_cf_section *cf_section; + + /* internal variables for checks */ + int service_set; + int customs_set; + int inputs_set; + int filters_set; + int outputs_set; +}; + +/* yaml_* functions return 1 on success and 0 on failure. */ +enum status { + YAML_SUCCESS = 1, + YAML_FAILURE = 0 +}; + +static int add_section_type(struct flb_cf *cf, struct parser_state *s) +{ + if (s->section == SECTION_CUSTOM) { + s->cf_section = flb_cf_section_create(cf, "CUSTOM", 0); + } + else if (s->section == SECTION_INPUT) { + s->cf_section = flb_cf_section_create(cf, "INPUT", 0); + } + else if (s->section == SECTION_FILTER) { + s->cf_section = flb_cf_section_create(cf, "FILTER", 0); + } + else if (s->section == SECTION_OUTPUT) { + s->cf_section = flb_cf_section_create(cf, "OUTPUT", 0); + } + + if (!s->cf_section) { + return -1; + } + + return 0; +} + +static void yaml_error_event(struct parser_state *s, yaml_event_t *event) +{ + flb_error("[config] YAML error found in file \"%s\", line %i, column %i: " + "unexpected event %d in state %d.", + s->file, event->start_mark.line + 1, event->start_mark.column, + event->type, s->state); +} + +static void yaml_error_definition(struct parser_state *s, yaml_event_t *event, + char *value) +{ + flb_error("[config] YAML error found in file \"%s\", line %i, column %i: " + "duplicated definition of '%s'", + s->file, event->start_mark.line + 1, event->start_mark.column, + value); +} + +static int consume_event(struct flb_cf *cf, struct parser_state *s, + yaml_event_t *event) +{ + int len; + int ret; + char *value; + struct flb_kv *kv; + + switch (s->state) { + case STATE_START: + switch (event->type) { + case YAML_STREAM_START_EVENT: + s->state = STATE_STREAM; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_STREAM: + switch (event->type) { + case YAML_DOCUMENT_START_EVENT: + s->state = STATE_DOCUMENT; + break; + case YAML_STREAM_END_EVENT: + s->state = STATE_STOP; /* all done */ + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_DOCUMENT: + switch (event->type) { + case YAML_MAPPING_START_EVENT: + s->state = STATE_SECTION; + break; + case YAML_DOCUMENT_END_EVENT: + s->state = STATE_STREAM; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_SECTION: + s->section = SECTION_OTHER; + + switch (event->type) { + case YAML_SCALAR_EVENT: + value = (char *)event->data.scalar.value; + if (strcasecmp(value, "service") == 0) { + if (s->service_set) { + yaml_error_definition(s, event, value); + return YAML_FAILURE; + } + s->state = STATE_SERVICE; + s->section = SECTION_SERVICE; + s->service_set = 1; + s->cf_section = flb_cf_section_create(cf, value, 0); + if (!s->cf_section) { + return YAML_FAILURE; + } + } + else if (strcasecmp(value, "customs") == 0) { + if (s->customs_set) { + yaml_error_definition(s, event, value); + return YAML_FAILURE; + } + s->state = STATE_PLUGIN_CUSTOM; + s->section = SECTION_CUSTOM; + s->customs_set = 1; + } + else if (strcasecmp(value, "inputs") == 0) { + if (s->inputs_set) { + yaml_error_definition(s, event, value); + return YAML_FAILURE; + } + s->state = STATE_PLUGIN_INPUT; + s->section = SECTION_INPUT; + s->inputs_set = 1; + } + else if (strcasecmp(value, "filters") == 0) { + if (s->filters_set) { + yaml_error_definition(s, event, value); + return YAML_FAILURE; + } + s->state = STATE_PLUGIN_FILTER; + s->section = SECTION_FILTER; + s->filters_set = 1; + } + else if (strcasecmp(value, "outputs") == 0) { + if (s->outputs_set) { + yaml_error_definition(s, event, value); + return YAML_FAILURE; + } + s->state = STATE_PLUGIN_OUTPUT; + s->section = SECTION_OUTPUT; + s->outputs_set = 1; + } + else { + /* any other main section definition (e.g: similar to STATE_SERVICE) */ + s->state = STATE_OTHER; + s->cf_section = flb_cf_section_create(cf, value, 0); + if (!s->cf_section) { + return YAML_FAILURE; + } + } + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_DOCUMENT; + break; + case YAML_DOCUMENT_END_EVENT: + s->state = STATE_STREAM; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + /* service or others */ + case STATE_SERVICE: + case STATE_OTHER: + switch(event->type) { + case YAML_MAPPING_START_EVENT: + s->state = STATE_SECTION_KEY; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_SECTION; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_SECTION_KEY: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_SECTION_VAL; + value = (char *) event->data.scalar.value; + s->key = flb_sds_create(value); + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_SECTION; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_SECTION_VAL: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_SECTION_KEY; + value = (char *) event->data.scalar.value; + s->val = flb_sds_create(value); + + /* register key/value pair as a property */ + kv = flb_cf_property_add(cf, &s->cf_section->properties, + s->key, flb_sds_len(s->key), + s->val, flb_sds_len(s->val)); + if (!kv) { + return YAML_FAILURE; + } + flb_sds_destroy(s->key); + flb_sds_destroy(s->val); + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + /* Plugin types */ + case STATE_PLUGIN_CUSTOM: + case STATE_PLUGIN_INPUT: + case STATE_PLUGIN_FILTER: + case STATE_PLUGIN_OUTPUT: + switch(event->type) { + case YAML_MAPPING_START_EVENT: + s->state = STATE_PLUGIN_TYPE; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_SECTION_KEY; + break; + case YAML_SCALAR_EVENT: + s->state = STATE_SECTION; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_PLUGIN_TYPE: + switch(event->type) { + case YAML_SCALAR_EVENT: + /* register the section by type */ + ret = add_section_type(cf, s); + if (ret == -1) { + return YAML_FAILURE; + } + + /* the 'type' is the plugin name, so we add it as a 'name' property */ + value = (char *) event->data.scalar.value; + len = strlen(value); + + /* register the type: name = abc */ + kv = flb_cf_property_add(cf, &s->cf_section->properties, + "name", 4, + value, len); + if (!kv) { + return YAML_FAILURE; + } + + /* next state are key value pairs */ + s->state = STATE_PLUGIN_KEY_VALUE_PAIR; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_SECTION; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_PLUGIN_KEY_VALUE_PAIR: + switch(event->type) { + case YAML_MAPPING_START_EVENT: + s->state = STATE_PLUGIN_KEY; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_PLUGIN_TYPE; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_PLUGIN_KEY: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_PLUGIN_VAL; + value = (char *) event->data.scalar.value; + s->key = flb_sds_create(value); + break; + case YAML_MAPPING_START_EVENT: + s->state = STATE_PLUGIN_TYPE; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_PLUGIN_TYPE; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_PLUGIN_VAL: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_PLUGIN_KEY; + value = (char *) event->data.scalar.value; + s->val = flb_sds_create(value); + + /* register key/value pair as a property */ + flb_cf_property_add(cf, &s->cf_section->properties, + s->key, flb_sds_len(s->key), + s->val, flb_sds_len(s->val)); + flb_sds_destroy(s->key); + flb_sds_destroy(s->val); + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + + case STATE_STOP: + break; + } + + return YAML_SUCCESS; +} + +static int read_config(struct flb_cf *cf, void *ctx, char *cfg_file) +{ + int code = 0; + int status; + struct parser_state state; + yaml_parser_t parser; + yaml_event_t event; + FILE *fh; + + fh = fopen(cfg_file, "r"); + if (!fh) { + flb_errno(); + return -1; + } + + memset(&state, '\0', sizeof(state)); + state.file = flb_sds_create(cfg_file); + if (!state.file) { + fclose(fh); + return -1; + } + + yaml_parser_initialize(&parser); + yaml_parser_set_input_file(&parser, fh); + + do { + status = yaml_parser_parse(&parser, &event); + if (status == YAML_FAILURE) { + flb_error("[config] invalid YAML format"); + code = -1; + goto done; + } + status = consume_event(cf, &state, &event); + if (status == YAML_FAILURE) { + code = -1; + goto done; + } + yaml_event_delete(&event); + } while (state.state != STATE_STOP); + +done: + if (code == -1) { + yaml_event_delete(&event); + } + flb_sds_destroy(state.file); + yaml_parser_delete(&parser); + fclose(fh); + + return code; +} + +struct flb_cf *flb_cf_yaml_create(char *file_path, char *buf, size_t size) +{ + int ret; + struct flb_cf *cf; + + cf = flb_cf_create(); + if (!cf) { + return NULL; + } + + ret = read_config(cf, NULL, file_path); + if (ret == -1) { + flb_cf_destroy(cf); + return NULL; + } + + return cf; +} diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c new file mode 100644 index 00000000000..10a0f5dc9e1 --- /dev/null +++ b/src/config_format/flb_config_format.c @@ -0,0 +1,394 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +int flb_cf_file_read() +{ + return 0; +} + +struct flb_cf *flb_cf_create() +{ + struct flb_cf *ctx; + + ctx = flb_calloc(1, sizeof(struct flb_cf)); + if (!ctx) { + flb_errno(); + return NULL; + } + + /* meta commands */ + mk_list_init(&ctx->metas); + + /* parsers */ + mk_list_init(&ctx->parsers); + mk_list_init(&ctx->multiline_parsers); + + /* custom plugins */ + mk_list_init(&ctx->customs); + + /* pipeline */ + mk_list_init(&ctx->inputs); + mk_list_init(&ctx->filters); + mk_list_init(&ctx->outputs); + + /* other sections */ + mk_list_init(&ctx->others); + + /* general list for sections */ + mk_list_init(&ctx->sections); + + return ctx; +} + +void flb_cf_destroy(struct flb_cf *cf) +{ + flb_kv_release(&cf->metas); + flb_cf_section_destroy_all(cf); + flb_free(cf); +} + +static int get_section_type(char *name) +{ + if (strcasecmp(name, "SERVICE") == 0) { + return FLB_CF_SERVICE; + } + else if (strcasecmp(name, "PARSER") == 0) { + return FLB_CF_PARSER; + } + else if (strcasecmp(name, "MULTILINE_PARSER") == 0) { + return FLB_CF_MULTILINE_PARSER; + } + else if (strcasecmp(name, "CUSTOM") == 0 || strcasecmp(name, "CUSTOMS") == 0) { + return FLB_CF_CUSTOM; + } + else if (strcasecmp(name, "INPUT") == 0 || strcasecmp(name, "INPUTS") == 0) { + return FLB_CF_INPUT; + } + else if (strcasecmp(name, "FILTER") == 0 || strcasecmp(name, "FILTERS") == 0) { + return FLB_CF_FILTER; + } + else if (strcasecmp(name, "OUTPUT") == 0 || strcasecmp(name, "OUTPUT") == 0) { + return FLB_CF_OUTPUT; + } + + return FLB_CF_OTHER; +} + +struct flb_kv *flb_cf_property_add(struct flb_cf *cf, + struct mk_list *kv_list, + char *k_buf, size_t k_len, + char *v_buf, size_t v_len) +{ + int ret; + struct flb_kv *kv; + + kv = flb_kv_item_create_len(kv_list, k_buf, k_len, v_buf, v_len); + if (!kv) { + return NULL; + } + + /* sanitize key and value by removing empty spaces */ + ret = flb_sds_trim(kv->key); + if (ret == -1) { + flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_KEY); + flb_kv_item_destroy(kv); + return NULL; + } + + ret = flb_sds_trim(kv->val); + if (ret == -1) { + flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_VAL); + flb_kv_item_destroy(kv); + return NULL; + } + + return kv; +} + +struct flb_kv *flb_cf_meta_create(struct flb_cf *cf, char *meta, int len) +{ + int xlen; + char *p; + char *tmp; + struct flb_kv *kv; + + if (len <= 0) { + len = strlen(meta); + if (len == 0) { + return NULL; + } + } + + if (meta[0] != '@') { + flb_cf_error_set(cf, FLB_CF_ERROR_META_CHAR); + return NULL; + } + + p = meta; + tmp = strchr(p, ' '); + xlen = (tmp - p); + + /* create k/v pair */ + kv = flb_cf_property_add(cf, &cf->metas, + meta + 1, xlen - 1, + meta + xlen + 1, len - xlen - 1); + if (!kv) { + return NULL; + } + + return kv; +} + +struct flb_cf_group *flb_cf_group_create(struct flb_cf *cf, struct flb_cf_section *s, + char *name, int len) +{ + struct flb_cf_group *g; + + if (!name || strlen(name) == 0 || len < 1) { + return NULL; + } + + /* section context */ + g = flb_malloc(sizeof(struct flb_cf_group)); + if (!g) { + flb_errno(); + return NULL; + } + + /* initialize lists */ + flb_kv_init(&g->properties); + + /* determinate type by name */ + if (len <= 0) { + len = strlen(name); + } + + /* create a NULL terminated name */ + g->name = flb_sds_create_len(name, len); + if (!g->name) { + flb_free(g); + return NULL; + } + + /* link to global section */ + mk_list_add(&g->_head, &s->groups); + + return g; +} + +void flb_cf_group_destroy(struct flb_cf_group *g) +{ + if (g->name) { + flb_sds_destroy(g->name); + } + + flb_kv_release(&g->properties); + mk_list_del(&g->_head); + flb_free(g); +} + +struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int len) +{ + int type; + struct flb_cf_section *s; + + /* section context */ + s = flb_malloc(sizeof(struct flb_cf_section)); + if (!s) { + flb_errno(); + return NULL; + } + + /* initialize lists */ + flb_kv_init(&s->properties); + mk_list_init(&s->groups); + + /* determinate type by name */ + if (len <= 0) { + len = strlen(name); + } + + /* create a NULL terminated name */ + s->name = flb_sds_create_len(name, len); + if (!s->name) { + flb_free(s); + return NULL; + } + + /* get the section type */ + type = get_section_type(s->name); + s->type = type; + + /* restrict to only one SERVICE section */ + if (type == FLB_CF_SERVICE && cf->service) { + flb_cf_error_set(cf, FLB_CF_ERROR_SERVICE_EXISTS); + flb_sds_destroy(s->name); + flb_free(s); + return NULL; + } + + if (type == FLB_CF_SERVICE && !cf->service) { + cf->service = s; + } + + /* link to global section */ + mk_list_add(&s->_head, &cf->sections); + + /* link to list per type */ + if (type == FLB_CF_PARSER) { + mk_list_add(&s->_head_section, &cf->parsers); + } + else if (type == FLB_CF_MULTILINE_PARSER) { + mk_list_add(&s->_head_section, &cf->multiline_parsers); + } + else if (type == FLB_CF_CUSTOM) { + mk_list_add(&s->_head_section, &cf->customs); + } + else if (type == FLB_CF_INPUT) { + mk_list_add(&s->_head_section, &cf->inputs); + } + else if (type == FLB_CF_FILTER) { + mk_list_add(&s->_head_section, &cf->filters); + } + else if (type == FLB_CF_OUTPUT) { + mk_list_add(&s->_head_section, &cf->outputs); + } + else if (type == FLB_CF_OTHER) { + mk_list_add(&s->_head_section, &cf->others); + } + + return s; +} + +void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s) +{ + struct mk_list *tmp; + struct mk_list *head; + struct flb_cf_group *g; + + if (s->name) { + flb_sds_destroy(s->name); + } + flb_kv_release(&s->properties); + + /* groups */ + mk_list_foreach_safe(head, tmp, &s->groups) { + g = mk_list_entry(head, struct flb_cf_group, _head); + flb_cf_group_destroy(g); + } + + /* unlink */ + mk_list_del(&s->_head); + + if (s->type != FLB_CF_SERVICE) { + mk_list_del(&s->_head_section); + } + flb_free(s); +} + +static void section_destroy_list(struct flb_cf *cf, struct mk_list *list) +{ + struct mk_list *tmp; + struct mk_list *head; + struct flb_cf_section *s; + + mk_list_foreach_safe(head, tmp, list) { + s = mk_list_entry(head, struct flb_cf_section, _head); + flb_cf_section_destroy(cf, s); + } +} + +void flb_cf_section_destroy_all(struct flb_cf *cf) +{ + section_destroy_list(cf, &cf->sections); +} + +/* + * Helpers + * ------- + */ + +static char *section_type_str(int type) +{ + switch (type) { + case FLB_CF_SERVICE: + return "SERVICE"; + case FLB_CF_PARSER: + return "PARSER"; + case FLB_CF_MULTILINE_PARSER: + return "MULTILINE_PARSER"; + case FLB_CF_CUSTOM: + return "CUSTOM"; + case FLB_CF_INPUT: + return "INPUT"; + case FLB_CF_FILTER: + return "FILTER"; + case FLB_CF_OUTPUT: + return "OUTPUT"; + case FLB_CF_OTHER: + return "OTHER"; + default: + return "error / unknown"; + } + + return NULL; +} + +static void dump_section(struct flb_cf_section *s) +{ + struct mk_list *head; + struct flb_kv *kv; + + printf("> section:\n name: %s\n type: %s\n", + s->name, section_type_str(s->type)); + + if (mk_list_size(&s->properties) > 0) { + printf(" properties:\n"); + mk_list_foreach(head, &s->properties) { + kv = mk_list_entry(head, struct flb_kv, _head); + printf(" - %-15s: %s\n", kv->key, kv->val); + } + } + else { + printf(" properties: NONE\n"); + } +} + +static void dump_section_list(struct mk_list *list) +{ + struct mk_list *head; + struct flb_cf_section *s; + + mk_list_foreach(head, list) { + s = mk_list_entry(head, struct flb_cf_section, _head); + dump_section(s); + } +} + +void flb_cf_dump(struct flb_cf *cf) +{ + dump_section_list(&cf->sections); +} \ No newline at end of file From 786c0d7bdc7b87c77113368046adab182ad28e43 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 16 Jan 2022 22:40:09 -0600 Subject: [PATCH 04/51] tests: internal: add 'config_format' for fluentbit and YAML Signed-off-by: Eduardo Silva --- tests/internal/CMakeLists.txt | 14 +++ tests/internal/config_format.c | 90 +++++++++++++++++++ tests/internal/config_format_fluentbit.c | 48 ++++++++++ tests/internal/config_format_yaml.c | 53 +++++++++++ .../data/config_format/fluent-bit.conf | 31 +++++++ .../data/config_format/fluent-bit.yaml | 40 +++++++++ 6 files changed, 276 insertions(+) create mode 100644 tests/internal/config_format.c create mode 100644 tests/internal/config_format_fluentbit.c create mode 100644 tests/internal/config_format_yaml.c create mode 100644 tests/internal/data/config_format/fluent-bit.conf create mode 100644 tests/internal/data/config_format/fluent-bit.yaml diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 5075b4a4aee..194d57c1147 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -24,6 +24,20 @@ set(UNIT_TESTS_FILES typecast.c ) +# Config format +set(UNIT_TESTS_FILES + ${UNIT_TESTS_FILES} + config_format.c + config_format_fluentbit.c +) + +if(FLB_HAVE_LIBYAML) + set(UNIT_TESTS_FILES + ${UNIT_TESTS_FILES} + config_format_yaml.c + ) +endif() + if (NOT WIN32) set(UNIT_TESTS_FILES ${UNIT_TESTS_FILES} diff --git a/tests/internal/config_format.c b/tests/internal/config_format.c new file mode 100644 index 00000000000..9f24fffbd03 --- /dev/null +++ b/tests/internal/config_format.c @@ -0,0 +1,90 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include + +#include "flb_tests_internal.h" + +void test_api() +{ + struct flb_cf *cf; + struct flb_cf_section *s_tmp; + struct flb_cf_section *service; + struct flb_cf_group *g_tmp; + struct flb_kv *meta; + struct flb_kv *kv; + + /* create context */ + cf = flb_cf_create(); + TEST_CHECK(cf != NULL); + + /* create service section */ + service = flb_cf_section_create(cf, "SERVICE", 7); + TEST_CHECK(service != NULL); + + /* add a property */ + kv = flb_cf_property_add(cf, &service->properties, "key", 3, "val", 3); + TEST_CHECK(kv != NULL); + + /* add a property with empty spaces on left/right */ + kv = flb_cf_property_add(cf, &service->properties, " key ", 5, " val ", 7); + TEST_CHECK(kv != NULL); + + /* property: check key */ + TEST_CHECK(flb_sds_len(kv->key) == 3); + TEST_CHECK(strcmp(kv->key, "key") == 0); + + /* property: check val */ + TEST_CHECK(flb_sds_len(kv->key) == 3); + TEST_CHECK(strcmp(kv->key, "key") == 0); + + /* add an invalid property */ + kv = flb_cf_property_add(cf, &service->properties, " ", 3, "", 0); + TEST_CHECK(kv == NULL); + + /* try to add another 'SERVICE' section, it should fail */ + s_tmp = flb_cf_section_create(cf, "SERVICE", 7); + TEST_CHECK(s_tmp == NULL); + + /* add a valid section */ + s_tmp = flb_cf_section_create(cf, "INPUT", 5); + TEST_CHECK(s_tmp != NULL); + + TEST_CHECK(mk_list_size(&cf->inputs) == 1); + + /* add property to the section recently created */ + kv = flb_cf_property_add(cf, &s_tmp->properties, "key", 3, "val", 3); + TEST_CHECK(kv != NULL); + + /* groups: add groups to the last section created */ + g_tmp = flb_cf_group_create(cf, s_tmp, "FLUENT GROUP", 12); + TEST_CHECK(g_tmp != NULL); + + /* add properties to the group */ + kv = flb_cf_property_add(cf, &g_tmp->properties, "key", 3, "val", 3); + TEST_CHECK(kv != NULL); + + /* groups: invalid group */ + g_tmp = flb_cf_group_create(cf, s_tmp, "", 0); + TEST_CHECK(g_tmp == NULL); + + /* Meta commands */ + meta = flb_cf_meta_create(cf, "@SET a=1 ", 20); + TEST_CHECK(meta != NULL); + TEST_CHECK(flb_sds_len(meta->key) == 3 && strcmp(meta->key, "SET") == 0); + TEST_CHECK(flb_sds_len(meta->val) == 3 && strcmp(meta->val, "a=1") == 0); + + /* invalid meta */ + meta = flb_cf_meta_create(cf, "@a=1 ", 5); + TEST_CHECK(meta == NULL); + + /* destroy context */ + flb_cf_destroy(cf); +} + +TEST_LIST = { + { "api" , test_api}, + { 0 } +}; diff --git a/tests/internal/config_format_fluentbit.c b/tests/internal/config_format_fluentbit.c new file mode 100644 index 00000000000..e0e92602e72 --- /dev/null +++ b/tests/internal/config_format_fluentbit.c @@ -0,0 +1,48 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include + +#include "flb_tests_internal.h" + +#define FLB_000 FLB_TESTS_DATA_PATH "/data/config_format/fluent-bit.conf" + +/* data/config_format/fluent-bit.conf */ +void test_basic() +{ + struct flb_cf *cf; + + cf = flb_cf_fluentbit_create(FLB_000, NULL, 0); + TEST_CHECK(cf != NULL); + + /* Total number of sections */ + TEST_CHECK(mk_list_size(&cf->sections) == 8); + + /* SERVICE check */ + TEST_CHECK(cf->service != NULL); + TEST_CHECK(mk_list_size(&cf->service->properties) == 3); + + /* Meta commands */ + TEST_CHECK(mk_list_size(&cf->metas) == 2); + + /* Check number sections per list */ + TEST_CHECK(mk_list_size(&cf->parsers) == 1); + TEST_CHECK(mk_list_size(&cf->multiline_parsers) == 1); + TEST_CHECK(mk_list_size(&cf->customs) == 1); + TEST_CHECK(mk_list_size(&cf->inputs) == 1); + TEST_CHECK(mk_list_size(&cf->filters) == 1); + TEST_CHECK(mk_list_size(&cf->outputs) == 1); + TEST_CHECK(mk_list_size(&cf->others) == 1); + + printf("\n"); + flb_cf_dump(cf); + + flb_cf_destroy(cf); +} + +TEST_LIST = { + { "basic" , test_basic}, + { 0 } +}; diff --git a/tests/internal/config_format_yaml.c b/tests/internal/config_format_yaml.c new file mode 100644 index 00000000000..6735598e1c9 --- /dev/null +++ b/tests/internal/config_format_yaml.c @@ -0,0 +1,53 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include + +#include "flb_tests_internal.h" + +#define FLB_000 FLB_TESTS_DATA_PATH "/data/config_format/fluent-bit.yaml" + +/* data/config_format/fluent-bit.yaml */ +void test_basic() +{ + struct flb_cf *cf; + + cf = flb_cf_yaml_create(FLB_000, NULL, 0); + TEST_CHECK(cf != NULL); + if (!cf) { + exit(EXIT_FAILURE); + } + + /* Total number of sections */ + TEST_CHECK(mk_list_size(&cf->sections) == 9); + + /* SERVICE check */ + TEST_CHECK(cf->service != NULL); + + if (cf->service) { + TEST_CHECK(mk_list_size(&cf->service->properties) == 3); + } + + /* Check number sections per list */ + TEST_CHECK(mk_list_size(&cf->parsers) == 0); + TEST_CHECK(mk_list_size(&cf->multiline_parsers) == 0); + TEST_CHECK(mk_list_size(&cf->customs) == 1); + TEST_CHECK(mk_list_size(&cf->inputs) == 3); + TEST_CHECK(mk_list_size(&cf->filters) == 1); + TEST_CHECK(mk_list_size(&cf->outputs) == 2); + TEST_CHECK(mk_list_size(&cf->others) == 1); + + printf("\n"); + flb_cf_dump(cf); + + flb_cf_destroy(cf); + + +} + +TEST_LIST = { + { "basic" , test_basic}, + { 0 } +}; diff --git a/tests/internal/data/config_format/fluent-bit.conf b/tests/internal/data/config_format/fluent-bit.conf new file mode 100644 index 00000000000..67b1cbbf473 --- /dev/null +++ b/tests/internal/data/config_format/fluent-bit.conf @@ -0,0 +1,31 @@ +@SET a=1 +@SET b=2 + +[SERVICE] + flush 1 + log_level info + http_server on + +[PARSER] + name test_api + +[MULTILINE_PARSER] + name abc + +[CUSTOM] + name calyptia + +[INPUT] + name tail + path /var/log/containers/*.log + +[FILTER] + name stdout + match * + +[OUTPUT] + name null + match * + +[UNKNOWN] + name blah diff --git a/tests/internal/data/config_format/fluent-bit.yaml b/tests/internal/data/config_format/fluent-bit.yaml new file mode 100644 index 00000000000..ac1bbe948b1 --- /dev/null +++ b/tests/internal/data/config_format/fluent-bit.yaml @@ -0,0 +1,40 @@ +service: + flush: 1 + log_level: info + http_server: on + +customs: + calyptia: + api_key: abcdefg + +inputs: + tail: + path: "/var/log/containers/*.log" + parser: docker + + tail: + path: "/var/log/messages" + parser: syslog + + forward: + listen: 0.0.0.0 + +filters: + grep: + exclude: $kubernetes['labels']['app'] myapp + +outputs: + stdout: + match: "*" + retry_limit: false + + http: + match: "*" + host: 192.168.3.2 + port: 9444 + tls: on + tls.verify: off + retry_limit: false + +unknown: + a: b From 9bf14655c6bdecc7fdd578ada75525e6666dbbae Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 16 Jan 2022 23:00:22 -0600 Subject: [PATCH 05/51] config_format: use 'compat' interface Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index 9b30ee9b7ac..59fa3fe8978 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -24,13 +24,13 @@ #include #include #include +#include + #include #include #include #include -#include - #ifndef _MSC_VER #include #endif From 7602d9cef8ae3185a86a20bb41eba68ee0350059 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 17 Jan 2022 23:03:35 -0600 Subject: [PATCH 06/51] config_format: new wrapper to read from file Signed-off-by: Eduardo Silva --- include/fluent-bit/config_format/flb_cf.h | 19 ++++++++- src/config_format/flb_config_format.c | 47 +++++++++++++++++++++-- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h index 9e11afff6ca..5dc535d15b9 100644 --- a/include/fluent-bit/config_format/flb_cf.h +++ b/include/fluent-bit/config_format/flb_cf.h @@ -38,8 +38,15 @@ /* meta commands: handled as key value pairs */ #define flb_cf_meta flb_kv +enum cf_file_format { + FLB_CF_FLUENTBIT, +#ifdef FLB_HAVE_LIBYAML + FLB_CF_YAML +#endif +}; + enum section_type { - FLB_CF_SERVICE, /* [SERVICE] */ + FLB_CF_SERVICE = 0, /* [SERVICE] */ FLB_CF_PARSER, /* [PARSER] */ FLB_CF_MULTILINE_PARSER, /* [MULTILINE_PARSER] */ FLB_CF_CUSTOM, /* [CUSTOM] */ @@ -97,6 +104,8 @@ struct flb_cf { struct flb_cf *flb_cf_create(); +struct flb_cf *flb_cf_create_from_file(char *file); + void flb_cf_destroy(struct flb_cf *cf); void flb_cf_dump(struct flb_cf *cf); @@ -104,6 +113,9 @@ void flb_cf_dump(struct flb_cf *cf); /* metas */ struct flb_kv *flb_cf_meta_create(struct flb_cf *cf, char *meta, int len); +#define flb_cf_foreach_meta(cf) \ + + void flb_cf_meta_destroy(struct flb_cf *cf, struct flb_cf_meta *meta); void flb_cf_meta_destroy_all(struct flb_cf *cf); @@ -122,4 +134,9 @@ struct flb_kv *flb_cf_property_add(struct flb_cf *cf, struct mk_list *kv_list, char *k_buf, size_t k_len, char *v_buf, size_t v_len); + + +char *flb_cf_section_property_get(struct flb_cf *cf, struct flb_cf_section *s, + char *key); + #endif diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index 10a0f5dc9e1..75b5a0a01a1 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -70,7 +70,7 @@ void flb_cf_destroy(struct flb_cf *cf) flb_free(cf); } -static int get_section_type(char *name) +static enum section_type get_section_type(char *name) { if (strcasecmp(name, "SERVICE") == 0) { return FLB_CF_SERVICE; @@ -128,6 +128,13 @@ struct flb_kv *flb_cf_property_add(struct flb_cf *cf, return kv; } +char *flb_cf_section_property_get(struct flb_cf *cf, struct flb_cf_section *s, + char *key) +{ + (void) cf; + return flb_kv_get_key_value(key, &s->properties); +} + struct flb_kv *flb_cf_meta_create(struct flb_cf *cf, char *meta, int len) { int xlen; @@ -379,7 +386,7 @@ static void dump_section(struct flb_cf_section *s) static void dump_section_list(struct mk_list *list) { - struct mk_list *head; + struct mk_list *head; struct flb_cf_section *s; mk_list_foreach(head, list) { @@ -391,4 +398,38 @@ static void dump_section_list(struct mk_list *list) void flb_cf_dump(struct flb_cf *cf) { dump_section_list(&cf->sections); -} \ No newline at end of file +} + +struct flb_cf *flb_cf_create_from_file(char *file) +{ + int format = FLB_CF_FLUENTBIT; + char *ptr; + struct flb_cf *cf; + + ptr = strrchr(file, '.'); + if (!ptr) { + format = FLB_CF_FLUENTBIT; + } + else { + if (strcasecmp(ptr, ".conf") == 0) { + format = FLB_CF_FLUENTBIT; + } +#ifdef FLB_HAVE_LIBYAML + else if (strcasecmp(ptr, ".yaml") == 0) { + format = FLB_CF_YAML; + } +#endif + } + + if (format == FLB_CF_FLUENTBIT) { + cf = flb_cf_fluentbit_create(file, NULL, 0); + } +#ifdef FLB_HAVE_LIBYAML + else if (format == FLB_CF_YAML) { + cf = flb_cf_yaml_create(file, NULL, 0); + } +#endif + + return cf; + } + From 12cf969f3e2c72ed057bc7021e6c3bf53d5a28b3 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 17 Jan 2022 23:04:05 -0600 Subject: [PATCH 07/51] bin: read configuration by using new config_format API Signed-off-by: Eduardo Silva --- src/fluent-bit.c | 354 ++++++++++++++++++++--------------------------- 1 file changed, 152 insertions(+), 202 deletions(-) diff --git a/src/fluent-bit.c b/src/fluent-bit.c index 76103050b33..4a4bb558e7a 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -53,6 +53,8 @@ #include #include #include +#include +#include #ifdef FLB_HAVE_MTRACE #include @@ -692,12 +694,6 @@ static int filter_set_property(struct flb_filter_instance *filter, char *kv) return ret; } -static void flb_service_conf_err(struct mk_rconf_section *section, char *key) -{ - fprintf(stderr, "Invalid configuration value at %s.%s\n", - section->name, key); -} - static int flb_service_conf_path_set(struct flb_config *config, char *file) { char *end; @@ -723,247 +719,201 @@ static int flb_service_conf_path_set(struct flb_config *config, char *file) return 0; } -static int flb_service_conf(struct flb_config *config, char *file) +static int service_configure_plugin(struct flb_config *config, + struct flb_cf *cf, enum section_type type) { - int ret = -1; + int ret; char *tmp; char *name; + char *s_type; + struct mk_list *list; struct mk_list *head; struct mk_list *h_prop; - struct mk_rconf *fconf = NULL; - struct mk_rconf_entry *entry; - struct mk_rconf_section *section; - struct flb_custom_instance *custom; - struct flb_input_instance *in; - struct flb_output_instance *out; - struct flb_filter_instance *filter; - -#ifdef FLB_HAVE_STATIC_CONF - fconf = flb_config_static_open(file); -#else - fconf = mk_rconf_open(file); -#endif + struct flb_kv *kv; + struct flb_cf_section *s; + void *ins; - if (!fconf) { - return -1; + if (type == FLB_CF_CUSTOM) { + s_type = "custom"; + list = &cf->customs; } - - /* Process all meta commands */ - mk_list_foreach(head, &fconf->metas) { - entry = mk_list_entry(head, struct mk_rconf_entry, _head); - flb_meta_run(config, entry->key, entry->val); + else if (type == FLB_CF_INPUT) { + s_type = "input"; + list = &cf->inputs; } - - /* Set configuration root path */ - flb_service_conf_path_set(config, file); - - /* Validate sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - - if (strcasecmp(section->name, "SERVICE") == 0 || - strcasecmp(section->name, "CUSTOM") == 0 || - strcasecmp(section->name, "INPUT") == 0 || - strcasecmp(section->name, "FILTER") == 0 || - strcasecmp(section->name, "OUTPUT") == 0) { - - /* continue on valid sections */ - continue; - } - - /* Extra sanity checks */ - if (strcasecmp(section->name, "PARSER") == 0 || - strcasecmp(section->name, "MULTILINE_PARSER") == 0) { - fprintf(stderr, - "Sections [MULTILINE_PARSER] and [PARSER] are not valid in " - "the main configuration file. It belongs to \n" - "the 'parsers_file' configuration files.\n"); - } - else { - fprintf(stderr, - "Error: unexpected section [%s] in the main " - "configuration file.\n", section->name); - } - exit(EXIT_FAILURE); + else if (type == FLB_CF_FILTER) { + s_type = "filter"; + list = &cf->filters; } - - /* Read main [SERVICE] section */ - section = mk_rconf_section_get(fconf, "SERVICE"); - if (section) { - /* Iterate properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - /* Set the property */ - flb_config_set_property(config, entry->key, entry->val); - } + else if (type == FLB_CF_OUTPUT) { + s_type = "output"; + list = &cf->outputs; + } + else { + return -1; } - /* Read all [CUSTOM] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "CUSTOM") != 0) { - continue; - } - - /* Get the input plugin name */ - name = s_get_key(section, "name", MK_RCONF_STR); + mk_list_foreach(head, list) { + s = mk_list_entry(head, struct flb_cf_section, _head_section); + name = flb_cf_section_property_get(cf, s, "name"); if (!name) { - flb_service_conf_err(section, "name"); - goto flb_service_conf_end; + flb_error("[config] section '%s' is missing the 'name' property", + s_type); + return -1; } - flb_debug("[service] loading custom plugin: %s", name); - - /* Create an instace of the plugin */ + /* translate the variable */ tmp = flb_env_var_translate(config->env, name); - custom = flb_custom_new(config, tmp, NULL); - mk_mem_free(name); - if (!custom) { - fprintf(stderr, "Custom plugin '%s' cannot be loaded\n", tmp); - flb_sds_destroy(tmp); - goto flb_service_conf_end; - } - flb_sds_destroy(tmp); - /* Iterate other properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "name") == 0) { - continue; - } - - /* Set the property */ - ret = flb_custom_set_property(custom, entry->key, entry->val); - if (ret == -1) { - fprintf(stderr, "Error setting up %s plugin property '%s'\n", - custom->name, entry->key); - goto flb_service_conf_end; - } + /* create an instance of the plugin */ + ins = NULL; + if (type == FLB_CF_CUSTOM) { + ins = flb_custom_new(config, tmp, NULL); } - } - - /* Read all [INPUT] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "INPUT") != 0) { - continue; + else if (type == FLB_CF_INPUT) { + ins = flb_input_new(config, tmp, NULL, FLB_TRUE); } - - /* Get the input plugin name */ - name = s_get_key(section, "Name", MK_RCONF_STR); - if (!name) { - flb_service_conf_err(section, "Name"); - goto flb_service_conf_end; + else if (type == FLB_CF_FILTER) { + ins = flb_filter_new(config, tmp, NULL); } - - flb_debug("[service] loading input: %s", name); - - /* Create an instace of the plugin */ - tmp = flb_env_var_translate(config->env, name); - in = flb_input_new(config, tmp, NULL, FLB_TRUE); - mk_mem_free(name); - if (!in) { - fprintf(stderr, "Input plugin '%s' cannot be loaded\n", tmp); - flb_sds_destroy(tmp); - goto flb_service_conf_end; + else if (type == FLB_CF_OUTPUT) { + ins = flb_output_new(config, tmp, NULL, FLB_TRUE); } flb_sds_destroy(tmp); - /* Iterate other properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "Name") == 0) { + /* validate the instance creation */ + if (!ins) { + flb_error("[config] section '%s' tried to instance a plugin name " + "that don't exists"); + return -1; + } + + /* + * iterate section properties and populate instance by using specific + * api function. + */ + mk_list_foreach(h_prop, &s->properties) { + kv = mk_list_entry(h_prop, struct flb_kv, _head); + if (strcasecmp(kv->key, "name") == 0) { continue; } - /* Set the property */ - ret = flb_input_set_property(in, entry->key, entry->val); + ret = -1; + if (type == FLB_CF_CUSTOM) { + ret = flb_custom_set_property(ins, kv->key, kv->val); + } + else if (type == FLB_CF_INPUT) { + ret = flb_input_set_property(ins, kv->key, kv->val); + } + else if (type == FLB_CF_FILTER) { + ret = flb_filter_set_property(ins, kv->key, kv->val); + } + else if (type == FLB_CF_OUTPUT) { + ret = flb_output_set_property(ins, kv->key, kv->val); + } + if (ret == -1) { - fprintf(stderr, "Error setting up %s plugin property '%s'\n", - in->name, entry->key); - goto flb_service_conf_end; + flb_error("[config] could not configure property '%s' on " + "%s plugin with section name '%s'", + kv->key, s_type, name); } } } - /* Read all [OUTPUT] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "OUTPUT") != 0) { - continue; - } + return 0; +} - /* Get the output plugin name */ - name = s_get_key(section, "Name", MK_RCONF_STR); - if (!name) { - flb_service_conf_err(section, "Name"); - goto flb_service_conf_end; - } +static int service_configure(struct flb_config *config, char *file) +{ + int ret = -1; + struct flb_cf *cf = NULL; + struct flb_cf_section *s; + struct flb_kv *kv; + struct mk_list *head; - /* Create an instace of the plugin */ - tmp = flb_env_var_translate(config->env, name); - out = flb_output_new(config, tmp, NULL, FLB_TRUE); - mk_mem_free(name); - if (!out) { - fprintf(stderr, "Output plugin '%s' cannot be loaded\n", tmp); - flb_sds_destroy(tmp); - goto flb_service_conf_end; - } - flb_sds_destroy(tmp); +#ifdef FLB_HAVE_STATIC_CONF + /* DISABLED/FIXME fconf = flb_config_static_open(file); */ +#else + cf = flb_cf_create_from_file(file); +#endif - /* Iterate other properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "Name") == 0) { - continue; - } + if (!cf) { + return -1; + } - /* Set the property */ - flb_output_set_property(out, entry->key, entry->val); - } + /* Set configuration root path */ + flb_service_conf_path_set(config, file); + + /* Process all meta commands */ + mk_list_foreach(head, &cf->metas) { + kv = mk_list_entry(head, struct flb_kv, _head); + flb_meta_run(config, kv->key, kv->val); } - /* Read all [FILTER] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "FILTER") != 0) { + /* Validate sections */ + mk_list_foreach(head, &cf->sections) { + s = mk_list_entry(head, struct flb_cf_section, _head); + + if (strcasecmp(s->name, "service") == 0 || + strcasecmp(s->name, "custom") == 0 || + strcasecmp(s->name, "input") == 0 || + strcasecmp(s->name, "filter") == 0 || + strcasecmp(s->name, "output") == 0) { + + /* continue on valid sections */ continue; } - /* Get the filter plugin name */ - name = s_get_key(section, "Name", MK_RCONF_STR); - if (!name) { - flb_service_conf_err(section, "Name"); - goto flb_service_conf_end; + + /* Extra sanity checks */ + if (strcasecmp(s->name, "parser") == 0 || + strcasecmp(s->name, "multiline_parser") == 0) { + fprintf(stderr, + "Sections 'multiline_parser' and 'parser' are not valid in " + "the main configuration file. It belongs to \n" + "the 'parsers_file' configuration files.\n"); } - /* Create an instace of the plugin */ - tmp = flb_env_var_translate(config->env, name); - filter = flb_filter_new(config, tmp, NULL); - flb_sds_destroy(tmp); - mk_mem_free(name); - if (!filter) { - flb_service_conf_err(section, "Name"); - goto flb_service_conf_end; + else { + fprintf(stderr, + "Error: unexpected section '%s' in the main " + "configuration file.\n", s->name); } + exit(EXIT_FAILURE); + } - /* Iterate other properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "Name") == 0) { - continue; - } - - /* Set the property */ - flb_filter_set_property(filter, entry->key, entry->val); + /* Read main 'service' section */ + s = cf->service; + if (!s) { + /* Iterate properties */ + mk_list_foreach(head, &s->properties) { + kv = mk_list_entry(head, struct flb_kv, _head); + flb_config_set_property(config, kv->key, kv->val); } } - ret = 0; + ret = service_configure_plugin(config, cf, FLB_CF_CUSTOM); + if (ret == -1) { + goto error; + } -flb_service_conf_end: - if (fconf != NULL) { - mk_rconf_free(fconf); + ret = service_configure_plugin(config, cf, FLB_CF_INPUT); + if (ret == -1) { + goto error; } - return ret; + ret = service_configure_plugin(config, cf, FLB_CF_FILTER); + if (ret == -1) { + goto error; + } + ret = service_configure_plugin(config, cf, FLB_CF_OUTPUT); + if (ret == -1) { + goto error; + } + + flb_cf_destroy(cf); + return 0; + +error: + flb_cf_destroy(cf); + return -1; } int flb_main(int argc, char **argv) @@ -1232,7 +1182,7 @@ int flb_main(int argc, char **argv) } /* Load the service configuration file */ - ret = flb_service_conf(config, cfg_file); + ret = service_configure(config, cfg_file); if (ret != 0) { flb_utils_error(FLB_ERR_CFG_FILE_STOP); } From c03a0afd0b0b20e608fc3b1f3f17ef9853be3906 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 19 Jan 2022 18:27:48 -0600 Subject: [PATCH 08/51] utils: lowercase text errors Signed-off-by: Eduardo Silva --- src/flb_utils.c | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/flb_utils.c b/src/flb_utils.c index d76a83e42f0..ff02221e337 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -61,55 +61,55 @@ void flb_utils_error(int err) switch (err) { case FLB_ERR_CFG_FILE: - msg = "Could not open configuration file"; + msg = "could not open configuration file"; break; case FLB_ERR_CFG_FILE_FORMAT: - msg = "Configuration file contains format errors"; + msg = "configuration file contains format errors"; break; case FLB_ERR_CFG_FILE_STOP: - msg = "Configuration file contains errors"; + msg = "configuration file contains errors"; break; case FLB_ERR_CFG_FLUSH: - msg = "Invalid flush value"; + msg = "invalid flush value"; break; case FLB_ERR_CFG_FLUSH_CREATE: - msg = "Could not create timer for flushing"; + msg = "could not create timer for flushing"; break; case FLB_ERR_CFG_FLUSH_REGISTER: - msg = "Could not register timer for flushing"; + msg = "could not register timer for flushing"; break; case FLB_ERR_INPUT_INVALID: - msg = "Invalid input type"; + msg = "invalid input type"; break; case FLB_ERR_INPUT_UNDEF: - msg = "No Input(s) have been defined"; + msg = "no input(s) have been defined"; break; case FLB_ERR_INPUT_UNSUP: - msg = "Unsupported Input"; + msg = "unsupported Input"; break; case FLB_ERR_OUTPUT_UNDEF: - msg = "You must specify an output target"; + msg = "you must specify an output target"; break; case FLB_ERR_OUTPUT_INVALID: - msg = "Invalid output target"; + msg = "invalid output target"; break; case FLB_ERR_OUTPUT_UNIQ: - msg = "Just one output type is supported"; + msg = "just one output type is supported"; break; case FLB_ERR_FILTER_INVALID: - msg = "Invalid filter plugin"; + msg = "invalid filter plugin"; break; case FLB_ERR_CFG_PARSER_FILE: - msg = "Could not open parser configuration file"; + msg = "could not open parser configuration file"; break; case FLB_ERR_JSON_INVAL: - msg = "Invalid JSON string"; + msg = "invalid JSON string"; break; case FLB_ERR_JSON_PART: - msg = "Truncated JSON string"; + msg = "truncated JSON string"; break; case FLB_ERR_CORO_STACK_SIZE: - msg = "Invalid coroutine stack size"; + msg = "invalid coroutine stack size"; break; } @@ -125,10 +125,7 @@ void flb_utils_error(int err) } else { - fprintf(stderr, - "%sError%s: %s. Aborting\n\n", - ANSI_BOLD ANSI_RED, ANSI_RESET, msg); - + flb_error("%s, aborting.", msg); #ifdef FLB_HAVE_AWS_ERROR_REPORTER if (is_error_reporting_enabled()) { flb_aws_error_reporter_write(error_reporter, msg); From eeff8082ea79a19c2be702194d0c2b06d3943b5c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 19 Jan 2022 18:28:29 -0600 Subject: [PATCH 09/51] config_format: pass context as reference Signed-off-by: Eduardo Silva --- include/fluent-bit/config_format/flb_cf.h | 2 +- .../fluent-bit/config_format/flb_cf_fluentbit.h | 3 ++- include/fluent-bit/config_format/flb_cf_yaml.h | 3 ++- src/config_format/flb_cf_fluentbit.c | 11 ++++++----- src/config_format/flb_cf_yaml.c | 10 ++++++---- src/config_format/flb_config_format.c | 14 ++++++++++---- 6 files changed, 27 insertions(+), 16 deletions(-) diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h index 5dc535d15b9..c68215888a0 100644 --- a/include/fluent-bit/config_format/flb_cf.h +++ b/include/fluent-bit/config_format/flb_cf.h @@ -104,7 +104,7 @@ struct flb_cf { struct flb_cf *flb_cf_create(); -struct flb_cf *flb_cf_create_from_file(char *file); +struct flb_cf *flb_cf_create_from_file(struct flb_cf *cf, char *file); void flb_cf_destroy(struct flb_cf *cf); diff --git a/include/fluent-bit/config_format/flb_cf_fluentbit.h b/include/fluent-bit/config_format/flb_cf_fluentbit.h index 99fbc392c9c..5766f16df39 100644 --- a/include/fluent-bit/config_format/flb_cf_fluentbit.h +++ b/include/fluent-bit/config_format/flb_cf_fluentbit.h @@ -21,6 +21,7 @@ #ifndef FLB_CONFIG_FORMAT_FLUENTBIT_H #define FLB_CONFIG_FORMAT_FLUENTBIT_H -struct flb_cf *flb_cf_fluentbit_create(char *file_path, char *buf, size_t size); +struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, + char *file_path, char *buf, size_t size); #endif \ No newline at end of file diff --git a/include/fluent-bit/config_format/flb_cf_yaml.h b/include/fluent-bit/config_format/flb_cf_yaml.h index b0fba5cae99..43b865092dc 100644 --- a/include/fluent-bit/config_format/flb_cf_yaml.h +++ b/include/fluent-bit/config_format/flb_cf_yaml.h @@ -21,6 +21,7 @@ #ifndef FLB_CONFIG_FORMAT_YAML_H #define FLB_CONFIG_FORMAT_YAML_H -struct flb_cf *flb_cf_yaml_create(char *file_path, char *buf, size_t size); +struct flb_cf *flb_cf_yaml_create(struct flb_cf *cf, + char *file_path, char *buf, size_t size); #endif diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index 59fa3fe8978..60c6cc30c52 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -574,15 +574,17 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) return 0; } -struct flb_cf *flb_cf_fluentbit_create(char *file_path, char *buf, size_t size) +struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, + char *file_path, char *buf, size_t size) { int ret; - struct flb_cf *cf; struct local_ctx ctx; - cf = flb_cf_create(); if (!cf) { - return NULL; + cf = flb_cf_create(); + if (!cf) { + return NULL; + } } local_init(&ctx, file_path); @@ -595,6 +597,5 @@ struct flb_cf *flb_cf_fluentbit_create(char *file_path, char *buf, size_t size) exit(28); } - return cf; } diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c index 4baf1e06ed8..dc9056f37bd 100644 --- a/src/config_format/flb_cf_yaml.c +++ b/src/config_format/flb_cf_yaml.c @@ -477,14 +477,16 @@ static int read_config(struct flb_cf *cf, void *ctx, char *cfg_file) return code; } -struct flb_cf *flb_cf_yaml_create(char *file_path, char *buf, size_t size) +struct flb_cf *flb_cf_yaml_create(struct flb_cf *cf, char *file_path, + char *buf, size_t size) { int ret; - struct flb_cf *cf; - cf = flb_cf_create(); if (!cf) { - return NULL; + cf = flb_cf_create(); + if (!cf) { + return NULL; + } } ret = read_config(cf, NULL, file_path); diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index 75b5a0a01a1..9dce5a81006 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -105,6 +105,13 @@ struct flb_kv *flb_cf_property_add(struct flb_cf *cf, int ret; struct flb_kv *kv; + if (k_len == 0) { + k_len = strlen(k_buf); + } + if (v_len == 0) { + v_len = strlen(v_buf); + } + kv = flb_kv_item_create_len(kv_list, k_buf, k_len, v_buf, v_len); if (!kv) { return NULL; @@ -400,11 +407,10 @@ void flb_cf_dump(struct flb_cf *cf) dump_section_list(&cf->sections); } -struct flb_cf *flb_cf_create_from_file(char *file) +struct flb_cf *flb_cf_create_from_file(struct flb_cf *cf, char *file) { int format = FLB_CF_FLUENTBIT; char *ptr; - struct flb_cf *cf; ptr = strrchr(file, '.'); if (!ptr) { @@ -422,11 +428,11 @@ struct flb_cf *flb_cf_create_from_file(char *file) } if (format == FLB_CF_FLUENTBIT) { - cf = flb_cf_fluentbit_create(file, NULL, 0); + cf = flb_cf_fluentbit_create(cf, file, NULL, 0); } #ifdef FLB_HAVE_LIBYAML else if (format == FLB_CF_YAML) { - cf = flb_cf_yaml_create(file, NULL, 0); + cf = flb_cf_yaml_create(cf, file, NULL, 0); } #endif From fe7abe43763d9d679a728d8084cf917447336adb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 19 Jan 2022 18:29:29 -0600 Subject: [PATCH 10/51] bin: use config_format to compose pipeline from the command line Signed-off-by: Eduardo Silva --- src/fluent-bit.c | 265 +++++++++++++++++++---------------------------- 1 file changed, 104 insertions(+), 161 deletions(-) diff --git a/src/fluent-bit.c b/src/fluent-bit.c index 4a4bb558e7a..526638e5ae6 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -421,30 +421,53 @@ static void help_format_text(void *help_buf, size_t help_size) static void flb_help_plugin(int rc, int format, struct flb_config *config, int type, - struct flb_input_instance *in, - struct flb_filter_instance *filter, - struct flb_output_instance *out) - - + struct flb_cf *cf, + struct flb_cf_section *s) { - struct flb_config_map *m = NULL; struct flb_config_map *opt = NULL; void *help_buf; size_t help_size; + char *name; + struct flb_input_instance *i = NULL; + struct flb_filter_instance *f = NULL; + struct flb_output_instance *o = NULL; flb_banner(); + name = flb_cf_section_property_get(cf, s, "name"); + if (!name) { + exit(EXIT_FAILURE); + } + if (type == PLUGIN_INPUT) { - opt = in->p->config_map; - flb_help_input(in, &help_buf, &help_size); + i = flb_input_new(config, name, 0, FLB_TRUE); + if (!i) { + fprintf(stderr, "invalid input plugin '%s'", name); + return; + } + opt = i->p->config_map; + flb_help_input(i, &help_buf, &help_size); + flb_input_instance_destroy(i); } else if (type == PLUGIN_FILTER) { - opt = filter->p->config_map; - flb_help_filter(filter, &help_buf, &help_size); + f = flb_filter_new(config, name, 0); + if (!f) { + fprintf(stderr, "invalid filter plugin '%s'", name); + return; + } + opt = f->p->config_map; + flb_help_filter(f, &help_buf, &help_size); + flb_filter_instance_destroy(f); } else if (type == PLUGIN_OUTPUT) { - opt = m = out->p->config_map; - flb_help_output(out, &help_buf, &help_size); + o = flb_output_new(config, name, 0, FLB_TRUE); + if (!o) { + fprintf(stderr, "invalid output plugin '%s'", name); + return; + } + opt = o->p->config_map; + flb_help_output(o, &help_buf, &help_size); + flb_output_instance_destroy(o); } if (!opt) { @@ -457,8 +480,8 @@ static void flb_help_plugin(int rc, int format, else if (format == FLB_HELP_JSON) { help_format_json(help_buf, help_size); } - flb_free(help_buf); + flb_free(help_buf); exit(rc); } @@ -581,13 +604,13 @@ static void flb_signal_init() signal(SIGFPE, &flb_signal_handler); } -static int custom_set_property(struct flb_custom_instance *in, char *kv) +static int set_property(struct flb_cf *cf, struct flb_cf_section *s, char *kv) { - int ret; int len; int sep; char *key; char *value; + struct flb_kv *tmp; len = strlen(kv); sep = mk_string_char_search(kv, '=', len); @@ -602,96 +625,13 @@ static int custom_set_property(struct flb_custom_instance *in, char *kv) return -1; } - ret = flb_custom_set_property(in, key, value); - if (ret == -1) { - fprintf(stderr, "[error] setting up '%s' plugin property '%s'\n", - in->p->name, key); - } - - mk_mem_free(key); - return ret; -} - -static int input_set_property(struct flb_input_instance *in, char *kv) -{ - int ret; - int len; - int sep; - char *key; - char *value; - - len = strlen(kv); - sep = mk_string_char_search(kv, '=', len); - if (sep == -1) { - return -1; - } - - key = mk_string_copy_substr(kv, 0, sep); - value = kv + sep + 1; - - if (!key) { - return -1; - } - - ret = flb_input_set_property(in, key, value); - if (ret == -1) { - fprintf(stderr, "[error] setting up '%s' plugin property '%s'\n", - in->p->name, key); - } - - mk_mem_free(key); - return ret; -} - -static int output_set_property(struct flb_output_instance *out, char *kv) -{ - int ret; - int len; - int sep; - char *key; - char *value; - len = strlen(kv); - sep = mk_string_char_search(kv, '=', len); - if (sep == -1) { - return -1; - } - - key = mk_string_copy_substr(kv, 0, sep); - value = kv + sep + 1; - - if (!key) { - return -1; - } - - ret = flb_output_set_property(out, key, value); - mk_mem_free(key); - return ret; -} - -static int filter_set_property(struct flb_filter_instance *filter, char *kv) -{ - int ret; - int len; - int sep; - char *key; - char *value; - - len = strlen(kv); - sep = mk_string_char_search(kv, '=', len); - if (sep == -1) { - return -1; + tmp = flb_cf_property_add(cf, &s->properties, key, 0, value, 0); + if (!tmp) { + fprintf(stderr, "[error] setting up section '%s' plugin property '%s'\n", + s->name, key); } - - key = mk_string_copy_substr(kv, 0, sep); - value = kv + sep + 1; - - if (!key) { - return -1; - } - - ret = flb_filter_set_property(filter, key, value); mk_mem_free(key); - return ret; + return 0; } static int flb_service_conf_path_set(struct flb_config *config, char *file) @@ -784,7 +724,7 @@ static int service_configure_plugin(struct flb_config *config, /* validate the instance creation */ if (!ins) { flb_error("[config] section '%s' tried to instance a plugin name " - "that don't exists"); + "that don't exists", name); return -1; } @@ -823,10 +763,9 @@ static int service_configure_plugin(struct flb_config *config, return 0; } -static int service_configure(struct flb_config *config, char *file) +static int service_configure(struct flb_cf *cf, struct flb_config *config, char *file) { int ret = -1; - struct flb_cf *cf = NULL; struct flb_cf_section *s; struct flb_kv *kv; struct mk_list *head; @@ -834,15 +773,15 @@ static int service_configure(struct flb_config *config, char *file) #ifdef FLB_HAVE_STATIC_CONF /* DISABLED/FIXME fconf = flb_config_static_open(file); */ #else - cf = flb_cf_create_from_file(file); -#endif - - if (!cf) { - return -1; + if (file) { + cf = flb_cf_create_from_file(cf, file); } +#endif /* Set configuration root path */ - flb_service_conf_path_set(config, file); + if (file) { + flb_service_conf_path_set(config, file); + } /* Process all meta commands */ mk_list_foreach(head, &cf->metas) { @@ -882,7 +821,7 @@ static int service_configure(struct flb_config *config, char *file) /* Read main 'service' section */ s = cf->service; - if (!s) { + if (s) { /* Iterate properties */ mk_list_foreach(head, &s->properties) { kv = mk_list_entry(head, struct flb_kv, _head); @@ -908,11 +847,9 @@ static int service_configure(struct flb_config *config, char *file) goto error; } - flb_cf_destroy(cf); return 0; error: - flb_cf_destroy(cf); return -1; } @@ -926,10 +863,11 @@ int flb_main(int argc, char **argv) /* local variables to handle config options */ char *cfg_file = NULL; - struct flb_custom_instance *custom = NULL; - struct flb_input_instance *in = NULL; - struct flb_output_instance *out = NULL; - struct flb_filter_instance *filter = NULL; + + /* config format context */ + struct flb_cf *cf; + struct flb_cf_section *service; + struct flb_cf_section *s; #ifdef FLB_HAVE_LIBBACKTRACE flb_stacktrace_init(argv[0], &flb_st); @@ -990,6 +928,17 @@ int flb_main(int argc, char **argv) } config = ctx->config; + /* Initialize config_format context */ + cf = flb_cf_create(); + if (!cf) { + exit(EXIT_FAILURE); + } + service = flb_cf_section_create(cf, "service", 0); + if (!service) { + flb_cf_destroy(cf); + exit(EXIT_FAILURE); + } + #ifndef FLB_HAVE_STATIC_CONF /* Parse the command line options */ @@ -1000,13 +949,16 @@ int flb_main(int argc, char **argv) switch (opt) { case 'b': - config->storage_path = flb_strdup(optarg); + flb_cf_property_add(cf, &service->properties, + "storage.path", 0, optarg, 0); break; case 'c': cfg_file = flb_strdup(optarg); break; #ifdef FLB_HAVE_FORK case 'd': + flb_cf_property_add(cf, &service->properties, + "daemon", 0, "on", 0); config->daemon = FLB_TRUE; break; #endif @@ -1020,35 +972,36 @@ int flb_main(int argc, char **argv) } break; case 'f': - config->flush = atof(optarg); + flb_cf_property_add(cf, &service->properties, + "flush", 0, optarg, 0); break; case 'C': - custom = flb_custom_new(config, optarg, NULL); - if (!custom) { + s = flb_cf_section_create(cf, "custom", 0); + if (!s) { flb_utils_error(FLB_ERR_CUSTOM_INVALID); } + flb_cf_property_add(cf, &s->properties, "name", 0, optarg, 0); last_plugin = PLUGIN_CUSTOM; break; case 'i': - in = flb_input_new(config, optarg, NULL, FLB_TRUE); - if (!in) { + s = flb_cf_section_create(cf, "input", 0); + if (!s) { flb_utils_error(FLB_ERR_INPUT_INVALID); } + flb_cf_property_add(cf, &s->properties, "name", 0, optarg, 0); last_plugin = PLUGIN_INPUT; break; case 'm': - if (last_plugin == PLUGIN_FILTER) { - flb_filter_set_property(filter, "match", optarg); - } - else if (last_plugin == PLUGIN_OUTPUT) { - flb_output_set_property(out, "match", optarg); + if (last_plugin == PLUGIN_FILTER || last_plugin == PLUGIN_OUTPUT) { + flb_cf_property_add(cf, &s->properties, "match", 0, optarg, 0); } break; case 'o': - out = flb_output_new(config, optarg, NULL, FLB_TRUE); - if (!out) { + s = flb_cf_section_create(cf, "output", 0); + if (!s) { flb_utils_error(FLB_ERR_OUTPUT_INVALID); } + flb_cf_property_add(cf, &s->properties, "name", 0, optarg, 0); last_plugin = PLUGIN_OUTPUT; break; #ifdef FLB_HAVE_PARSER @@ -1060,35 +1013,25 @@ int flb_main(int argc, char **argv) break; #endif case 'F': - filter = flb_filter_new(config, optarg, NULL); - if (!filter) { + s = flb_cf_section_create(cf, "filter", 0); + if (!s) { flb_utils_error(FLB_ERR_FILTER_INVALID); } + flb_cf_property_add(cf, &s->properties, "name", 0, optarg, 0); last_plugin = PLUGIN_FILTER; break; case 'l': - config->log_file = flb_strdup(optarg); + flb_cf_property_add(cf, &service->properties, + "log_file", 0, optarg, 0); break; case 'p': - if (last_plugin == PLUGIN_INPUT) { - ret = input_set_property(in, optarg); - if (ret != 0) { - exit(EXIT_FAILURE); - } - } - else if (last_plugin == PLUGIN_OUTPUT) { - output_set_property(out, optarg); - } - else if (last_plugin == PLUGIN_FILTER) { - filter_set_property(filter, optarg); - } - else if (last_plugin == PLUGIN_CUSTOM) { - custom_set_property(custom, optarg); + if (s) { + set_property(cf, s, optarg); } break; case 't': - if (in) { - flb_input_set_property(in, "tag", optarg); + if (s) { + flb_cf_property_add(cf, &s->properties, "tag", 0, optarg, 0); } break; #ifdef FLB_HAVE_STREAM_PROCESSOR @@ -1103,7 +1046,7 @@ int flb_main(int argc, char **argv) else { flb_help_plugin(EXIT_SUCCESS, FLB_HELP_TEXT, config, - last_plugin, in, filter, out); + last_plugin, cf, s); } break; case 'J': @@ -1112,12 +1055,12 @@ int flb_main(int argc, char **argv) } else { flb_help_plugin(EXIT_SUCCESS, FLB_HELP_JSON, config, - last_plugin, in, filter, out); + last_plugin, cf, s); } break; #ifdef FLB_HAVE_HTTP_SERVER case 'H': - config->http_server = FLB_TRUE; + flb_cf_property_add(cf, &service->properties, "http_server", 0, "on", 0); break; case 'L': if (config->http_listen) { @@ -1178,15 +1121,15 @@ int flb_main(int argc, char **argv) #ifndef FLB_HAVE_STATIC_CONF if (cfg_file) { if (access(cfg_file, R_OK) != 0) { + flb_free(cfg_file); flb_utils_error(FLB_ERR_CFG_FILE); } + } - /* Load the service configuration file */ - ret = service_configure(config, cfg_file); - if (ret != 0) { - flb_utils_error(FLB_ERR_CFG_FILE_STOP); - } - flb_free(cfg_file); + /* Load the service configuration file */ + ret = service_configure(cf, config, cfg_file); + if (ret != 0) { + flb_utils_error(FLB_ERR_CFG_FILE_STOP); } #else ret = flb_service_conf(config, "fluent-bit.conf"); @@ -1195,6 +1138,7 @@ int flb_main(int argc, char **argv) } #endif + /* Check co-routine stack size */ if (config->coro_stack_size < getpagesize()) { flb_utils_error(FLB_ERR_CORO_STACK_SIZE); @@ -1241,7 +1185,6 @@ int flb_main(int argc, char **argv) ret = config->exit_status_code; flb_destroy(ctx); - return ret; } From 80fd171145b6261403a47890375453f1022d8b27 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 19 Jan 2022 18:29:58 -0600 Subject: [PATCH 11/51] tests: internal: config_format: use new config_format prototype Signed-off-by: Eduardo Silva --- tests/internal/config_format_fluentbit.c | 2 +- tests/internal/config_format_yaml.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/internal/config_format_fluentbit.c b/tests/internal/config_format_fluentbit.c index e0e92602e72..e43a4d56bdb 100644 --- a/tests/internal/config_format_fluentbit.c +++ b/tests/internal/config_format_fluentbit.c @@ -14,7 +14,7 @@ void test_basic() { struct flb_cf *cf; - cf = flb_cf_fluentbit_create(FLB_000, NULL, 0); + cf = flb_cf_fluentbit_create(NULL, FLB_000, NULL, 0); TEST_CHECK(cf != NULL); /* Total number of sections */ diff --git a/tests/internal/config_format_yaml.c b/tests/internal/config_format_yaml.c index 6735598e1c9..e84a08f19d5 100644 --- a/tests/internal/config_format_yaml.c +++ b/tests/internal/config_format_yaml.c @@ -14,7 +14,7 @@ void test_basic() { struct flb_cf *cf; - cf = flb_cf_yaml_create(FLB_000, NULL, 0); + cf = flb_cf_yaml_create(NULL, FLB_000, NULL, 0); TEST_CHECK(cf != NULL); if (!cf) { exit(EXIT_FAILURE); From 2d19c97ca81dd78f3b42381505bd60fbf3168d40 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 09:34:38 -0600 Subject: [PATCH 12/51] config_format: fluentbit: do not abort on strrchr() failure Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index 60c6cc30c52..b1b57133091 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -249,11 +249,10 @@ static int local_init(struct local_ctx *ctx, char *file) /* lookup path ending and truncate */ end = strrchr(path, '/'); - if (!end) { - return -1; + if (end) { + end++; + *end = '\0'; } - end++; - *end = '\0'; if (file) { ctx->file = flb_sds_create(file); From 5d82225c1087876296d176c2f7c0dfe81338bfde Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 16:32:32 -0600 Subject: [PATCH 13/51] config_format: allow to reuse 'service' definition Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index b1b57133091..0dfa09962e3 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -474,6 +474,9 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) /* Create new section */ current = flb_cf_section_create(cf, buf + 1, end - 1); + if (!current) { + continue; + } n_keys = 0; continue; } From 1b0b14eccc2787dba11156ace0eeb76eae8994f5 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 16:32:56 -0600 Subject: [PATCH 14/51] config_format: fluentbit: on exception, continue Signed-off-by: Eduardo Silva --- src/config_format/flb_config_format.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index 9dce5a81006..aea06a1b74e 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -258,10 +258,9 @@ struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int /* restrict to only one SERVICE section */ if (type == FLB_CF_SERVICE && cf->service) { - flb_cf_error_set(cf, FLB_CF_ERROR_SERVICE_EXISTS); flb_sds_destroy(s->name); flb_free(s); - return NULL; + return cf->service; } if (type == FLB_CF_SERVICE && !cf->service) { From e4ed1a0d8c15e9540a64429a1a82a7bae7ed91e9 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 16:50:14 -0600 Subject: [PATCH 15/51] tests: internal: config_format: allow 'service' section to be set twice Signed-off-by: Eduardo Silva --- tests/internal/config_format.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/internal/config_format.c b/tests/internal/config_format.c index 9f24fffbd03..53660d692fc 100644 --- a/tests/internal/config_format.c +++ b/tests/internal/config_format.c @@ -44,9 +44,9 @@ void test_api() kv = flb_cf_property_add(cf, &service->properties, " ", 3, "", 0); TEST_CHECK(kv == NULL); - /* try to add another 'SERVICE' section, it should fail */ + /* try to add another 'SERVICE' section, it should return the same one */ s_tmp = flb_cf_section_create(cf, "SERVICE", 7); - TEST_CHECK(s_tmp == NULL); + TEST_CHECK(s_tmp == service); /* add a valid section */ s_tmp = flb_cf_section_create(cf, "INPUT", 5); From b480829bb33d75227ff0e496effb4c66219b6880 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 19:37:16 -0600 Subject: [PATCH 16/51] config: expose new config_format context in main structure Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_config.h | 3 +++ src/flb_config.c | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index 76ed3cd3ef0..985296b7a27 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -69,6 +69,9 @@ struct flb_config { struct mk_rconf *file; + /* main configuration */ + struct flb_cf *cf_main; + flb_sds_t program_name; /* argv[0] */ /* diff --git a/src/flb_config.c b/src/flb_config.c index 4697bf8b126..be4dac1f132 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -41,6 +41,7 @@ #include #include #include +#include #include const char *FLB_CONF_ENV_LOGLEVEL = "FLB_LOG_LEVEL"; @@ -166,6 +167,8 @@ struct flb_config *flb_config_init() { int ret; struct flb_config *config; + struct flb_cf *cf; + struct flb_cf_section *section; config = flb_calloc(1, sizeof(struct flb_config)); if (!config) { @@ -183,6 +186,18 @@ struct flb_config *flb_config_init() /* Is the engine (event loop) actively running ? */ config->is_running = FLB_TRUE; + /* Initialize config_format context */ + cf = flb_cf_create(); + if (!cf) { + return NULL; + } + section = flb_cf_section_create(cf, "service", 0); + if (!section) { + flb_cf_destroy(cf); + return NULL; + } + config->cf_main = cf; + /* Flush */ config->flush = FLB_CONFIG_FLUSH_SECS; config->daemon = FLB_FALSE; @@ -444,6 +459,8 @@ void flb_config_exit(struct flb_config *config) } flb_plugins_unregister(config); + + flb_cf_destroy(config->cf_main); flb_free(config); } From 3be23efb7b68602c70c27b9bf4df156870f42a45 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 19:37:52 -0600 Subject: [PATCH 17/51] bin: release config file optarg and always stop the context Signed-off-by: Eduardo Silva --- src/fluent-bit.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/fluent-bit.c b/src/fluent-bit.c index 526638e5ae6..17f786a9111 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -927,17 +927,8 @@ int flb_main(int argc, char **argv) exit(EXIT_FAILURE); } config = ctx->config; - - /* Initialize config_format context */ - cf = flb_cf_create(); - if (!cf) { - exit(EXIT_FAILURE); - } - service = flb_cf_section_create(cf, "service", 0); - if (!service) { - flb_cf_destroy(cf); - exit(EXIT_FAILURE); - } + cf = config->cf_main; + service = cf->service; #ifndef FLB_HAVE_STATIC_CONF @@ -1128,6 +1119,8 @@ int flb_main(int argc, char **argv) /* Load the service configuration file */ ret = service_configure(cf, config, cfg_file); + flb_free(cfg_file); + if (ret != 0) { flb_utils_error(FLB_ERR_CFG_FILE_STOP); } @@ -1179,10 +1172,13 @@ int flb_main(int argc, char **argv) while (ctx->status == FLB_LIB_OK && exit_signal == 0) { sleep(1); } + if (exit_signal) { flb_signal_exit(exit_signal); } ret = config->exit_status_code; + + flb_stop(ctx); flb_destroy(ctx); return ret; From efedbdf2477d989547e6ea0362015b645f5ba60a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 19:38:11 -0600 Subject: [PATCH 18/51] lib: on stop, always double-check the child thread Signed-off-by: Eduardo Silva --- src/flb_lib.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/flb_lib.c b/src/flb_lib.c index 040f7dc78b2..ba156681c3b 100644 --- a/src/flb_lib.c +++ b/src/flb_lib.c @@ -704,7 +704,15 @@ int flb_stop(flb_ctx_t *ctx) int ret; pthread_t tid; + tid = ctx->config->worker; + if (ctx->status == FLB_LIB_NONE || ctx->status == FLB_LIB_ERROR) { + /* + * There is a chance the worker thread is still active while + * the service exited for some reason (plugin action). Always + * wait and double check that the child thread is not running. + */ + pthread_join(tid, NULL); return 0; } @@ -718,9 +726,11 @@ int flb_stop(flb_ctx_t *ctx) flb_debug("[lib] sending STOP signal to the engine"); - tid = ctx->config->worker; flb_engine_exit(ctx->config); ret = pthread_join(tid, NULL); + if (ret != 0) { + flb_errno(); + } flb_debug("[lib] Fluent Bit engine stopped"); return ret; From 38e28e8a7bc672fd23597409d7b6d49f96f49785 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 22:05:28 -0600 Subject: [PATCH 19/51] config: intialize config_format for parsers Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_config.h | 1 + src/flb_config.c | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index 985296b7a27..5ff6d4e81bf 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -71,6 +71,7 @@ struct flb_config { /* main configuration */ struct flb_cf *cf_main; + struct flb_cf *cf_parsers; flb_sds_t program_name; /* argv[0] */ diff --git a/src/flb_config.c b/src/flb_config.c index be4dac1f132..96af2042b75 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -198,6 +198,9 @@ struct flb_config *flb_config_init() } config->cf_main = cf; + /* config_format for parsers */ + config->cf_parsers = flb_cf_create(); + /* Flush */ config->flush = FLB_CONFIG_FLUSH_SECS; config->daemon = FLB_FALSE; @@ -460,7 +463,12 @@ void flb_config_exit(struct flb_config *config) flb_plugins_unregister(config); - flb_cf_destroy(config->cf_main); + if (config->cf_main) { + flb_cf_destroy(config->cf_main); + } + if (config->cf_parsers) { + flb_cf_destroy(config->cf_parsers); + } flb_free(config); } From 3faf171ea6d0c82721265994e540fb698f90c636 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 22:06:00 -0600 Subject: [PATCH 20/51] parser: use new config_format api Signed-off-by: Eduardo Silva --- src/flb_parser.c | 136 ++++++++++++++++++++++------------------------- 1 file changed, 65 insertions(+), 71 deletions(-) diff --git a/src/flb_parser.c b/src/flb_parser.c index 0445678f7fe..d2571dca614 100644 --- a/src/flb_parser.c +++ b/src/flb_parser.c @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include #include @@ -424,21 +426,20 @@ static int proc_types_str(const char *types_str, struct flb_parser_types **types return i; } -static flb_sds_t get_parser_key(char *key, - struct flb_config *config, - struct mk_rconf_section *section) +static flb_sds_t get_parser_key(struct flb_config *config, + struct flb_cf *cf, struct flb_cf_section *s, + char *key) + { char *tmp; flb_sds_t val; - tmp = mk_rconf_section_get_key(section, key, MK_RCONF_STR); + tmp = flb_cf_section_property_get(cf, s, key); if (!tmp) { return NULL; } val = flb_env_var_translate(config->env, tmp); - flb_free(tmp); - if (!val) { return NULL; } @@ -451,8 +452,8 @@ static flb_sds_t get_parser_key(char *key, return val; } -/* Config file: read [PARSER] definitions */ -static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, +/* Config file: read 'parser' definitions */ +static int parser_conf_file(const char *cfg, struct flb_cf *cf, struct flb_config *config) { flb_sds_t name; @@ -469,11 +470,11 @@ static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, int types_len; struct mk_list *head; struct mk_list *decoders = NULL; - struct mk_rconf_section *section; + struct flb_cf_section *s; struct flb_parser_types *types = NULL; - /* Read all [PARSER] sections */ - mk_list_foreach(head, &fconf->sections) { + /* Read all 'parser' sections */ + mk_list_foreach(head, &cf->parsers) { name = NULL; format = NULL; regex = NULL; @@ -483,68 +484,67 @@ static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, types_str = NULL; tmp_str = NULL; - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "PARSER") != 0) { - continue; - } + /* retrieve the section context */ + s = mk_list_entry(head, struct flb_cf_section, _head_section); - /* Name */ - name = get_parser_key("Name", config, section); + /* name */ + name = get_parser_key(config, cf, s, "name"); if (!name) { flb_error("[parser] no parser 'name' found in file '%s'", cfg); goto fconf_error; } - /* Format */ - format = get_parser_key("Format", config, section); + /* format */ + format = get_parser_key(config, cf, s, "format"); if (!format) { flb_error("[parser] no parser 'format' found for '%s' in file '%s'", name, cfg); goto fconf_error; } - /* Regex (if 'format' == 'regex') */ - regex = get_parser_key("Regex", config, section); + /* regex (if 'format' == 'regex') */ + regex = get_parser_key(config, cf, s, "regex"); if (!regex && strcmp(format, "regex") == 0) { - flb_error("[parser] no parser 'regex' found for '%s' in file '%s", name, cfg); + flb_error("[parser] no parser 'regex' found for '%s' in file '%s", + name, cfg); goto fconf_error; } - /* Skip_Empty_Values */ + /* skip_empty_values */ skip_empty = FLB_TRUE; - tmp_str = get_parser_key("Skip_Empty_Values", config, section); + tmp_str = get_parser_key(config, cf, s, "skip_empty_values"); if (tmp_str) { skip_empty = flb_utils_bool(tmp_str); flb_sds_destroy(tmp_str); } - /* Time_Format */ - time_fmt = get_parser_key("Time_Format", config, section); + /* time_format */ + time_fmt = get_parser_key(config, cf, s, "time_format"); - /* Time_Key */ - time_key = get_parser_key("Time_Key", config, section); + /* time_key */ + time_key = get_parser_key(config, cf, s, "time_key"); - /* Time_Keep */ + /* time_keep */ time_keep = FLB_FALSE; - tmp_str = get_parser_key("Time_Keep", config, section); + tmp_str = get_parser_key(config, cf, s, "time_keep"); if (tmp_str) { time_keep = flb_utils_bool(tmp_str); flb_sds_destroy(tmp_str); } - /* Time_Strict */ + /* time_strict */ time_strict = FLB_TRUE; - tmp_str = get_parser_key("Time_Strict", config, section); + tmp_str = get_parser_key(config, cf, s, "time_strict"); if (tmp_str) { time_strict = flb_utils_bool(tmp_str); flb_sds_destroy(tmp_str); } - /* Time_Offset (UTC offset) */ - time_offset = get_parser_key("Time_Offset", config, section); + /* time_offset (UTC offset) */ + time_offset = get_parser_key(config, cf, s, "time_offset"); - /* Types */ - types_str = get_parser_key("Types", config, section); + /* types */ + types_str = get_parser_key(config, cf, s, "types"); if (types_str) { types_len = proc_types_str(types_str, &types); } @@ -553,7 +553,7 @@ static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, } /* Decoders */ - decoders = flb_parser_decoder_list_create(section); + decoders = flb_parser_decoder_list_create(s); /* Create the parser context */ if (!flb_parser_create(name, format, regex, skip_empty, @@ -612,20 +612,20 @@ static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, } static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, - struct mk_rconf_section *section, + struct flb_cf_section *section, struct flb_config *config) { int ret; char *to_state = NULL; struct mk_list list; struct mk_list *head; - struct mk_rconf_entry *entry; + struct flb_kv *entry; struct flb_slist_entry *from_state; struct flb_slist_entry *regex_pattern; struct flb_slist_entry *tmp; - mk_list_foreach(head, §ion->entries) { - entry = mk_list_entry(head, struct mk_rconf_entry, _head); + mk_list_foreach(head, §ion->properties) { + entry = mk_list_entry(head, struct flb_kv, _head); /* only process 'rule' keys */ if (strcasecmp(entry->key, "rule") != 0) { @@ -692,8 +692,8 @@ static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, } -/* Config file: read [MULTILINE_PARSER] definitions */ -static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, +/* config file: read 'multiline_parser' sections */ +static int multiline_parser_conf_file(const char *cfg, struct flb_cf *cf, struct flb_config *config) { int ret; @@ -709,11 +709,11 @@ static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, int flush_timeout; struct flb_parser *parser_ctx; struct mk_list *head; - struct mk_rconf_section *section; + struct flb_cf_section *s; struct flb_ml_parser *ml_parser; - /* Read all [PARSER] sections */ - mk_list_foreach(head, &fconf->sections) { + /* read all 'multiline_parser' sections */ + mk_list_foreach(head, &cf->multiline_parsers) { ml_parser = NULL; name = NULL; type = -1; @@ -726,20 +726,17 @@ static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, flush_timeout = -1; tmp = NULL; - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "MULTILINE_PARSER") != 0) { - continue; - } + s = mk_list_entry(head, struct flb_cf_section, _head_section); /* name */ - name = get_parser_key("name", config, section); + name = get_parser_key(config, cf, s, "name"); if (!name) { flb_error("[multiline_parser] no 'name' defined in file '%s'", cfg); goto fconf_error; } /* type */ - tmp = get_parser_key("type", config, section); + tmp = get_parser_key(config, cf, s, "type"); if (!tmp) { flb_error("[multiline_parser] no 'type' defined in file '%s'", cfg); goto fconf_error; @@ -755,29 +752,29 @@ static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, } /* match_string */ - match_string = get_parser_key("match_string", config, section); + match_string = get_parser_key(config, cf, s, "match_string"); /* negate */ - tmp = get_parser_key("negate", config, section); + tmp = get_parser_key(config, cf, s, "negate"); if (tmp) { negate = flb_utils_bool(tmp); flb_sds_destroy(tmp); } /* key_content */ - key_content = get_parser_key("key_content", config, section); + key_content = get_parser_key(config, cf, s, "key_content"); /* key_pattern */ - key_pattern = get_parser_key("key_pattern", config, section); + key_pattern = get_parser_key(config, cf, s, "key_pattern"); /* key_group */ - key_group = get_parser_key("key_group", config, section); + key_group = get_parser_key(config, cf, s, "key_group"); /* parser */ - parser = get_parser_key("parser", config, section); + parser = get_parser_key(config, cf, s, "parser"); /* flush_timeout */ - tmp = get_parser_key("flush_timeout", config, section); + tmp = get_parser_key(config, cf, s, "flush_timeout"); if (tmp) { flush_timeout = atoi(tmp); } @@ -795,7 +792,7 @@ static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, /* if type is regex, process rules */ if (type == FLB_ML_REGEX) { - ret = multiline_load_regex_rules(ml_parser, section, config); + ret = multiline_load_regex_rules(ml_parser, s, config); if (ret != 0) { goto fconf_error; } @@ -832,9 +829,9 @@ int flb_parser_conf_file(const char *file, struct flb_config *config) { int ret; char tmp[PATH_MAX + 1]; - const char *cfg = NULL; - struct mk_rconf *fconf; + char *cfg = NULL; struct stat st; + struct flb_cf *cf; #ifndef FLB_HAVE_STATIC_CONF ret = stat(file, &st); @@ -851,33 +848,30 @@ int flb_parser_conf_file(const char *file, struct flb_config *config) } } else { - cfg = file; + cfg = (char *) file; } - fconf = mk_rconf_open(cfg); + cf = flb_cf_create_from_file(config->cf_parsers, cfg); #else - fconf = flb_config_static_open(file); + //DISABLED/FIXME fconf = flb_config_static_open(file); #endif - if (!fconf) { + if (!cf) { return -1; } /* process [PARSER]'s sections */ - ret = parser_conf_file(cfg, fconf, config); + ret = parser_conf_file(cfg, cf, config); if (ret == -1) { - mk_rconf_free(fconf); return -1; } /* processs [MULTILINE_PARSER]'s sections */ - ret = multiline_parser_conf_file(cfg, fconf, config); + ret = multiline_parser_conf_file(cfg, cf, config); if (ret == -1) { - mk_rconf_free(fconf); return -1; } - mk_rconf_free(fconf); return 0; } From d503b5f310c1e0f6e21109562e1f9eceec31dca4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 20 Jan 2022 22:06:19 -0600 Subject: [PATCH 21/51] parser: decoder: use new config_format api Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_parser_decoder.h | 5 +++-- src/flb_parser_decoder.c | 15 ++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/include/fluent-bit/flb_parser_decoder.h b/include/fluent-bit/flb_parser_decoder.h index 1f1c9c8f4d1..265854d1854 100644 --- a/include/fluent-bit/flb_parser_decoder.h +++ b/include/fluent-bit/flb_parser_decoder.h @@ -23,7 +23,7 @@ #include #include -#include +#include /* Decoder behavior */ #define FLB_PARSER_DEC_DEFAULT 0 /* results place as separate keys */ @@ -59,7 +59,8 @@ struct flb_parser_dec { struct mk_list _head; /* link to parser->decoders */ }; -struct mk_list *flb_parser_decoder_list_create(struct mk_rconf_section *section); +struct mk_list *flb_parser_decoder_list_create(struct flb_cf_section *section); + int flb_parser_decoder_list_destroy(struct mk_list *list); int flb_parser_decoder_do(struct mk_list *decoders, const char *in_buf, size_t in_size, diff --git a/src/flb_parser_decoder.c b/src/flb_parser_decoder.c index 2f37730d0f5..1588e453fa7 100644 --- a/src/flb_parser_decoder.c +++ b/src/flb_parser_decoder.c @@ -26,6 +26,8 @@ #include #include #include +#include + #include #define TYPE_OUT_STRING 0 /* unstructured text */ @@ -596,13 +598,12 @@ static struct flb_parser_dec *get_decoder_key_context(const char *key_name, int return dec; } -struct mk_list *flb_parser_decoder_list_create(struct mk_rconf_section *section) +struct mk_list *flb_parser_decoder_list_create(struct flb_cf_section *section) { int c = 0; int type; int backend; int size; - struct mk_rconf_entry *entry; struct mk_list *head; struct mk_list *list = NULL; struct mk_list *split; @@ -611,6 +612,7 @@ struct mk_list *flb_parser_decoder_list_create(struct mk_rconf_section *section) struct flb_split_entry *action; struct flb_parser_dec *dec; struct flb_parser_dec_rule *dec_rule; + struct flb_kv *entry; /* Global list to be referenced by parent parser definition */ list = flb_malloc(sizeof(struct mk_list)); @@ -620,15 +622,14 @@ struct mk_list *flb_parser_decoder_list_create(struct mk_rconf_section *section) } mk_list_init(list); - - mk_list_foreach(head, §ion->entries) { - entry = mk_list_entry(head, struct mk_rconf_entry, _head); + mk_list_foreach(head, §ion->properties) { + entry = mk_list_entry(head, struct flb_kv, _head); /* Lookup for specific Decode rules */ - if (strcasecmp(entry->key, "Decode_Field") == 0) { + if (strcasecmp(entry->key, "decode_field") == 0) { type = FLB_PARSER_DEC_DEFAULT; } - else if (strcasecmp(entry->key, "Decode_Field_As") == 0) { + else if (strcasecmp(entry->key, "decode_field_as") == 0) { type = FLB_PARSER_DEC_AS; } else { From 15669c3d33c9f538b9414e74f1a11c7d87ece4fd Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 25 Jan 2022 12:56:07 -0600 Subject: [PATCH 22/51] config_format: new api to get section by name lookup Signed-off-by: Eduardo Silva --- include/fluent-bit/config_format/flb_cf.h | 1 + src/config_format/flb_config_format.c | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h index c68215888a0..445495d1281 100644 --- a/include/fluent-bit/config_format/flb_cf.h +++ b/include/fluent-bit/config_format/flb_cf.h @@ -126,6 +126,7 @@ void flb_cf_group_destroy(struct flb_cf_group *g); /* sections */ struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int len); +struct flb_cf_section *flb_cf_section_get_by_name(struct flb_cf *cf, char *name); void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s); void flb_cf_section_destroy_all(struct flb_cf *cf); diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index aea06a1b74e..dbdabc38159 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -139,7 +139,7 @@ char *flb_cf_section_property_get(struct flb_cf *cf, struct flb_cf_section *s, char *key) { (void) cf; - return flb_kv_get_key_value(key, &s->properties); + return (char *) flb_kv_get_key_value(key, &s->properties); } struct flb_kv *flb_cf_meta_create(struct flb_cf *cf, char *meta, int len) @@ -296,6 +296,22 @@ struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int return s; } +/* returns the first match of a section that it name matches 'name' parameter */ +struct flb_cf_section *flb_cf_section_get_by_name(struct flb_cf *cf, char *name) +{ + struct mk_list *head; + struct flb_cf_section *s; + + mk_list_foreach(head, &cf->sections) { + s = mk_list_entry(head, struct flb_cf_section, _head); + if (strcasecmp(s->name, name) == 0) { + return s; + } + } + + return NULL; +} + void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s) { struct mk_list *tmp; @@ -420,7 +436,7 @@ struct flb_cf *flb_cf_create_from_file(struct flb_cf *cf, char *file) format = FLB_CF_FLUENTBIT; } #ifdef FLB_HAVE_LIBYAML - else if (strcasecmp(ptr, ".yaml") == 0) { + else if (strcasecmp(ptr, ".yaml") == 0 || strcasecmp(ptr, ".yml") == 0) { format = FLB_CF_YAML; } #endif From 4be978832b15e9fc5359dfac05b1098f74743bba Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 25 Jan 2022 12:56:56 -0600 Subject: [PATCH 23/51] upstream: ha: use new config format reader Signed-off-by: Eduardo Silva --- src/flb_upstream_ha.c | 102 +++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 57 deletions(-) diff --git a/src/flb_upstream_ha.c b/src/flb_upstream_ha.c index 9a719a839fc..347cf9c983a 100644 --- a/src/flb_upstream_ha.c +++ b/src/flb_upstream_ha.c @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include #include @@ -108,7 +110,8 @@ struct flb_upstream_node *flb_upstream_ha_node_get(struct flb_upstream_ha *ctx) } static struct flb_upstream_node *create_node(int id, - struct mk_rconf_section *s, + struct flb_cf *cf, + struct flb_cf_section *s, struct flb_config *config) { int i; @@ -131,7 +134,7 @@ static struct flb_upstream_node *create_node(int id, char *tls_key_file = NULL; char *tls_key_passwd = NULL; struct mk_list *head; - struct mk_rconf_entry *entry; + struct flb_kv *entry; struct flb_hash *ht; const char *known_keys[] = {"name", "host", "port", "tls", "tls.vhost", "tls.verify", "tls.debug", @@ -141,7 +144,7 @@ static struct flb_upstream_node *create_node(int id, struct flb_upstream_node *node; /* name */ - name = mk_rconf_section_get_key(s, "name", MK_RCONF_STR); + name = flb_cf_section_property_get(cf, s, "name"); if (!name) { flb_error("[upstream_ha] no 'name' has been set on node #%i", id + 1); @@ -149,7 +152,7 @@ static struct flb_upstream_node *create_node(int id, } /* host */ - host = mk_rconf_section_get_key(s, "host", MK_RCONF_STR); + host = flb_cf_section_property_get(cf, s, "host"); if (!host) { flb_error("[upstream_ha] no 'host' has been set on node #%i", id + 1); @@ -157,7 +160,7 @@ static struct flb_upstream_node *create_node(int id, } /* port */ - port = mk_rconf_section_get_key(s, "port", MK_RCONF_STR); + port = flb_cf_section_property_get(cf, s, "port"); if (!port) { flb_error("[upstream_ha] no 'port' has been set on node #%i", id + 1); @@ -165,44 +168,40 @@ static struct flb_upstream_node *create_node(int id, } /* tls */ - tmp = mk_rconf_section_get_key(s, "tls", MK_RCONF_STR); + tmp = flb_cf_section_property_get(cf, s, "tls"); if (tmp) { tls = flb_utils_bool(tmp); - flb_free(tmp); } /* tls.verify */ - tmp = mk_rconf_section_get_key(s, "tls.verify", MK_RCONF_STR); + tmp = flb_cf_section_property_get(cf, s, "tls.verify"); if (tmp) { tls_verify = flb_utils_bool(tmp); - flb_free(tmp); } /* tls.debug */ - tmp = mk_rconf_section_get_key(s, "tls.debug", MK_RCONF_STR); + tmp = flb_cf_section_property_get(cf, s, "tls.debug"); if (tmp) { tls_debug = atoi(tmp); - flb_free(tmp); } /* tls.vhost */ - tls_vhost = mk_rconf_section_get_key(s, "tls.vhost", MK_RCONF_STR); + tls_vhost = flb_cf_section_property_get(cf, s, "tls.vhost"); /* tls.ca_path */ - tls_ca_path = mk_rconf_section_get_key(s, "tls.ca_path", MK_RCONF_STR); + tls_ca_path = flb_cf_section_property_get(cf, s, "tls.ca_path"); /* tls.ca_file */ - tls_ca_file = mk_rconf_section_get_key(s, "tls.ca_file", MK_RCONF_STR); + tls_ca_file = flb_cf_section_property_get(cf, s, "tls.ca_file"); /* tls.crt_file */ - tls_crt_file = mk_rconf_section_get_key(s, "tls.crt_file", MK_RCONF_STR); + tls_crt_file = flb_cf_section_property_get(cf, s, "tls.crt_file"); /* tls.key_file */ - tls_key_file = mk_rconf_section_get_key(s, "tls.key_file", MK_RCONF_STR); + tls_key_file = flb_cf_section_property_get(cf, s, "tls.key_file"); /* tls.key_file */ - tls_key_passwd = mk_rconf_section_get_key(s, "tls.key_passwd", - MK_RCONF_STR); + tls_key_passwd = flb_cf_section_property_get(cf, s, "tls.key_passwd"); /* * Create hash table to store unknown key/values that might be used @@ -211,16 +210,15 @@ static struct flb_upstream_node *create_node(int id, ht = flb_hash_create(FLB_HASH_EVICT_NONE, 32, 256); if (!ht) { flb_error("[upstream_ha] error creating hash table"); - node = NULL; - goto error; + return NULL; } /* * Iterate mk_rconf section internals, find all unknown keys and add * them to the hash table associated to the node. */ - mk_list_foreach(head, &s->entries) { - entry = mk_list_entry(head, struct mk_rconf_entry, _head); + mk_list_foreach(head, &s->properties) { + entry = mk_list_entry(head, struct flb_kv, _head); /* If this is a known entry, just skip it */ skip = FLB_FALSE; @@ -234,8 +232,8 @@ static struct flb_upstream_node *create_node(int id, continue; } - klen = strlen(entry->key); - vlen = strlen(entry->val); + klen = flb_sds_len(entry->key); + vlen = flb_sds_len(entry->val); /* Always store keys in lowercase */ for (i = 0; i < klen; i++) { @@ -255,11 +253,6 @@ static struct flb_upstream_node *create_node(int id, tls_debug, tls_vhost, tls_ca_path, tls_ca_file, tls_crt_file, tls_key_file, tls_key_passwd, ht, config); - error: - flb_free(name); - flb_free(host); - flb_free(port); - return node; } @@ -272,13 +265,12 @@ struct flb_upstream_ha *flb_upstream_ha_from_file(const char *file, const char *cfg = NULL; char *tmp; char path[PATH_MAX + 1]; - struct mk_rconf_section *u_section; - struct mk_rconf_section *n_section; - struct mk_rconf *fconf; struct stat st; struct mk_list *head; struct flb_upstream_ha *ups; struct flb_upstream_node *node; + struct flb_cf *cf = NULL; + struct flb_cf_section *section; #ifndef FLB_HAVE_STATIC_CONF ret = stat(file, &st); @@ -297,70 +289,66 @@ struct flb_upstream_ha *flb_upstream_ha_from_file(const char *file, cfg = file; } flb_debug("[upstream_ha] opening file %s", cfg); - fconf = mk_rconf_open(cfg); + cf = flb_cf_create_from_file(NULL, (char *) cfg); #else - fconf = flb_config_static_open(file); + //DISABLED/FIXME fconf = flb_config_static_open(file); #endif - if (!fconf) { + if (!cf) { return NULL; } - /* First section must be [UPSTREAM] */ - u_section = mk_list_entry_first(&fconf->sections, - struct mk_rconf_section, _head); - if (strcasecmp(u_section->name, "UPSTREAM") != 0) { - flb_error("[upstream_ha] invalid first section name, " - "expected UPSTREAM"); - mk_rconf_free(fconf); + /* 'upstream' sections are under enum section_type FLB_CF_OTHER */ + section = flb_cf_section_get_by_name(cf, "upstream"); + if (!section) { + flb_error("[upstream_ha] section name 'upstream' could not be found"); + flb_cf_destroy(cf); return NULL; } - /* Get Upstream name */ - tmp = mk_rconf_section_get_key(u_section, "name", MK_RCONF_STR); + /* upstream name */ + tmp = flb_cf_section_property_get(cf, section, "name"); if (!tmp) { - flb_error("[upstream_ha] missing name for upstream at %s", file); - mk_rconf_free(fconf); + flb_error("[upstream_ha] missing name for upstream at %s", cfg); + flb_cf_destroy(cf); return NULL; } ups = flb_upstream_ha_create(tmp); if (!ups) { flb_error("[upstream_ha] cannot create context"); - mk_rconf_free(fconf); + flb_cf_destroy(cf); return NULL; } - /* Register [NODE] sections */ - mk_list_foreach(head, &fconf->sections) { - n_section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(n_section->name, "NODE") != 0) { + /* 'node' sections */ + mk_list_foreach(head, &cf->sections) { + section = mk_list_entry(head, struct flb_cf_section, _head); + if (strcasecmp(section->name, "node") != 0) { continue; } /* Read section info and create a Node context */ - node = create_node(c, n_section, config); + node = create_node(c, cf, section, config); if (!node) { flb_error("[upstream_ha] cannot register node on upstream '%s'", tmp); - mk_rconf_free(fconf); flb_upstream_ha_destroy(ups); - flb_free(tmp); + flb_cf_destroy(cf); return NULL; } flb_upstream_ha_node_add(ups, node); c++; } - flb_free(tmp); if (c == 0) { flb_error("[upstream_ha] no nodes defined"); - mk_rconf_free(fconf); flb_upstream_ha_destroy(ups); + flb_cf_destroy(cf); return NULL; } - mk_rconf_free(fconf); + flb_cf_destroy(cf); return ups; } From edbee76392df29317ea41ac9df99b1d40817981c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 25 Jan 2022 21:40:10 -0600 Subject: [PATCH 24/51] config_format: dump api now handles groups information Signed-off-by: Eduardo Silva --- src/config_format/flb_config_format.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index dbdabc38159..e958b6f1955 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -389,21 +389,44 @@ static char *section_type_str(int type) static void dump_section(struct flb_cf_section *s) { struct mk_list *head; + struct mk_list *p_head; struct flb_kv *kv; + struct flb_cf_group *g; printf("> section:\n name: %s\n type: %s\n", s->name, section_type_str(s->type)); if (mk_list_size(&s->properties) > 0) { printf(" properties:\n"); - mk_list_foreach(head, &s->properties) { - kv = mk_list_entry(head, struct flb_kv, _head); + mk_list_foreach(p_head, &s->properties) { + kv = mk_list_entry(p_head, struct flb_kv, _head); printf(" - %-15s: %s\n", kv->key, kv->val); } } else { printf(" properties: NONE\n"); } + + if (mk_list_size(&s->groups) <= 0) { + printf(" groups : NONE\n"); + return; + } + + mk_list_foreach(head, &s->groups) { + g = mk_list_entry(head, struct flb_cf_group, _head); + printf(" > group:\n name: %s\n", g->name); + + if (mk_list_size(&g->properties) > 0) { + printf(" properties:\n"); + mk_list_foreach(p_head, &g->properties) { + kv = mk_list_entry(p_head, struct flb_kv, _head); + printf(" - %-11s: %s\n", kv->key, kv->val); + } + } + else { + printf(" properties: NONE\n"); + } + } } static void dump_section_list(struct mk_list *list) From add3606844edd7188aa70324ee428eff89def693 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 25 Jan 2022 21:40:51 -0600 Subject: [PATCH 25/51] config_format: fluentbit: expose new 'groups' feature Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 132 +++++++++++++++++++++------ 1 file changed, 104 insertions(+), 28 deletions(-) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index 0dfa09962e3..d91388697a4 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -41,7 +41,12 @@ #define PATH_MAX MAX_PATH #endif -#define FLB_CF_BUF_SIZE 4096 +#define FLB_CF_BUF_SIZE 4096 + +/* indent checker return codes */ +#define INDENT_ERROR -1 +#define INDENT_OK 0 +#define INDENT_GROUP_CONTENT 1 /* Included file by configuration */ struct local_file { @@ -308,11 +313,15 @@ static int is_file_included(struct local_ctx *ctx, const char *path) return FLB_FALSE; } -static int check_indent(const char *line, const char *indent) +static int check_indent(const char *line, const char *indent, int *out_level) { + int extra = 0; + int level = 0; + while (*line == *indent && *indent) { line++; indent++; + level++; } if (*indent != '\0') { @@ -322,15 +331,27 @@ static int check_indent(const char *line, const char *indent) else { flb_error("[config] indentation level is too low"); } - return -1; + return INDENT_ERROR;; } if (isblank(*line)) { - flb_error("[config] Extra indentation level found"); + /* check if we have a 'group' key/value line */ + while (isblank(*line)) { + line++; + extra++; + } + + if (extra == level) { + *out_level = level + extra; + return INDENT_GROUP_CONTENT; + } + + flb_error("[config] extra indentation level found"); return -1; } - return 0; + *out_level = level; + return INDENT_OK; } static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) @@ -338,12 +359,14 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) int i; int len; int ret; + int end; + int level; int line = 0; int indent_len = -1; int n_keys = 0; - char *key; + char *key = NULL; int key_len; - char *val; + char *val = NULL; int val_len; char *buf; char tmp[PATH_MAX]; @@ -352,7 +375,9 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) struct stat st; struct local_file *file; struct flb_cf_meta *meta; - struct flb_cf_section *current = NULL; + struct flb_cf_section *current_section = NULL; + struct flb_cf_group *current_group = NULL; + struct flb_kv *kv; FILE *f; @@ -460,23 +485,25 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) /* Section definition */ if (buf[0] == '[') { - int end = -1; + current_group = NULL; + end = char_search(buf, ']', len); if (end > 0) { /* * Before to add a new section, lets check the previous * one have at least one key set */ - if (current && n_keys == 0) { + if (current_section && n_keys == 0) { config_warn(cfg_file, line, "previous section did not have keys"); } /* Create new section */ - current = flb_cf_section_create(cf, buf + 1, end - 1); - if (!current) { + current_section = flb_cf_section_create(cf, buf + 1, end - 1); + if (!current_section) { continue; } + current_group = NULL; n_keys = 0; continue; } @@ -503,46 +530,95 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) } /* Validate indentation level */ - if (check_indent(buf, indent) < 0) { - config_error(cfg_file, line, "Invalid indentation level"); - flb_sds_destroy(key); - flb_sds_destroy(val); + ret = check_indent(buf, indent, &level); + if (ret == INDENT_ERROR) { + config_error(cfg_file, line, "invalid indentation level"); return -1; } - - if (buf[indent_len] == '#' || indent_len == len) { - continue; + else { + if (ret == INDENT_OK && current_group) { + current_group = NULL; + } + indent_len = level; } - if (len - indent_len >= 3 && strncmp(buf + indent_len, "---", 3) == 0) { + if (buf[indent_len] == '#' || indent_len == len) { continue; } - /* Get the separator */ + /* get the key value separator */ i = char_search(buf + indent_len, ' ', len - indent_len); /* key */ key = buf + indent_len; key_len = i; + if (!key || i < 0) { + config_error(cfg_file, line, "undefined key"); + flb_free(buf); + return -1; + } + + /* Check possible start of a group */ + if (key[0] == '[') { + end = char_search(key, ']', len - indent_len); + if (end == -1) { + config_error(cfg_file, line, "expected a valid group name: [..]"); + flb_free(buf); + return -1; + } + + if (!current_section) { + config_warn(cfg_file, line, + "current group don't have a parent section"); + flb_free(buf); + return -1; + } + + /* check if a previous group exists with one key */ + if (current_group && n_keys == 0) { + config_warn(cfg_file, line, "previous group did not have keys"); + flb_free(buf); + return -1; + } + + /* Create new group */ + current_group = flb_cf_group_create(cf, current_section, + key + 1, end - 1); + if (!current_group) { + continue; + } + n_keys = 0; + + /* continue processing since we need key/value pairs */ + continue; + } + /* val */ val = buf + indent_len + i + 1; val_len = len - indent_len - i - 1; if (!key || !val || i < 0) { - config_error(cfg_file, line, "Each key must have a value"); + config_error(cfg_file, line, "each key must have a value"); return -1; } if (val_len == 0) { - config_error(cfg_file, line, "Key has an empty value"); + config_error(cfg_file, line, "key has an empty value"); return -1; } - /* Register entry: key and val are copied as duplicated */ - kv = flb_cf_property_add(cf, ¤t->properties, - key, key_len, - val, val_len); + /* register entry: key and val are copied as duplicated */ + if (current_group) { + kv = flb_cf_property_add(cf, ¤t_group->properties, + key, key_len, + val, val_len); + } + else { + kv = flb_cf_property_add(cf, ¤t_section->properties, + key, key_len, + val, val_len); + } if (!kv) { config_error(cfg_file, line, "could not allocate key value pair"); return -1; @@ -596,7 +672,7 @@ struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, local_exit(&ctx); if (ret == -1) { - exit(28); + flb_cf_destroy(cf); } return cf; From 9835ed28a7ec7ad582b52671a74e6ee95b4b342b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 25 Jan 2022 21:45:40 -0600 Subject: [PATCH 26/51] tests: internal: config_format: fluentbit: check groups handling Signed-off-by: Eduardo Silva --- tests/internal/config_format_fluentbit.c | 17 ++++++++++++++++- .../internal/data/config_format/fluent-bit.conf | 8 ++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/internal/config_format_fluentbit.c b/tests/internal/config_format_fluentbit.c index e43a4d56bdb..99d07fd9c09 100644 --- a/tests/internal/config_format_fluentbit.c +++ b/tests/internal/config_format_fluentbit.c @@ -12,7 +12,10 @@ /* data/config_format/fluent-bit.conf */ void test_basic() { + struct mk_list *head; struct flb_cf *cf; + struct flb_cf_section *s; + struct flb_cf_group *g; cf = flb_cf_fluentbit_create(NULL, FLB_000, NULL, 0); TEST_CHECK(cf != NULL); @@ -22,7 +25,9 @@ void test_basic() /* SERVICE check */ TEST_CHECK(cf->service != NULL); - TEST_CHECK(mk_list_size(&cf->service->properties) == 3); + if (cf->service) { + TEST_CHECK(mk_list_size(&cf->service->properties) == 3); + } /* Meta commands */ TEST_CHECK(mk_list_size(&cf->metas) == 2); @@ -36,6 +41,16 @@ void test_basic() TEST_CHECK(mk_list_size(&cf->outputs) == 1); TEST_CHECK(mk_list_size(&cf->others) == 1); + /* groups */ + s = flb_cf_section_get_by_name(cf, "input"); + TEST_CHECK(s != NULL); + TEST_CHECK(mk_list_size(&s->groups) == 2); + + mk_list_foreach(head, &s->groups) { + g = mk_list_entry(head, struct flb_cf_group, _head); + TEST_CHECK(mk_list_size(&g->properties) == 2); + } + printf("\n"); flb_cf_dump(cf); diff --git a/tests/internal/data/config_format/fluent-bit.conf b/tests/internal/data/config_format/fluent-bit.conf index 67b1cbbf473..72d6c18cd90 100644 --- a/tests/internal/data/config_format/fluent-bit.conf +++ b/tests/internal/data/config_format/fluent-bit.conf @@ -19,6 +19,14 @@ name tail path /var/log/containers/*.log + [GROUP 1] + key1 aa + key2 bb + + [GROUP 2] + key3 cc + key4 dd + [FILTER] name stdout match * From 87bc1371bb9ce24b99fbc913d44cb8192797daa4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 25 Jan 2022 22:29:47 -0600 Subject: [PATCH 27/51] config_format: yaml: add support for 'groups' Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_yaml.c | 51 +++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c index dc9056f37bd..3f950dcf6f0 100644 --- a/src/config_format/flb_cf_yaml.c +++ b/src/config_format/flb_cf_yaml.c @@ -59,6 +59,9 @@ enum state { STATE_PLUGIN_KEY, STATE_PLUGIN_VAL, + STATE_GROUP_KEY, + STATE_GROUP_VAL, + STATE_STOP /* end state */ }; @@ -79,6 +82,9 @@ struct parser_state { /* active section */ struct flb_cf_section *cf_section; + /* active group */ + struct flb_cf_group *cf_group; + /* internal variables for checks */ int service_set; int customs_set; @@ -412,12 +418,57 @@ static int consume_event(struct flb_cf *cf, struct parser_state *s, flb_sds_destroy(s->key); flb_sds_destroy(s->val); break; + case YAML_MAPPING_START_EVENT: /* start a new group */ + s->state = STATE_GROUP_KEY; + s->cf_group = flb_cf_group_create(cf, s->cf_section, + s->key, flb_sds_len(s->key)); + flb_sds_destroy(s->key); + if (!s->cf_group) { + return YAML_FAILURE; + } + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_GROUP_KEY: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_GROUP_VAL; + value = (char *) event->data.scalar.value; + s->key = flb_sds_create(value); + break; + case YAML_MAPPING_END_EVENT: + s->cf_group = NULL; + s->state = STATE_PLUGIN_KEY; + break; default: yaml_error_event(s, event); return YAML_FAILURE; } break; + case STATE_GROUP_VAL: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_GROUP_KEY; + value = (char *) event->data.scalar.value; + s->val = flb_sds_create(value); + + /* register key/value pair as a property */ + flb_cf_property_add(cf, &s->cf_group->properties, + s->key, flb_sds_len(s->key), + s->val, flb_sds_len(s->val)); + flb_sds_destroy(s->key); + flb_sds_destroy(s->val); + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; case STATE_STOP: break; From 400682fff035f5c4e288ee540a66577f81457ec3 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 25 Jan 2022 22:30:27 -0600 Subject: [PATCH 28/51] tests: config_format: yaml: test new 'groups' feature Signed-off-by: Eduardo Silva --- tests/internal/config_format_yaml.c | 16 ++++++++++++++-- .../internal/data/config_format/fluent-bit.yaml | 6 ++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/internal/config_format_yaml.c b/tests/internal/config_format_yaml.c index e84a08f19d5..dc1e63362de 100644 --- a/tests/internal/config_format_yaml.c +++ b/tests/internal/config_format_yaml.c @@ -12,7 +12,10 @@ /* data/config_format/fluent-bit.yaml */ void test_basic() { - struct flb_cf *cf; + struct mk_list *head; + struct flb_cf *cf; + struct flb_cf_section *s; + struct flb_cf_group *g; cf = flb_cf_yaml_create(NULL, FLB_000, NULL, 0); TEST_CHECK(cf != NULL); @@ -39,9 +42,18 @@ void test_basic() TEST_CHECK(mk_list_size(&cf->outputs) == 2); TEST_CHECK(mk_list_size(&cf->others) == 1); + /* groups */ + s = flb_cf_section_get_by_name(cf, "input"); + TEST_CHECK(s != NULL); + TEST_CHECK(mk_list_size(&s->groups) == 2); + + mk_list_foreach(head, &s->groups) { + g = mk_list_entry(head, struct flb_cf_group, _head); + TEST_CHECK(mk_list_size(&g->properties) == 2); + } + printf("\n"); flb_cf_dump(cf); - flb_cf_destroy(cf); diff --git a/tests/internal/data/config_format/fluent-bit.yaml b/tests/internal/data/config_format/fluent-bit.yaml index ac1bbe948b1..8ce2eb17560 100644 --- a/tests/internal/data/config_format/fluent-bit.yaml +++ b/tests/internal/data/config_format/fluent-bit.yaml @@ -11,6 +11,12 @@ inputs: tail: path: "/var/log/containers/*.log" parser: docker + group1: + key1: aa + key2: bb + group2: + key3: cc + key4: dd tail: path: "/var/log/messages" From 07325e7d20c7f1ae180e59b4dce74aa03148bfb3 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 16:05:57 -0600 Subject: [PATCH 29/51] config_static: use new config_format API Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_config.h | 2 +- src/flb_config_static.c | 367 ++------------------------------ 2 files changed, 19 insertions(+), 350 deletions(-) diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index 5ff6d4e81bf..f6353beeab4 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -259,7 +259,7 @@ int flb_config_set_program_name(struct flb_config *config, char *name); int set_log_level_from_env(struct flb_config *config); #ifdef FLB_HAVE_STATIC_CONF -struct mk_rconf *flb_config_static_open(const char *file); +struct flb_cf *flb_config_static_open(struct flb_config *config, const char *file); #endif struct flb_service_config { diff --git a/src/flb_config_static.c b/src/flb_config_static.c index eca662293a0..066f30fc84f 100644 --- a/src/flb_config_static.c +++ b/src/flb_config_static.c @@ -19,349 +19,27 @@ */ #include -#include +#include #include -#include -#include - -static int is_file_included(struct mk_rconf *conf, const char *path) -{ - struct mk_list *head; - struct mk_rconf_file *file; - - mk_list_foreach(head, &conf->includes) { - file = mk_list_entry(head, struct mk_rconf_file, _head); - if (strcmp(file->path, path) == 0) { - return MK_TRUE; - } - } - - return MK_FALSE; -} - -static int rconf_meta_add(struct mk_rconf *conf, char *buf, int len) -{ - int xlen; - char *p; - char *tmp; - struct mk_rconf_entry *meta; - - if (buf[0] != '@') { - return -1; - } - - meta = mk_mem_alloc(sizeof(struct mk_rconf_entry)); - if (!meta) { - perror("malloc"); - return -1; - } - - p = buf; - tmp = strchr(p, ' '); - xlen = (tmp - p); - meta->key = mk_string_copy_substr(buf, 1, xlen); - mk_string_trim(&meta->key); - - meta->val = mk_string_copy_substr(buf, xlen + 1, len); - mk_string_trim(&meta->val); - - mk_list_add(&meta->_head, &conf->metas); - return 0; -} - -static void rconf_section_entry_add(struct mk_rconf *conf, - const char *key, const char *val) -{ - struct mk_rconf_section *section; - struct mk_rconf_entry *new; - struct mk_list *head = &conf->sections; - - if (mk_list_is_empty(&conf->sections) == 0) { - mk_err("Error: there are not sections available on %s!", conf->file); - return; - } - - /* Last section */ - section = mk_list_entry_last(head, struct mk_rconf_section, _head); - - /* Alloc new entry */ - new = mk_mem_alloc(sizeof(struct mk_rconf_entry)); - new->key = mk_string_dup(key); - new->val = mk_string_dup(val); - - mk_list_add(&new->_head, §ion->entries); -} - -struct mk_rconf_section *rconf_section_add(struct mk_rconf *conf, - char *name) -{ - struct mk_rconf_section *new; - - /* Alloc section node */ - new = mk_mem_alloc(sizeof(struct mk_rconf_section)); - new->name = mk_string_dup(name); - mk_list_init(&new->entries); - mk_list_add(&new->_head, &conf->sections); - - return new; -} - -/* - * Helper function to simulate a fgets(2) but instead of using - * a real file stream uses the data buffer provided. - */ -static int static_fgets(char *out, size_t size, const char *data, size_t *off) -{ - size_t len; - const char *start = data + *off; - char *end; - - end = strchr(start, '\n'); - - if (!end || *off >= size) { - len = size - *off - 1; - memcpy(out, start, len); - out[len] = '\0'; - *off += len + 1; - return 0; - } - - len = end - start; - if (len >= size) { - len = size - 1; - } - memcpy(out, start, len); - out[len] = '\0'; - *off += len + 1; - - return 1; -} - -static int flb_config_static_read(struct mk_rconf *conf, - const char *fname, const char *data) -{ - int i; - int len; - int ret; - int line = 0; - int indent_len = -1; - int n_keys = 0; - char *buf; - char *section = NULL; - char *indent = NULL; - char *key, *val; - size_t off; - struct mk_rconf_file *file; - struct mk_rconf_section *current = NULL; - - /* Check this file have not been included before */ - ret = is_file_included(conf, fname); - if (ret == MK_TRUE) { - mk_err("[config] file already included %s", fname); - return -1; - } - - conf->level++; - - /* Allocate temporary buffer to read file content */ - buf = mk_mem_alloc(MK_RCONF_KV_SIZE); - if (!buf) { - perror("malloc"); - return -1; - } - - /* looking for configuration directives */ - off = 0; - while (static_fgets(buf, MK_RCONF_KV_SIZE, data, &off)) { - len = strlen(buf); - if (buf[len - 1] == '\n') { - buf[--len] = 0; - if (len && buf[len - 1] == '\r') { - buf[--len] = 0; - } - } - - /* Line number */ - line++; - - if (!buf[0]) { - continue; - } - - /* Skip commented lines */ - if (buf[0] == '#') { - continue; - } - - if (len > 9 && strncasecmp(buf, "@INCLUDE ", 9) == 0) { - if (strchr(buf + 9, '*') != NULL) { - //ret = mk_rconf_read_glob(conf, buf + 9); - } - else { - //ret = flb_config_static_read(conf, - // const char *fname, const char *data) - //ret = mk_rconf_read(conf, buf + 9); - } - if (ret == -1) { - conf->level--; - if (indent) { - mk_mem_free(indent); - } - mk_mem_free(buf); - return -1; - } - continue; - } - else if (buf[0] == '@' && len > 3) { - ret = rconf_meta_add(conf, buf, len); - if (ret == -1) { - if (indent) { - mk_mem_free(indent); - } - mk_mem_free(buf); - return -1; - } - continue; - } - - /* Section definition */ - if (buf[0] == '[') { - int end = -1; - end = mk_string_char_search(buf, ']', len); - if (end > 0) { - /* - * Before to add a new section, lets check the previous - * one have at least one key set - */ - if (current && n_keys == 0) { - flb_warn("[static conf: %s] " - "Section do not have keys", fname); - } - - /* Create new section */ - section = mk_string_copy_substr(buf, 1, end); - current = rconf_section_add(conf, section); - mk_mem_free(section); - n_keys = 0; - continue; - } - else { - flb_error("[static conf: %s] Bad header definition", - fname); - } - } - - /* No separator defined */ - if (!indent) { - i = 0; - - do { i++; } while (i < len && isblank(buf[i])); - - indent = mk_string_copy_substr(buf, 0, i); - indent_len = strlen(indent); - - /* Blank indented line */ - if (i == len) { - continue; - } - } - - /* Validate indentation level */ - if (strncmp(buf, indent, indent_len) != 0 || - isblank(buf[indent_len]) != 0) { - //mk_config_error(path, line, "Invalid indentation level"); - } - - if (buf[indent_len] == '#' || indent_len == len) { - continue; - } - - /* Get key and val */ - i = mk_string_char_search(buf + indent_len, ' ', len - indent_len); - key = mk_string_copy_substr(buf + indent_len, 0, i); - val = mk_string_copy_substr(buf + indent_len + i, 1, len - indent_len - i); - - if (!key || !val || i < 0) { - //mk_config_error(path, line, "Each key must have a value"); - } - - /* Trim strings */ - mk_string_trim(&key); - mk_string_trim(&val); - - if (strlen(val) == 0) { - //mk_config_error(path, line, "Key has an empty value"); - } - - /* Register entry: key and val are copied as duplicated */ - rconf_section_entry_add(conf, key, val); - - /* Free temporary key and val */ - mk_mem_free(key); - mk_mem_free(val); - - n_keys++; - } - - if (section && n_keys == 0) { - /* No key, no warning */ - } - - /* - struct mk_config_section *s; - struct mk_rconf_entry *e; - - s = conf->section; - while(s) { - printf("\n[%s]", s->name); - e = s->entry; - while(e) { - printf("\n %s = %s", e->key, e->val); - e = e->next; - } - s = s->next; - } - fflush(stdout); - */ - if (indent) { - mk_mem_free(indent); - } - mk_mem_free(buf); - - /* Append this file to the list */ - file = mk_mem_alloc(sizeof(struct mk_rconf_file)); - if (!file) { - perror("malloc"); - conf->level--; - return -1; - } - - file->path = mk_string_dup(fname); - mk_list_add(&file->_head, &conf->includes); - conf->level--; - return 0; -} - /* - * If Fluent Bit have static configuration support, this function - * allows to lookup, parse and create configuration file contexts - * from the entries generated by CMake at build-time. + * If Fluent Bit has static configuration support, this function allows to lookup, + * parse and create configuration file contexts from the entries generated by + * CMake at build-time. * - * The routine is a modified version from mk_rconf_read() but with - * the differences that it does all operations on top of the global - * entries at flb_config_files[] defined at: + * This routing passes a 'virtual' file name that should be registered into the + * static array 'flb_config_files'. Learn more about it at: * * include/fluent-bit/conf/flb_static_conf.h * */ -struct mk_rconf *flb_config_static_open(const char *file) +struct flb_cf *flb_config_static_open(struct flb_config *config, const char *file) { int i; int ret; - const char *k; - const char *v; - struct mk_rconf *conf = NULL; + const char *k = NULL; + const char *v = NULL; + struct flb_cf *cf; /* Iterate static array and lookup the file name */ for (i = 0; i < flb_config_files_size; i++) { @@ -378,24 +56,15 @@ struct mk_rconf *flb_config_static_open(const char *file) return NULL; } - /* Alloc configuration node */ - conf = mk_mem_alloc(sizeof(struct mk_rconf)); - conf->created = time(NULL); - conf->file = mk_string_dup(file); - conf->level = -1; - mk_list_init(&conf->sections); - mk_list_init(&conf->includes); - mk_list_init(&conf->metas); - - /* Set the absolute path for the entrypoint file */ - conf->root_path = mk_string_dup("/"); - - /* Read entrypoint */ - ret = flb_config_static_read(conf, k, v); - if (ret == -1) { - mk_rconf_free(conf); + cf = flb_cf_fluentbit_create(NULL, (char *) file, (char *) v, 0); + if (!cf) { return NULL; } - return conf; + if (config->cf_main) { + flb_cf_destroy(config->cf_main); + } + config->cf_main = cf; + + return cf; } From c5741b48c24aa80ddd820e6f42e6ce8fc5f5d98b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 16:07:08 -0600 Subject: [PATCH 30/51] config_format: fluentit: add support for static configs Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 82 ++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 10 deletions(-) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index d91388697a4..bb132d9849a 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -70,7 +70,8 @@ struct local_ctx { struct mk_list sections; }; -static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file); +static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file, + char *buf, size_t size); /* Raise a configuration schema error */ static void config_error(const char *path, int line, const char *msg) @@ -102,6 +103,37 @@ static int char_search(const char *string, int c, int len) return -1; } +/* + * Helper function to simulate a fgets(2) but instead of using a real file stream + * uses the data buffer provided. + */ +static int static_fgets(char *out, size_t size, const char *data, size_t *off) +{ + size_t len; + const char *start = data + *off; + char *end; + + end = strchr(start, '\n'); + + if (!end || *off >= size) { + len = size - *off - 1; + memcpy(out, start, len); + out[len] = '\0'; + *off += len + 1; + return 0; + } + + len = end - start; + if (len >= size) { + len = size - 1; + } + memcpy(out, start, len); + out[len] = '\0'; + *off += len + 1; + + return 1; +} + #ifndef _WIN32 static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char * path) { @@ -140,7 +172,7 @@ static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char * path } for (i = 0; i < glb.gl_pathc; i++) { - ret = read_config(cf, ctx, glb.gl_pathv[i]); + ret = read_config(cf, ctx, glb.gl_pathv[i], NULL, 0); if (ret < 0) { break; } @@ -237,9 +269,11 @@ static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char *path) static int local_init(struct local_ctx *ctx, char *file) { - char *p; char *end; - char path[PATH_MAX + 1]; + char path[PATH_MAX + 1] = {0}; + +#ifndef FLB_HAVE_STATIC_CONF + char *p; if (file) { #ifdef _MSC_VER @@ -251,6 +285,7 @@ static int local_init(struct local_ctx *ctx, char *file) return -1; } } +#endif /* lookup path ending and truncate */ end = strrchr(path, '/'); @@ -354,7 +389,8 @@ static int check_indent(const char *line, const char *indent, int *out_level) return INDENT_OK; } -static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) +static int read_config(struct flb_cf *cf, struct local_ctx *ctx, + char *cfg_file, char *in_data, size_t in_size) { int i; int len; @@ -382,6 +418,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) FILE *f; /* Check if the path exists (relative cases for included files) */ +#ifndef FLB_HAVE_STATIC_CONF if (ctx->level >= 0) { ret = stat(cfg_file, &st); if (ret == -1 && errno == ENOENT) { @@ -396,6 +433,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) } } } +#endif /* Check this file have not been included before */ ret = is_file_included(ctx, cfg_file); @@ -405,11 +443,13 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) } ctx->level++; +#ifndef FLB_HAVE_STATIC_CONF /* Open configuration file */ if ((f = fopen(cfg_file, "r")) == NULL) { flb_warn("[config] I cannot open %s file", cfg_file); return -1; } +#endif /* Allocate temporal buffer to read file content */ buf = flb_malloc(FLB_CF_BUF_SIZE); @@ -418,8 +458,17 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) return -1; } - /* looking for configuration directives */ +#ifdef FLB_HAVE_STATIC_CONF + /* + * a static configuration comes from a buffer, so we use the static_fgets() + * workaround to retrieve the lines. + */ + size_t off = 0; + while (static_fgets(buf, FLB_CF_BUF_SIZE, in_data, &off)) { +#else + /* normal mode, read lines into a buffer */ while (fgets(buf, FLB_CF_BUF_SIZE, f)) { +#endif len = strlen(buf); if (len > 0 && buf[len - 1] == '\n') { buf[--len] = 0; @@ -427,6 +476,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) buf[--len] = 0; } } +#ifndef FLB_HAVE_STATIC_CONF else { /* * If we don't find a break line, validate if we got an EOF or not. No EOF @@ -439,6 +489,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) return -1; } } +#endif /* Line number */ line++; @@ -457,7 +508,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) ret = read_glob(cf, ctx, buf + 9); } else { - ret = read_config(cf, ctx, buf + 9); + ret = read_config(cf, ctx, buf + 9, NULL, 0); } if (ret == -1) { ctx->level--; @@ -632,7 +683,10 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file) /* No key, no warning */ } +#ifndef FLB_HAVE_STATIC_CONF fclose(f); +#endif + if (indent) { flb_sds_destroy(indent); } @@ -656,6 +710,7 @@ struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, char *file_path, char *buf, size_t size) { int ret; + int created = FLB_FALSE; struct local_ctx ctx; if (!cf) { @@ -663,15 +718,22 @@ struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, if (!cf) { return NULL; } + created = FLB_TRUE; } - local_init(&ctx, file_path); + ret = local_init(&ctx, file_path); + if (ret != 0) { + if (cf && created) { + flb_cf_destroy(cf); + } + return NULL; + } - ret = read_config(cf, &ctx, file_path); + ret = read_config(cf, &ctx, file_path, buf, size); local_exit(&ctx); - if (ret == -1) { + if (ret == -1 && created) { flb_cf_destroy(cf); } From 60a4ada8f7b87d7c614236885161024df7abbda7 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 16:07:36 -0600 Subject: [PATCH 31/51] bin: add support for static config Signed-off-by: Eduardo Silva --- src/fluent-bit.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/fluent-bit.c b/src/fluent-bit.c index 17f786a9111..acf4c1bb11d 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -771,13 +771,17 @@ static int service_configure(struct flb_cf *cf, struct flb_config *config, char struct mk_list *head; #ifdef FLB_HAVE_STATIC_CONF - /* DISABLED/FIXME fconf = flb_config_static_open(file); */ + cf = flb_config_static_open(file); #else if (file) { cf = flb_cf_create_from_file(cf, file); } #endif + if (!cf) { + return -1; + } + /* Set configuration root path */ if (file) { flb_service_conf_path_set(config, file); @@ -1125,7 +1129,7 @@ int flb_main(int argc, char **argv) flb_utils_error(FLB_ERR_CFG_FILE_STOP); } #else - ret = flb_service_conf(config, "fluent-bit.conf"); + ret = service_configure(NULL, config, "fluent-bit.conf"); if (ret != 0) { flb_utils_error(FLB_ERR_CFG_FILE_STOP); } From ad6da2a4b8d7df91d210a369b05e1a85f9263684 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 16:07:55 -0600 Subject: [PATCH 32/51] parser: add support for static config Signed-off-by: Eduardo Silva --- src/flb_parser.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/flb_parser.c b/src/flb_parser.c index d2571dca614..d8ddcd2a959 100644 --- a/src/flb_parser.c +++ b/src/flb_parser.c @@ -853,20 +853,20 @@ int flb_parser_conf_file(const char *file, struct flb_config *config) cf = flb_cf_create_from_file(config->cf_parsers, cfg); #else - //DISABLED/FIXME fconf = flb_config_static_open(file); + cf = flb_config_static_open(file); #endif if (!cf) { return -1; } - /* process [PARSER]'s sections */ + /* process 'parser' sections */ ret = parser_conf_file(cfg, cf, config); if (ret == -1) { return -1; } - /* processs [MULTILINE_PARSER]'s sections */ + /* processs 'multiline_parser' sections */ ret = multiline_parser_conf_file(cfg, cf, config); if (ret == -1) { return -1; From fb42fe82c9c8cf6687878d792974aa85240926a2 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 18:25:55 -0600 Subject: [PATCH 33/51] bin: do not overwrite context on static config Signed-off-by: Eduardo Silva --- src/fluent-bit.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/fluent-bit.c b/src/fluent-bit.c index acf4c1bb11d..c9e31512ea2 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -763,7 +763,8 @@ static int service_configure_plugin(struct flb_config *config, return 0; } -static int service_configure(struct flb_cf *cf, struct flb_config *config, char *file) +static struct flb_cf *service_configure(struct flb_cf *cf, + struct flb_config *config, char *file) { int ret = -1; struct flb_cf_section *s; @@ -779,9 +780,11 @@ static int service_configure(struct flb_cf *cf, struct flb_config *config, char #endif if (!cf) { - return -1; + return NULL; } + config->cf_main = cf; + /* Set configuration root path */ if (file) { flb_service_conf_path_set(config, file); @@ -851,10 +854,10 @@ static int service_configure(struct flb_cf *cf, struct flb_config *config, char goto error; } - return 0; + return cf; error: - return -1; + return NULL; } int flb_main(int argc, char **argv) @@ -870,6 +873,7 @@ int flb_main(int argc, char **argv) /* config format context */ struct flb_cf *cf; + struct flb_cf *tmp; struct flb_cf_section *service; struct flb_cf_section *s; @@ -1114,6 +1118,7 @@ int flb_main(int argc, char **argv) /* Validate config file */ #ifndef FLB_HAVE_STATIC_CONF + if (cfg_file) { if (access(cfg_file, R_OK) != 0) { flb_free(cfg_file); @@ -1122,17 +1127,21 @@ int flb_main(int argc, char **argv) } /* Load the service configuration file */ - ret = service_configure(cf, config, cfg_file); + tmp = service_configure(cf, config, cfg_file); flb_free(cfg_file); - - if (ret != 0) { + if (!tmp) { flb_utils_error(FLB_ERR_CFG_FILE_STOP); } #else - ret = service_configure(NULL, config, "fluent-bit.conf"); - if (ret != 0) { + tmp = service_configure(cf, config, "fluent-bit.conf"); + if (!tmp) { flb_utils_error(FLB_ERR_CFG_FILE_STOP); } + + /* destroy previous context and override */ + flb_cf_destroy(cf); + config->cf_main = tmp; + cf = tmp; #endif From 715e40c3104db7576866041e47bf62ccfbf8b518 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 18:26:23 -0600 Subject: [PATCH 34/51] config_format: improve handling of service section Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 8 +++- src/config_format/flb_config_format.c | 54 +++++++++++++++------------ 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index bb132d9849a..46b3763c107 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -512,11 +512,13 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, } if (ret == -1) { ctx->level--; - fclose(f); if (indent) { flb_sds_destroy(indent); } flb_free(buf); +#ifndef FLB_HAVE_STATIC_CONF + fclose(f); +#endif return -1; } continue; @@ -524,11 +526,13 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, else if (buf[0] == '@' && len > 3) { meta = flb_cf_meta_create(cf, buf, len); if (!meta) { - fclose(f); if (indent) { flb_sds_destroy(indent); } flb_free(buf); +#ifndef FLB_HAVE_STATIC_CONF + fclose(f); +#endif return -1; } continue; diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index e958b6f1955..f3969056bf8 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -70,27 +70,31 @@ void flb_cf_destroy(struct flb_cf *cf) flb_free(cf); } -static enum section_type get_section_type(char *name) +static enum section_type get_section_type(char *name, int len) { - if (strcasecmp(name, "SERVICE") == 0) { + if (strncasecmp(name, "SERVICE", len) == 0) { return FLB_CF_SERVICE; } - else if (strcasecmp(name, "PARSER") == 0) { + else if (strncasecmp(name, "PARSER", len) == 0) { return FLB_CF_PARSER; } - else if (strcasecmp(name, "MULTILINE_PARSER") == 0) { + else if (strncasecmp(name, "MULTILINE_PARSER", len) == 0) { return FLB_CF_MULTILINE_PARSER; } - else if (strcasecmp(name, "CUSTOM") == 0 || strcasecmp(name, "CUSTOMS") == 0) { + else if (strncasecmp(name, "CUSTOM", len) == 0 || + strncasecmp(name, "CUSTOMS", len) == 0) { return FLB_CF_CUSTOM; } - else if (strcasecmp(name, "INPUT") == 0 || strcasecmp(name, "INPUTS") == 0) { + else if (strncasecmp(name, "INPUT", len) == 0 || + strncasecmp(name, "INPUTS", len) == 0) { return FLB_CF_INPUT; } - else if (strcasecmp(name, "FILTER") == 0 || strcasecmp(name, "FILTERS") == 0) { + else if (strncasecmp(name, "FILTER", len) == 0 || + strncasecmp(name, "FILTERS", len) == 0) { return FLB_CF_FILTER; } - else if (strcasecmp(name, "OUTPUT") == 0 || strcasecmp(name, "OUTPUT") == 0) { + else if (strncasecmp(name, "OUTPUT", len) == 0 || + strncasecmp(name, "OUTPUTS", len) == 0) { return FLB_CF_OUTPUT; } @@ -229,6 +233,23 @@ struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int int type; struct flb_cf_section *s; + if (!name) { + return NULL; + } + + /* determinate type by name */ + if (len <= 0) { + len = strlen(name); + } + + /* get the section type */ + type = get_section_type(name, len); + + /* check if 'service' already exists */ + if (type == FLB_CF_SERVICE && cf->service) { + return cf->service; + } + /* section context */ s = flb_malloc(sizeof(struct flb_cf_section)); if (!s) { @@ -240,29 +261,14 @@ struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int flb_kv_init(&s->properties); mk_list_init(&s->groups); - /* determinate type by name */ - if (len <= 0) { - len = strlen(name); - } - /* create a NULL terminated name */ s->name = flb_sds_create_len(name, len); if (!s->name) { flb_free(s); return NULL; } - - /* get the section type */ - type = get_section_type(s->name); s->type = type; - /* restrict to only one SERVICE section */ - if (type == FLB_CF_SERVICE && cf->service) { - flb_sds_destroy(s->name); - flb_free(s); - return cf->service; - } - if (type == FLB_CF_SERVICE && !cf->service) { cf->service = s; } @@ -320,6 +326,7 @@ void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s) if (s->name) { flb_sds_destroy(s->name); + s->name = NULL; } flb_kv_release(&s->properties); @@ -335,6 +342,7 @@ void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s) if (s->type != FLB_CF_SERVICE) { mk_list_del(&s->_head_section); } + flb_free(s); } From 919613d14302e867448588718ce072565d95ef16 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 18:26:50 -0600 Subject: [PATCH 35/51] config_static: change function handler prototype Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_config.h | 2 +- src/flb_config_static.c | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index f6353beeab4..6f1531d6ac8 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -259,7 +259,7 @@ int flb_config_set_program_name(struct flb_config *config, char *name); int set_log_level_from_env(struct flb_config *config); #ifdef FLB_HAVE_STATIC_CONF -struct flb_cf *flb_config_static_open(struct flb_config *config, const char *file); +struct flb_cf *flb_config_static_open(const char *file); #endif struct flb_service_config { diff --git a/src/flb_config_static.c b/src/flb_config_static.c index 066f30fc84f..700b3ee9e67 100644 --- a/src/flb_config_static.c +++ b/src/flb_config_static.c @@ -33,10 +33,9 @@ * include/fluent-bit/conf/flb_static_conf.h * */ -struct flb_cf *flb_config_static_open(struct flb_config *config, const char *file) +struct flb_cf *flb_config_static_open(const char *file) { int i; - int ret; const char *k = NULL; const char *v = NULL; struct flb_cf *cf; @@ -61,10 +60,5 @@ struct flb_cf *flb_config_static_open(struct flb_config *config, const char *fil return NULL; } - if (config->cf_main) { - flb_cf_destroy(config->cf_main); - } - config->cf_main = cf; - return cf; } From ebc1b344c2859a5ba9196ee5329f5baf69460432 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 18:27:15 -0600 Subject: [PATCH 36/51] config: change config format reference order Signed-off-by: Eduardo Silva --- src/flb_config.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/flb_config.c b/src/flb_config.c index 96af2042b75..20d63f8d3ff 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -191,12 +191,13 @@ struct flb_config *flb_config_init() if (!cf) { return NULL; } + config->cf_main = cf; + section = flb_cf_section_create(cf, "service", 0); if (!section) { flb_cf_destroy(cf); return NULL; } - config->cf_main = cf; /* config_format for parsers */ config->cf_parsers = flb_cf_create(); @@ -384,7 +385,8 @@ void flb_config_exit(struct flb_config *config) mk_event_timeout_destroy(config->evl, &collector->event); mk_event_closesocket(collector->fd_timer); } - } else { + } + else { mk_event_del(config->evl, &collector->event); } From cf8b778916ce8044112cd4dc21aa8eed130a42c1 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 18:27:39 -0600 Subject: [PATCH 37/51] parser: fix override of config_format context on static mode Signed-off-by: Eduardo Silva --- src/flb_parser.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/flb_parser.c b/src/flb_parser.c index d8ddcd2a959..e4ea16d300f 100644 --- a/src/flb_parser.c +++ b/src/flb_parser.c @@ -854,6 +854,12 @@ int flb_parser_conf_file(const char *file, struct flb_config *config) cf = flb_cf_create_from_file(config->cf_parsers, cfg); #else cf = flb_config_static_open(file); + if (cf) { + if (config->cf_parsers) { + flb_cf_destroy(config->cf_parsers); + } + config->cf_parsers = cf; + } #endif if (!cf) { From cfa188fd54dbbbb093aa14342035788a59b60ea2 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 18:28:05 -0600 Subject: [PATCH 38/51] plugin: add support for config format Signed-off-by: Eduardo Silva --- src/flb_plugin.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/flb_plugin.c b/src/flb_plugin.c index 015bdd47fc9..47e83b10f13 100644 --- a/src/flb_plugin.c +++ b/src/flb_plugin.c @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include #include #include @@ -327,12 +329,12 @@ int flb_plugin_load_config_file(const char *file, struct flb_config *config) int ret; char tmp[PATH_MAX + 1]; const char *cfg = NULL; - struct mk_rconf *fconf; - struct mk_rconf_section *section; - struct mk_rconf_entry *entry; struct mk_list *head; struct mk_list *head_e; struct stat st; + struct flb_cf *cf; + struct flb_cf_section *section; + struct flb_kv *entry; #ifndef FLB_HAVE_STATIC_CONF ret = stat(file, &st); @@ -353,38 +355,39 @@ int flb_plugin_load_config_file(const char *file, struct flb_config *config) } flb_debug("[plugin] opening configuration file %s", cfg); - fconf = mk_rconf_open(cfg); + + cf = flb_cf_create_from_file(NULL, cfg); #else - fconf = flb_config_static_open(file); + cf = flb_config_static_open(file); #endif - if (!fconf) { + if (!cf) { return -1; } - /* Read all [PLUGINS] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "PLUGINS") != 0) { + /* read all 'plugins' sections */ + mk_list_foreach(head, &cf->sections) { + section = mk_list_entry(head, struct flb_cf_section, _head); + if (strcasecmp(section->name, "plugins") != 0) { continue; } - mk_list_foreach(head_e, §ion->entries) { - entry = mk_list_entry(head_e, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "Path") != 0) { + mk_list_foreach(head_e, §ion->properties) { + entry = mk_list_entry(head_e, struct flb_kv, _head); + if (strcasecmp(entry->key, "path") != 0) { continue; } /* Load plugin with router function */ ret = flb_plugin_load_router(entry->val, config); if (ret == -1) { - mk_rconf_free(fconf); + flb_cf_destroy(cf); return -1; } } } - mk_rconf_free(fconf); + flb_cf_destroy(cf); return 0; } From 2d732e62401d5a75ccc2e17b343b5ad38a654ecc Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 18:28:35 -0600 Subject: [PATCH 39/51] sp: add support for config format Signed-off-by: Eduardo Silva --- src/stream_processor/flb_sp.c | 36 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/stream_processor/flb_sp.c b/src/stream_processor/flb_sp.c index 18ee3cb8a40..6cd82efdabf 100644 --- a/src/stream_processor/flb_sp.c +++ b/src/stream_processor/flb_sp.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -64,10 +65,10 @@ static int sp_config_file(struct flb_config *config, struct flb_sp *sp, const char *cfg = NULL; char tmp[PATH_MAX + 1]; struct stat st; - struct mk_rconf *fconf; - struct mk_rconf_section *section; struct mk_list *head; struct flb_sp_task *task; + struct flb_cf *cf; + struct flb_cf_section *section; #ifndef FLB_HAVE_STATIC_CONF ret = stat(file, &st); @@ -87,34 +88,34 @@ static int sp_config_file(struct flb_config *config, struct flb_sp *sp, cfg = file; } - fconf = mk_rconf_open(cfg); + cf = flb_cf_create_from_file(NULL, cfg); #else - fconf = flb_config_static_open(file); + cf = flb_config_static_open(file); #endif - if (!fconf) { + if (!cf) { return -1; } - /* Read all [STREAM_TASK] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "STREAM_TASK") != 0) { + /* Read all 'stream_task' sections */ + mk_list_foreach(head, &cf->sections) { + section = mk_list_entry(head, struct flb_cf_section, _head); + if (strcasecmp(section->name, "stream_task") != 0) { continue; } name = NULL; exec = NULL; - /* Name */ - name = mk_rconf_section_get_key(section, "Name", MK_RCONF_STR); + /* name */ + name = flb_cf_section_property_get(cf, section, "name"); if (!name) { flb_error("[sp] task 'name' not found in file '%s'", cfg); goto fconf_error; } - /* Exec */ - exec = mk_rconf_section_get_key(section, "Exec", MK_RCONF_STR); + /* exec */ + exec = flb_cf_section_property_get(cf, section, "exec"); if (!exec) { flb_error("[sp] task '%s' don't have an 'exec' command", name); goto fconf_error; @@ -125,18 +126,13 @@ static int sp_config_file(struct flb_config *config, struct flb_sp *sp, if (!task) { goto fconf_error; } - - flb_free(name); - flb_free(exec); } - mk_rconf_free(fconf); + flb_cf_destroy(cf); return 0; fconf_error: - flb_free(name); - flb_free(exec); - + flb_cf_destroy(cf); return -1; } From 1b401150cb0f7dcdc8fda2ff7fdcdbeef89350b0 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 19:14:57 -0600 Subject: [PATCH 40/51] config_format: add support for environment variables Signed-off-by: Eduardo Silva --- include/fluent-bit/config_format/flb_cf.h | 5 +++- .../fluent-bit/config_format/flb_cf_yaml.h | 5 ++-- src/config_format/flb_cf_yaml.c | 25 +++++++++++++++++-- src/config_format/flb_config_format.c | 4 +++ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h index 445495d1281..0d164fc72b4 100644 --- a/include/fluent-bit/config_format/flb_cf.h +++ b/include/fluent-bit/config_format/flb_cf.h @@ -77,7 +77,10 @@ struct flb_cf { /* global service */ struct flb_cf_section *service; - /* meta commands */ + /* config environment variables (env section, availble on YAML) */ + struct mk_list env; + + /* meta commands (used by fluentbit classic mode) */ struct mk_list metas; /* parsers */ diff --git a/include/fluent-bit/config_format/flb_cf_yaml.h b/include/fluent-bit/config_format/flb_cf_yaml.h index 43b865092dc..a8cc4f377c0 100644 --- a/include/fluent-bit/config_format/flb_cf_yaml.h +++ b/include/fluent-bit/config_format/flb_cf_yaml.h @@ -21,7 +21,6 @@ #ifndef FLB_CONFIG_FORMAT_YAML_H #define FLB_CONFIG_FORMAT_YAML_H -struct flb_cf *flb_cf_yaml_create(struct flb_cf *cf, - char *file_path, char *buf, size_t size); - +struct flb_cf *flb_cf_yaml_create(struct flb_cf *cf, char *file_path, + char *buf, size_t size); #endif diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c index 3f950dcf6f0..2a9c87c5890 100644 --- a/src/config_format/flb_cf_yaml.c +++ b/src/config_format/flb_cf_yaml.c @@ -29,6 +29,7 @@ #include enum section { + SECTION_ENV, SECTION_SERVICE, SECTION_CUSTOM, SECTION_INPUT, @@ -62,6 +63,10 @@ enum state { STATE_GROUP_KEY, STATE_GROUP_VAL, + /* environment variables */ + STATE_ENV, + + STATE_STOP /* end state */ }; @@ -86,6 +91,7 @@ struct parser_state { struct flb_cf_group *cf_group; /* internal variables for checks */ + int env_set; int service_set; int customs_set; int inputs_set; @@ -144,6 +150,7 @@ static int consume_event(struct flb_cf *cf, struct parser_state *s, int len; int ret; char *value; + struct mk_list *list; struct flb_kv *kv; switch (s->state) { @@ -192,7 +199,12 @@ static int consume_event(struct flb_cf *cf, struct parser_state *s, switch (event->type) { case YAML_SCALAR_EVENT: value = (char *)event->data.scalar.value; - if (strcasecmp(value, "service") == 0) { + if (strcasecmp(value, "env") == 0) { + s->state = STATE_ENV; + s->section = SECTION_ENV; + s->env_set = 1; + } + else if (strcasecmp(value, "service") == 0) { if (s->service_set) { yaml_error_definition(s, event, value); return YAML_FAILURE; @@ -263,6 +275,7 @@ static int consume_event(struct flb_cf *cf, struct parser_state *s, break; /* service or others */ + case STATE_ENV: case STATE_SERVICE: case STATE_OTHER: switch(event->type) { @@ -301,8 +314,16 @@ static int consume_event(struct flb_cf *cf, struct parser_state *s, value = (char *) event->data.scalar.value; s->val = flb_sds_create(value); + /* Check if the incoming k/v pair set a config environment variable */ + if (s->section == SECTION_ENV) { + list = &cf->env; + } + else { + list = &s->cf_section->properties; + } + /* register key/value pair as a property */ - kv = flb_cf_property_add(cf, &s->cf_section->properties, + kv = flb_cf_property_add(cf, list /*&s->cf_section->properties*/, s->key, flb_sds_len(s->key), s->val, flb_sds_len(s->val)); if (!kv) { diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index f3969056bf8..7b910542477 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -39,6 +39,9 @@ struct flb_cf *flb_cf_create() return NULL; } + /* env vars */ + mk_list_init(&ctx->env); + /* meta commands */ mk_list_init(&ctx->metas); @@ -65,6 +68,7 @@ struct flb_cf *flb_cf_create() void flb_cf_destroy(struct flb_cf *cf) { + flb_kv_release(&cf->env); flb_kv_release(&cf->metas); flb_cf_section_destroy_all(cf); flb_free(cf); From 58f0ea19f803963716f2248f638d2d8a94333356 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 19:15:37 -0600 Subject: [PATCH 41/51] bin: register config format env variables Signed-off-by: Eduardo Silva --- src/fluent-bit.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/fluent-bit.c b/src/fluent-bit.c index c9e31512ea2..088ab61b614 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -790,6 +790,17 @@ static struct flb_cf *service_configure(struct flb_cf *cf, flb_service_conf_path_set(config, file); } + /* Process config environment vars */ + mk_list_foreach(head, &cf->env) { + kv = mk_list_entry(head, struct flb_kv, _head); + ret = flb_env_set(config->env, kv->key, kv->val); + if (ret == -1) { + fprintf(stderr, "could not set config environment variable '%s'\n", + kv->key); + exit(EXIT_FAILURE); + } + } + /* Process all meta commands */ mk_list_foreach(head, &cf->metas) { kv = mk_list_entry(head, struct flb_kv, _head); @@ -800,7 +811,8 @@ static struct flb_cf *service_configure(struct flb_cf *cf, mk_list_foreach(head, &cf->sections) { s = mk_list_entry(head, struct flb_cf_section, _head); - if (strcasecmp(s->name, "service") == 0 || + if (strcasecmp(s->name, "env") == 0 || + strcasecmp(s->name, "service") == 0 || strcasecmp(s->name, "custom") == 0 || strcasecmp(s->name, "input") == 0 || strcasecmp(s->name, "filter") == 0 || @@ -817,13 +829,8 @@ static struct flb_cf *service_configure(struct flb_cf *cf, "Sections 'multiline_parser' and 'parser' are not valid in " "the main configuration file. It belongs to \n" "the 'parsers_file' configuration files.\n"); + exit(EXIT_FAILURE); } - else { - fprintf(stderr, - "Error: unexpected section '%s' in the main " - "configuration file.\n", s->name); - } - exit(EXIT_FAILURE); } /* Read main 'service' section */ From 5e6f30cfad5bde3fdbb8ad746121ef7bfbf27926 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 19:16:09 -0600 Subject: [PATCH 42/51] tests: internal: data: config_format: add env var Signed-off-by: Eduardo Silva --- tests/internal/data/config_format/fluent-bit.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/internal/data/config_format/fluent-bit.yaml b/tests/internal/data/config_format/fluent-bit.yaml index 8ce2eb17560..3f61abffc46 100644 --- a/tests/internal/data/config_format/fluent-bit.yaml +++ b/tests/internal/data/config_format/fluent-bit.yaml @@ -1,5 +1,9 @@ +# environment variable definition +env: + flush_interval: 1 + service: - flush: 1 + flush: ${flush_interval} log_level: info http_server: on From cec2e42abcce5a80fd4823101c9611eed3edcb72 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 21:06:07 -0600 Subject: [PATCH 43/51] config_format: yaml: add new 'pipeline' level for plugins the following patch makes possible to separate 'visually' only the components of a pipeline. This is useful when having uses that certain inputs must go through certain filters and outputs. This change do not touch routing, it's just a visual separation. --- start of fluent-bit.yaml --- env: flush_interval: 1 service: flush: ${flush_interval} log_level: info http_server: on pipeline: inputs: dummy: samples: 100 tag: example outputs: stdout: match: example pipeline: inputs: tail: path: /var/log/containers/coredns*.log read_from_head: true tag: kube.* filters: kubernetes: match: kube.* outputs: stdout: match: kube.* --- end of file --- Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_yaml.c | 118 +++++++++++++++++--------------- 1 file changed, 63 insertions(+), 55 deletions(-) diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c index 2a9c87c5890..a4d790a8433 100644 --- a/src/config_format/flb_cf_yaml.c +++ b/src/config_format/flb_cf_yaml.c @@ -31,6 +31,7 @@ enum section { SECTION_ENV, SECTION_SERVICE, + SECTION_PIPELINE, SECTION_CUSTOM, SECTION_INPUT, SECTION_FILTER, @@ -39,21 +40,23 @@ enum section { }; enum state { - STATE_START, /* start state */ - STATE_STREAM, /* start/end stream */ - STATE_DOCUMENT, /* start/end document */ + STATE_START, /* start state */ + STATE_STREAM, /* start/end stream */ + STATE_DOCUMENT, /* start/end document */ - STATE_SECTION, /* top level */ + STATE_SECTION, /* top level */ STATE_SECTION_KEY, STATE_SECTION_VAL, - STATE_SERVICE, /* service section */ - STATE_OTHER, /* service section */ + STATE_SERVICE, /* service section */ + STATE_OTHER, /* service section */ - STATE_PLUGIN_CUSTOM, /* custom plugins section */ - STATE_PLUGIN_INPUT, /* input plugins section */ - STATE_PLUGIN_FILTER, /* filter plugins section */ - STATE_PLUGIN_OUTPUT, /* output plugins section */ + STATE_PIPELINE, /* pipeline groups customs inputs, filters and outputs */ + + STATE_PLUGIN_CUSTOM, /* custom plugins section */ + STATE_PLUGIN_INPUT, /* input plugins section */ + STATE_PLUGIN_FILTER, /* filter plugins section */ + STATE_PLUGIN_OUTPUT, /* output plugins section */ STATE_PLUGIN_TYPE, STATE_PLUGIN_KEY_VALUE_PAIR, @@ -90,13 +93,7 @@ struct parser_state { /* active group */ struct flb_cf_group *cf_group; - /* internal variables for checks */ - int env_set; int service_set; - int customs_set; - int inputs_set; - int filters_set; - int outputs_set; }; /* yaml_* functions return 1 on success and 0 on failure. */ @@ -144,6 +141,16 @@ static void yaml_error_definition(struct parser_state *s, yaml_event_t *event, value); } +static void yaml_error_plugin_category(struct parser_state *s, yaml_event_t *event, + char *value) +{ + flb_error("[config] YAML error found in file \"%s\", line %i, column %i: " + "the pipeline component '%s' is not valid. Try one of these values: " + "customs, inputs, filters or outputs.", + s->file, event->start_mark.line + 1, event->start_mark.column, + value); +} + static int consume_event(struct flb_cf *cf, struct parser_state *s, yaml_event_t *event) { @@ -193,16 +200,53 @@ static int consume_event(struct flb_cf *cf, struct parser_state *s, } break; + case STATE_PIPELINE: + switch (event->type) { + case YAML_SCALAR_EVENT: + value = (char *)event->data.scalar.value; + if (strcasecmp(value, "customs") == 0) { + s->state = STATE_PLUGIN_CUSTOM; + s->section = SECTION_CUSTOM; + } + else if (strcasecmp(value, "inputs") == 0) { + s->state = STATE_PLUGIN_INPUT; + s->section = SECTION_INPUT; + } + else if (strcasecmp(value, "filters") == 0) { + s->state = STATE_PLUGIN_FILTER; + s->section = SECTION_FILTER; + } + else if (strcasecmp(value, "outputs") == 0) { + s->state = STATE_PLUGIN_OUTPUT; + s->section = SECTION_OUTPUT; + } + else { + yaml_error_plugin_category(s, event, value); + return YAML_FAILURE; + } + break; + case YAML_MAPPING_START_EVENT: + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_SECTION; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; case STATE_SECTION: s->section = SECTION_OTHER; - switch (event->type) { case YAML_SCALAR_EVENT: value = (char *)event->data.scalar.value; if (strcasecmp(value, "env") == 0) { s->state = STATE_ENV; s->section = SECTION_ENV; - s->env_set = 1; + } + else if (strcasecmp(value, "pipeline") == 0) { + s->state = STATE_PIPELINE; + s->section = SECTION_PIPELINE; } else if (strcasecmp(value, "service") == 0) { if (s->service_set) { @@ -217,42 +261,6 @@ static int consume_event(struct flb_cf *cf, struct parser_state *s, return YAML_FAILURE; } } - else if (strcasecmp(value, "customs") == 0) { - if (s->customs_set) { - yaml_error_definition(s, event, value); - return YAML_FAILURE; - } - s->state = STATE_PLUGIN_CUSTOM; - s->section = SECTION_CUSTOM; - s->customs_set = 1; - } - else if (strcasecmp(value, "inputs") == 0) { - if (s->inputs_set) { - yaml_error_definition(s, event, value); - return YAML_FAILURE; - } - s->state = STATE_PLUGIN_INPUT; - s->section = SECTION_INPUT; - s->inputs_set = 1; - } - else if (strcasecmp(value, "filters") == 0) { - if (s->filters_set) { - yaml_error_definition(s, event, value); - return YAML_FAILURE; - } - s->state = STATE_PLUGIN_FILTER; - s->section = SECTION_FILTER; - s->filters_set = 1; - } - else if (strcasecmp(value, "outputs") == 0) { - if (s->outputs_set) { - yaml_error_definition(s, event, value); - return YAML_FAILURE; - } - s->state = STATE_PLUGIN_OUTPUT; - s->section = SECTION_OUTPUT; - s->outputs_set = 1; - } else { /* any other main section definition (e.g: similar to STATE_SERVICE) */ s->state = STATE_OTHER; @@ -384,7 +392,7 @@ static int consume_event(struct flb_cf *cf, struct parser_state *s, s->state = STATE_PLUGIN_KEY_VALUE_PAIR; break; case YAML_MAPPING_END_EVENT: - s->state = STATE_SECTION; + s->state = STATE_PIPELINE; break; default: yaml_error_event(s, event); From aa7c18f1b6cd8a6f480ecc4995929e7900d72879 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 21:56:40 -0600 Subject: [PATCH 44/51] config_format: fluentbit: fix cleanup Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 63 +++++++++++++--------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index 46b3763c107..77dfdfa84f8 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -415,7 +415,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, struct flb_cf_group *current_group = NULL; struct flb_kv *kv; - FILE *f; + FILE *f = NULL; /* Check if the path exists (relative cases for included files) */ #ifndef FLB_HAVE_STATIC_CONF @@ -455,7 +455,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, buf = flb_malloc(FLB_CF_BUF_SIZE); if (!buf) { flb_errno(); - return -1; + goto error; } #ifdef FLB_HAVE_STATIC_CONF @@ -485,8 +485,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, */ if (!feof(f)) { config_error(cfg_file, line, "length of content has exceeded limit"); - flb_free(buf); - return -1; + goto error; } } #endif @@ -515,25 +514,14 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, if (indent) { flb_sds_destroy(indent); } - flb_free(buf); -#ifndef FLB_HAVE_STATIC_CONF - fclose(f); -#endif - return -1; + goto error; } continue; } else if (buf[0] == '@' && len > 3) { meta = flb_cf_meta_create(cf, buf, len); if (!meta) { - if (indent) { - flb_sds_destroy(indent); - } - flb_free(buf); -#ifndef FLB_HAVE_STATIC_CONF - fclose(f); -#endif - return -1; + goto error; } continue; } @@ -564,8 +552,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, } else { config_error(cfg_file, line, "bad header definition"); - flb_free(buf); - return -1; + goto error; } } @@ -588,7 +575,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, ret = check_indent(buf, indent, &level); if (ret == INDENT_ERROR) { config_error(cfg_file, line, "invalid indentation level"); - return -1; + goto error; } else { if (ret == INDENT_OK && current_group) { @@ -610,8 +597,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, if (!key || i < 0) { config_error(cfg_file, line, "undefined key"); - flb_free(buf); - return -1; + goto error; } /* Check possible start of a group */ @@ -619,22 +605,19 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, end = char_search(key, ']', len - indent_len); if (end == -1) { config_error(cfg_file, line, "expected a valid group name: [..]"); - flb_free(buf); - return -1; + goto error; } if (!current_section) { config_warn(cfg_file, line, "current group don't have a parent section"); - flb_free(buf); - return -1; + goto error; } /* check if a previous group exists with one key */ if (current_group && n_keys == 0) { config_warn(cfg_file, line, "previous group did not have keys"); - flb_free(buf); - return -1; + goto error; } /* Create new group */ @@ -655,12 +638,12 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, if (!key || !val || i < 0) { config_error(cfg_file, line, "each key must have a value"); - return -1; + goto error; } if (val_len == 0) { config_error(cfg_file, line, "key has an empty value"); - return -1; + goto error; } /* register entry: key and val are copied as duplicated */ @@ -676,7 +659,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, } if (!kv) { config_error(cfg_file, line, "could not allocate key value pair"); - return -1; + goto error; } /* Free temporary key and val */ @@ -687,9 +670,9 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, /* No key, no warning */ } -#ifndef FLB_HAVE_STATIC_CONF - fclose(f); -#endif + if (f) { + fclose(f); + } if (indent) { flb_sds_destroy(indent); @@ -701,13 +684,23 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, if (!file) { flb_errno(); ctx->level--; - return -1; + goto error; } file->path = flb_sds_create(cfg_file); mk_list_add(&file->_head, &ctx->includes); ctx->level--; return 0; + +error: + if (f) { + fclose(f); + } + if (indent) { + flb_sds_destroy(indent); + } + flb_free(buf); + return -1; } struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, From e3a946db621f88048877119c2c63eda8f7cff7db Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 21:57:13 -0600 Subject: [PATCH 45/51] tests: internal: config format: yaml: extend test case Signed-off-by: Eduardo Silva --- tests/internal/config_format_yaml.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/internal/config_format_yaml.c b/tests/internal/config_format_yaml.c index dc1e63362de..77efd4904f9 100644 --- a/tests/internal/config_format_yaml.c +++ b/tests/internal/config_format_yaml.c @@ -24,11 +24,10 @@ void test_basic() } /* Total number of sections */ - TEST_CHECK(mk_list_size(&cf->sections) == 9); + TEST_CHECK(mk_list_size(&cf->sections) == 10); /* SERVICE check */ TEST_CHECK(cf->service != NULL); - if (cf->service) { TEST_CHECK(mk_list_size(&cf->service->properties) == 3); } @@ -37,7 +36,7 @@ void test_basic() TEST_CHECK(mk_list_size(&cf->parsers) == 0); TEST_CHECK(mk_list_size(&cf->multiline_parsers) == 0); TEST_CHECK(mk_list_size(&cf->customs) == 1); - TEST_CHECK(mk_list_size(&cf->inputs) == 3); + TEST_CHECK(mk_list_size(&cf->inputs) == 4); TEST_CHECK(mk_list_size(&cf->filters) == 1); TEST_CHECK(mk_list_size(&cf->outputs) == 2); TEST_CHECK(mk_list_size(&cf->others) == 1); @@ -45,7 +44,7 @@ void test_basic() /* groups */ s = flb_cf_section_get_by_name(cf, "input"); TEST_CHECK(s != NULL); - TEST_CHECK(mk_list_size(&s->groups) == 2); + TEST_CHECK(mk_list_size(&s->groups) == 1); mk_list_foreach(head, &s->groups) { g = mk_list_entry(head, struct flb_cf_group, _head); From e189bfbdbb94b8390145871858eb0f00ea060063 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 21:57:22 -0600 Subject: [PATCH 46/51] tests: internal: config format: data: extend test case Signed-off-by: Eduardo Silva --- .../data/config_format/fluent-bit.yaml | 86 +++++++++++-------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/tests/internal/data/config_format/fluent-bit.yaml b/tests/internal/data/config_format/fluent-bit.yaml index 3f61abffc46..61e5a94c6f3 100644 --- a/tests/internal/data/config_format/fluent-bit.yaml +++ b/tests/internal/data/config_format/fluent-bit.yaml @@ -7,44 +7,54 @@ service: log_level: info http_server: on -customs: - calyptia: - api_key: abcdefg - -inputs: - tail: - path: "/var/log/containers/*.log" - parser: docker - group1: - key1: aa - key2: bb - group2: - key3: cc - key4: dd - - tail: - path: "/var/log/messages" - parser: syslog - - forward: - listen: 0.0.0.0 - -filters: - grep: - exclude: $kubernetes['labels']['app'] myapp - -outputs: - stdout: - match: "*" - retry_limit: false - - http: - match: "*" - host: 192.168.3.2 - port: 9444 - tls: on - tls.verify: off - retry_limit: false +pipeline: + inputs: + dummy: + samples: 1 + tag: dummy_tag + group1: + key1: aa + key2: bb + +pipeline: + customs: + calyptia: + api_key: abcdefg + + inputs: + tail: + path: "/var/log/containers/*.log" + parser: docker + group1: + key1: aa + key2: bb + group2: + key3: cc + key4: dd + + tail: + path: "/var/log/messages" + parser: syslog + + forward: + listen: 0.0.0.0 + + filters: + grep: + exclude: $kubernetes['labels']['app'] myapp + + outputs: + stdout: + match: "*" + retry_limit: false + + http: + match: "*" + host: 192.168.3.2 + port: 9444 + tls: on + tls.verify: off + retry_limit: false unknown: a: b From 3a8978e3a499e46e02b822a0701a30c02b1de8fe Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 21:57:38 -0600 Subject: [PATCH 47/51] tests: internal: fuzzers: add missing header Signed-off-by: Eduardo Silva --- tests/internal/fuzzers/config_fuzzer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/internal/fuzzers/config_fuzzer.c b/tests/internal/fuzzers/config_fuzzer.c index 300fe8581c8..4288e80b778 100644 --- a/tests/internal/fuzzers/config_fuzzer.c +++ b/tests/internal/fuzzers/config_fuzzer.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "flb_fuzz_header.h" /* A sample of configurations */ From fba8437d04d452502d3747cdd85342b179978d6f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 22:18:14 -0600 Subject: [PATCH 48/51] config_format: fluentbit: fix missing parameters for windows Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index 77dfdfa84f8..77ab8acb6ec 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -256,7 +256,7 @@ static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char *path) ret = stat(buf, &st); if (ret == 0 && (st.st_mode & S_IFMT) == S_IFREG) { - if (read_config(cf, ctx, buf) < 0) { + if (read_config(cf, ctx, buf, NULL, 0) < 0) { return -1; } } From ecefc906696c61ed9d0c149286cf00a80a92e201 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 22:36:57 -0600 Subject: [PATCH 49/51] config_format: fluentbit: validate section before adding props Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index 77ab8acb6ec..fd4644333ba 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -647,12 +647,13 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, } /* register entry: key and val are copied as duplicated */ + kv = NULL; if (current_group) { kv = flb_cf_property_add(cf, ¤t_group->properties, key, key_len, val, val_len); } - else { + else if (current_section) { kv = flb_cf_property_add(cf, ¤t_section->properties, key, key_len, val, val_len); From 8697f17d3998d7c49f1041d520f207861c392e22 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 26 Jan 2022 23:32:23 -0600 Subject: [PATCH 50/51] config_format: fluentbit: invalidate indent Signed-off-by: Eduardo Silva --- src/config_format/flb_cf_fluentbit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c index fd4644333ba..d257c10f20b 100644 --- a/src/config_format/flb_cf_fluentbit.c +++ b/src/config_format/flb_cf_fluentbit.c @@ -513,6 +513,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, ctx->level--; if (indent) { flb_sds_destroy(indent); + indent = NULL; } goto error; } @@ -677,6 +678,7 @@ static int read_config(struct flb_cf *cf, struct local_ctx *ctx, if (indent) { flb_sds_destroy(indent); + indent = NULL; } flb_free(buf); From 3874e83ab4c3bda39faf9910395da017144af34e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 27 Jan 2022 08:54:02 -0600 Subject: [PATCH 51/51] config_format: validate incoming file path is not null Signed-off-by: Eduardo Silva --- src/config_format/flb_config_format.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index 7b910542477..7d4f4469d74 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -462,6 +462,10 @@ struct flb_cf *flb_cf_create_from_file(struct flb_cf *cf, char *file) int format = FLB_CF_FLUENTBIT; char *ptr; + if (!file) { + return NULL; + } + ptr = strrchr(file, '.'); if (!ptr) { format = FLB_CF_FLUENTBIT;