diff --git a/CMakeLists.txt b/CMakeLists.txt index 5dfdff1b99d..73e85da8388 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -630,6 +630,17 @@ if(FLB_HAVE_UNIX_SOCKET) FLB_DEFINITION(FLB_HAVE_UNIX_SOCKET) endif() +# libyaml support +check_c_source_compiles(" + #include + int main() { + yaml_parser_t parser; + return 0; + }" FLB_HAVE_LIBYAML) +if(FLB_HAVE_LIBYAML) + FLB_DEFINITION(FLB_HAVE_LIBYAML) +endif() + # check attribute alloc_size check_c_source_compiles(" #include diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h new file mode 100644 index 00000000000..0d164fc72b4 --- /dev/null +++ b/include/fluent-bit/config_format/flb_cf.h @@ -0,0 +1,146 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_MAIN_H +#define FLB_CONFIG_FORMAT_MAIN_H + +#include +#include +#include + +#define FLB_CF_ERROR_SERVICE_EXISTS "SERVICE definition already exists" +#define FLB_CF_ERROR_META_CHAR "invalid first meta character: '@' expected" +#define FLB_CF_ERROR_KV_INVALID_KEY "invalid key content" +#define FLB_CF_ERROR_KV_INVALID_VAL "invalid value content" + +/* manipulate error state for the context */ +#define flb_cf_error_set(cf, err) cf->error_str = err +#define flb_cf_error_get(cf) cf->error_str +#define flb_cr_error_reset(cf) cf->error_str = "" + +/* meta commands: handled as key value pairs */ +#define flb_cf_meta flb_kv + +enum cf_file_format { + FLB_CF_FLUENTBIT, +#ifdef FLB_HAVE_LIBYAML + FLB_CF_YAML +#endif +}; + +enum section_type { + FLB_CF_SERVICE = 0, /* [SERVICE] */ + FLB_CF_PARSER, /* [PARSER] */ + FLB_CF_MULTILINE_PARSER, /* [MULTILINE_PARSER] */ + FLB_CF_CUSTOM, /* [CUSTOM] */ + FLB_CF_INPUT, /* [INPUT] */ + FLB_CF_FILTER, /* [FILTER] */ + FLB_CF_OUTPUT, /* [OUTPUT] */ + FLB_CF_OTHER, /* any other section.. */ +}; + +struct flb_cf_group { + flb_sds_t name; /* group name */ + struct mk_list properties; /* key value properties */ + struct mk_list _head; /* link to struct flb_cf_section->groups */ +}; + +struct flb_cf_section { + int type; + flb_sds_t name; /* name (used for FLB_CF_OTHER type) */ + struct mk_list properties; /* key value properties */ + + struct mk_list groups; /* list of groups */ + + struct mk_list _head; /* link to struct flb_cf->sections */ + struct mk_list _head_section; /* link to section type, e.g: inputs, filters.. */ +}; + +struct flb_cf { + /* global service */ + struct flb_cf_section *service; + + /* config environment variables (env section, availble on YAML) */ + struct mk_list env; + + /* meta commands (used by fluentbit classic mode) */ + struct mk_list metas; + + /* parsers */ + struct mk_list parsers; + struct mk_list multiline_parsers; + + /* custom plugins */ + struct mk_list customs; + + /* pipeline */ + struct mk_list inputs; + struct mk_list filters; + struct mk_list outputs; + + /* others */ + struct mk_list others; + + /* list head for all sections */ + struct mk_list sections; + + /* set the last error found */ + char *error_str; +}; + + +struct flb_cf *flb_cf_create(); +struct flb_cf *flb_cf_create_from_file(struct flb_cf *cf, char *file); + +void flb_cf_destroy(struct flb_cf *cf); + +void flb_cf_dump(struct flb_cf *cf); + +/* metas */ +struct flb_kv *flb_cf_meta_create(struct flb_cf *cf, char *meta, int len); + +#define flb_cf_foreach_meta(cf) \ + + +void flb_cf_meta_destroy(struct flb_cf *cf, struct flb_cf_meta *meta); +void flb_cf_meta_destroy_all(struct flb_cf *cf); + +/* groups */ +struct flb_cf_group *flb_cf_group_create(struct flb_cf *cf, struct flb_cf_section *s, + char *name, int len); +void flb_cf_group_destroy(struct flb_cf_group *g); + +/* sections */ +struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int len); +struct flb_cf_section *flb_cf_section_get_by_name(struct flb_cf *cf, char *name); +void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s); +void flb_cf_section_destroy_all(struct flb_cf *cf); + +/* properties */ +struct flb_kv *flb_cf_property_add(struct flb_cf *cf, + struct mk_list *kv_list, + char *k_buf, size_t k_len, + char *v_buf, size_t v_len); + + +char *flb_cf_section_property_get(struct flb_cf *cf, struct flb_cf_section *s, + char *key); + +#endif diff --git a/include/fluent-bit/config_format/flb_cf_fluentbit.h b/include/fluent-bit/config_format/flb_cf_fluentbit.h new file mode 100644 index 00000000000..5766f16df39 --- /dev/null +++ b/include/fluent-bit/config_format/flb_cf_fluentbit.h @@ -0,0 +1,27 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_FLUENTBIT_H +#define FLB_CONFIG_FORMAT_FLUENTBIT_H + +struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, + char *file_path, char *buf, size_t size); + +#endif \ No newline at end of file diff --git a/include/fluent-bit/config_format/flb_cf_yaml.h b/include/fluent-bit/config_format/flb_cf_yaml.h new file mode 100644 index 00000000000..a8cc4f377c0 --- /dev/null +++ b/include/fluent-bit/config_format/flb_cf_yaml.h @@ -0,0 +1,26 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_YAML_H +#define FLB_CONFIG_FORMAT_YAML_H + +struct flb_cf *flb_cf_yaml_create(struct flb_cf *cf, char *file_path, + char *buf, size_t size); +#endif diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index 76ed3cd3ef0..6f1531d6ac8 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -69,6 +69,10 @@ struct flb_config { struct mk_rconf *file; + /* main configuration */ + struct flb_cf *cf_main; + struct flb_cf *cf_parsers; + flb_sds_t program_name; /* argv[0] */ /* @@ -255,7 +259,7 @@ int flb_config_set_program_name(struct flb_config *config, char *name); int set_log_level_from_env(struct flb_config *config); #ifdef FLB_HAVE_STATIC_CONF -struct mk_rconf *flb_config_static_open(const char *file); +struct flb_cf *flb_config_static_open(const char *file); #endif struct flb_service_config { diff --git a/include/fluent-bit/flb_config_format.h b/include/fluent-bit/flb_config_format.h new file mode 100644 index 00000000000..27f11b3c322 --- /dev/null +++ b/include/fluent-bit/flb_config_format.h @@ -0,0 +1,33 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_CONFIG_FORMAT_H +#define FLB_CONFIG_FORMAT_H + +#include + +#include "config_format/flb_cf.h" +#include "config_format/flb_cf_fluentbit.h" + +#ifdef FLB_HAVE_LIBYAML +#include "config_format/flb_cf_yaml.h" +#endif + +#endif \ No newline at end of file diff --git a/include/fluent-bit/flb_parser_decoder.h b/include/fluent-bit/flb_parser_decoder.h index 1f1c9c8f4d1..265854d1854 100644 --- a/include/fluent-bit/flb_parser_decoder.h +++ b/include/fluent-bit/flb_parser_decoder.h @@ -23,7 +23,7 @@ #include #include -#include +#include /* Decoder behavior */ #define FLB_PARSER_DEC_DEFAULT 0 /* results place as separate keys */ @@ -59,7 +59,8 @@ struct flb_parser_dec { struct mk_list _head; /* link to parser->decoders */ }; -struct mk_list *flb_parser_decoder_list_create(struct mk_rconf_section *section); +struct mk_list *flb_parser_decoder_list_create(struct flb_cf_section *section); + int flb_parser_decoder_list_destroy(struct mk_list *list); int flb_parser_decoder_do(struct mk_list *decoders, const char *in_buf, size_t in_size, diff --git a/include/fluent-bit/flb_sds.h b/include/fluent-bit/flb_sds.h index 5665aff6612..4d303f1e06a 100644 --- a/include/fluent-bit/flb_sds.h +++ b/include/fluent-bit/flb_sds.h @@ -99,6 +99,7 @@ static inline int flb_sds_casecmp(flb_sds_t s, const char *str, int len) flb_sds_t flb_sds_create(const char *str); flb_sds_t flb_sds_create_len(const char *str, int len); flb_sds_t flb_sds_create_size(size_t size); +int flb_sds_trim(flb_sds_t s); flb_sds_t flb_sds_cat(flb_sds_t s, const char *str, int len); flb_sds_t flb_sds_cat_esc(flb_sds_t s, const char *str, int len, char *esc, size_t esc_size); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 37fb5e7eac0..977898f728a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -61,6 +61,19 @@ set(src flb_event.c ) +# Config format +set(src + ${src} + config_format/flb_config_format.c + config_format/flb_cf_fluentbit.c +) +if(FLB_HAVE_LIBYAML) + set(src + ${src} + config_format/flb_cf_yaml.c + ) +endif() + # Multiline subsystem add_subdirectory(multiline) set(src @@ -329,6 +342,14 @@ if(OPENSSL_FOUND) ) endif() +# libyaml +if(FLB_HAVE_LIBYAML) +set(FLB_DEPS + ${FLB_DEPS} + yaml + ) +endif() + # UTF8 Encoding if(FLB_UTF8_ENCODER) set(FLB_DEPS diff --git a/src/config_format/flb_cf_fluentbit.c b/src/config_format/flb_cf_fluentbit.c new file mode 100644 index 00000000000..d257c10f20b --- /dev/null +++ b/src/config_format/flb_cf_fluentbit.c @@ -0,0 +1,741 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#ifndef _MSC_VER +#include +#endif + +#ifdef _WIN32 +#include +#include +#define PATH_MAX MAX_PATH +#endif + +#define FLB_CF_BUF_SIZE 4096 + +/* indent checker return codes */ +#define INDENT_ERROR -1 +#define INDENT_OK 0 +#define INDENT_GROUP_CONTENT 1 + +/* Included file by configuration */ +struct local_file { + flb_sds_t path; + struct mk_list _head; +}; + +/* Local context to keep state of variables and general */ +struct local_ctx { + int level; + char *file; + flb_sds_t root_path; + + /* included files */ + struct mk_list includes; + + /* meta instructions */ + struct mk_list metas; + + /* list of sections */ + struct mk_list sections; +}; + +static int read_config(struct flb_cf *cf, struct local_ctx *ctx, char *cfg_file, + char *buf, size_t size); + +/* Raise a configuration schema error */ +static void config_error(const char *path, int line, const char *msg) +{ + flb_error("[config] error in %s:%i: %s", path, line, msg); +} + +/* Raise a warning */ +static void config_warn(const char *path, int line, const char *msg) +{ + mk_warn("Config file warning '%s':\n" + "\t\t\t\tat line %i: %s", + path, line, msg); +} + +static int char_search(const char *string, int c, int len) +{ + char *p; + + if (len < 0) { + len = strlen(string); + } + + p = memchr(string, c, len); + if (p) { + return (p - string); + } + + return -1; +} + +/* + * Helper function to simulate a fgets(2) but instead of using a real file stream + * uses the data buffer provided. + */ +static int static_fgets(char *out, size_t size, const char *data, size_t *off) +{ + size_t len; + const char *start = data + *off; + char *end; + + end = strchr(start, '\n'); + + if (!end || *off >= size) { + len = size - *off - 1; + memcpy(out, start, len); + out[len] = '\0'; + *off += len + 1; + return 0; + } + + len = end - start; + if (len >= size) { + len = size - 1; + } + memcpy(out, start, len); + out[len] = '\0'; + *off += len + 1; + + return 1; +} + +#ifndef _WIN32 +static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char * path) +{ + int ret = -1; + glob_t glb; + char tmp[PATH_MAX]; + + const char *glb_path; + size_t i; + int ret_glb = -1; + + if (ctx->root_path && path[0] != '/') { + snprintf(tmp, PATH_MAX, "%s/%s", ctx->root_path, path); + glb_path = tmp; + } + else { + glb_path = path; + } + + ret_glb = glob(glb_path, GLOB_NOSORT, NULL, &glb); + if (ret_glb != 0) { + switch(ret_glb){ + case GLOB_NOSPACE: + flb_warn("[%s] glob: [%s] no space", __FUNCTION__, glb_path); + break; + case GLOB_NOMATCH: + flb_warn("[%s] glob: [%s] no match", __FUNCTION__, glb_path); + break; + case GLOB_ABORTED: + flb_warn("[%s] glob: [%s] aborted", __FUNCTION__, glb_path); + break; + default: + flb_warn("[%s] glob: [%s] other error", __FUNCTION__, glb_path); + } + return ret; + } + + for (i = 0; i < glb.gl_pathc; i++) { + ret = read_config(cf, ctx, glb.gl_pathv[i], NULL, 0); + if (ret < 0) { + break; + } + } + + globfree(&glb); + return ret; +} +#else +static int read_glob(struct flb_cf *cf, struct local_ctx *ctx, const char *path) +{ + char *star, *p0, *p1; + char pattern[MAX_PATH]; + char buf[MAX_PATH]; + int ret; + struct stat st; + HANDLE h; + WIN32_FIND_DATA data; + + if (strlen(path) > MAX_PATH - 1) { + return -1; + } + + star = strchr(path, '*'); + if (star == NULL) { + return -1; + } + + /* + * C:\data\tmp\input_*.conf + * 0<-----| + */ + p0 = star; + while (path <= p0 && *p0 != '\\') { + p0--; + } + + /* + * C:\data\tmp\input_*.conf + * |---->1 + */ + p1 = star; + while (*p1 && *p1 != '\\') { + p1++; + } + + memcpy(pattern, path, (p1 - path)); + pattern[p1 - path] = '\0'; + + h = FindFirstFileA(pattern, &data); + if (h == INVALID_HANDLE_VALUE) { + return 0; + } + + do { + /* Ignore the current and parent dirs */ + if (!strcmp(".", data.cFileName) || !strcmp("..", data.cFileName)) { + continue; + } + + /* Avoid an infinite loop */ + if (strchr(data.cFileName, '*')) { + continue; + } + + /* Create a path (prefix + filename + suffix) */ + memcpy(buf, path, p0 - path + 1); + buf[p0 - path + 1] = '\0'; + + if (FAILED(StringCchCatA(buf, MAX_PATH, data.cFileName))) { + continue; + } + if (FAILED(StringCchCatA(buf, MAX_PATH, p1))) { + continue; + } + + if (strchr(p1, '*')) { + read_glob(cf, ctx, buf); /* recursive */ + continue; + } + + ret = stat(buf, &st); + if (ret == 0 && (st.st_mode & S_IFMT) == S_IFREG) { + if (read_config(cf, ctx, buf, NULL, 0) < 0) { + return -1; + } + } + } while (FindNextFileA(h, &data) != 0); + + FindClose(h); + return 0; +} +#endif + +static int local_init(struct local_ctx *ctx, char *file) +{ + char *end; + char path[PATH_MAX + 1] = {0}; + +#ifndef FLB_HAVE_STATIC_CONF + char *p; + + if (file) { +#ifdef _MSC_VER + p = _fullpath(path, file, PATH_MAX + 1); +#else + p = realpath(file, path); +#endif + if (!p) { + return -1; + } + } +#endif + + /* lookup path ending and truncate */ + end = strrchr(path, '/'); + if (end) { + end++; + *end = '\0'; + } + + if (file) { + ctx->file = flb_sds_create(file); + ctx->root_path = flb_sds_create(path); + } + else { + ctx->file = NULL; + ctx->root_path = NULL; + } + + ctx->level = 0; + mk_list_init(&ctx->metas); + mk_list_init(&ctx->sections); + mk_list_init(&ctx->includes); + + return 0; +} + +static void local_exit(struct local_ctx *ctx) +{ + struct mk_list *tmp; + struct mk_list *head; + struct local_file *f; + + mk_list_foreach_safe(head, tmp, &ctx->includes) { + f = mk_list_entry(head, struct local_file, _head); + flb_sds_destroy(f->path); + mk_list_del(&f->_head); + flb_free(f); + } + + if (ctx->file) { + flb_sds_destroy(ctx->file); + } + + if (ctx->root_path) { + flb_sds_destroy(ctx->root_path); + } +} + +static int is_file_included(struct local_ctx *ctx, const char *path) +{ + struct mk_list *head; + struct local_file *file; + + mk_list_foreach(head, &ctx->includes) { + file = mk_list_entry(head, struct local_file, _head); + if (strcmp(file->path, path) == 0) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + +static int check_indent(const char *line, const char *indent, int *out_level) +{ + int extra = 0; + int level = 0; + + while (*line == *indent && *indent) { + line++; + indent++; + level++; + } + + if (*indent != '\0') { + if (isblank(*line)) { + flb_error("[config] inconsistent use of tab and space"); + } + else { + flb_error("[config] indentation level is too low"); + } + return INDENT_ERROR;; + } + + if (isblank(*line)) { + /* check if we have a 'group' key/value line */ + while (isblank(*line)) { + line++; + extra++; + } + + if (extra == level) { + *out_level = level + extra; + return INDENT_GROUP_CONTENT; + } + + flb_error("[config] extra indentation level found"); + return -1; + } + + *out_level = level; + return INDENT_OK; +} + +static int read_config(struct flb_cf *cf, struct local_ctx *ctx, + char *cfg_file, char *in_data, size_t in_size) +{ + int i; + int len; + int ret; + int end; + int level; + int line = 0; + int indent_len = -1; + int n_keys = 0; + char *key = NULL; + int key_len; + char *val = NULL; + int val_len; + char *buf; + char tmp[PATH_MAX]; + flb_sds_t section = NULL; + flb_sds_t indent = NULL; + struct stat st; + struct local_file *file; + struct flb_cf_meta *meta; + struct flb_cf_section *current_section = NULL; + struct flb_cf_group *current_group = NULL; + + struct flb_kv *kv; + FILE *f = NULL; + + /* Check if the path exists (relative cases for included files) */ +#ifndef FLB_HAVE_STATIC_CONF + if (ctx->level >= 0) { + ret = stat(cfg_file, &st); + if (ret == -1 && errno == ENOENT) { + /* Try to resolve the real path (if exists) */ + if (cfg_file[0] == '/') { + return -1; + } + + if (ctx->root_path) { + snprintf(tmp, PATH_MAX, "%s/%s", ctx->root_path, cfg_file); + cfg_file = tmp; + } + } + } +#endif + + /* Check this file have not been included before */ + ret = is_file_included(ctx, cfg_file); + if (ret) { + flb_error("[config] file already included %s", cfg_file); + return -1; + } + ctx->level++; + +#ifndef FLB_HAVE_STATIC_CONF + /* Open configuration file */ + if ((f = fopen(cfg_file, "r")) == NULL) { + flb_warn("[config] I cannot open %s file", cfg_file); + return -1; + } +#endif + + /* Allocate temporal buffer to read file content */ + buf = flb_malloc(FLB_CF_BUF_SIZE); + if (!buf) { + flb_errno(); + goto error; + } + +#ifdef FLB_HAVE_STATIC_CONF + /* + * a static configuration comes from a buffer, so we use the static_fgets() + * workaround to retrieve the lines. + */ + size_t off = 0; + while (static_fgets(buf, FLB_CF_BUF_SIZE, in_data, &off)) { +#else + /* normal mode, read lines into a buffer */ + while (fgets(buf, FLB_CF_BUF_SIZE, f)) { +#endif + len = strlen(buf); + if (len > 0 && buf[len - 1] == '\n') { + buf[--len] = 0; + if (len && buf[len - 1] == '\r') { + buf[--len] = 0; + } + } +#ifndef FLB_HAVE_STATIC_CONF + else { + /* + * If we don't find a break line, validate if we got an EOF or not. No EOF + * means that the incoming string is not finished so we must raise an + * exception. + */ + if (!feof(f)) { + config_error(cfg_file, line, "length of content has exceeded limit"); + goto error; + } + } +#endif + + /* Line number */ + line++; + + if (!buf[0]) { + continue; + } + + /* Skip commented lines */ + if (buf[0] == '#') { + continue; + } + + if (len > 9 && strncasecmp(buf, "@INCLUDE ", 9) == 0) { + if (strchr(buf + 9, '*') != NULL) { + ret = read_glob(cf, ctx, buf + 9); + } + else { + ret = read_config(cf, ctx, buf + 9, NULL, 0); + } + if (ret == -1) { + ctx->level--; + if (indent) { + flb_sds_destroy(indent); + indent = NULL; + } + goto error; + } + continue; + } + else if (buf[0] == '@' && len > 3) { + meta = flb_cf_meta_create(cf, buf, len); + if (!meta) { + goto error; + } + continue; + } + + /* Section definition */ + if (buf[0] == '[') { + current_group = NULL; + + end = char_search(buf, ']', len); + if (end > 0) { + /* + * Before to add a new section, lets check the previous + * one have at least one key set + */ + if (current_section && n_keys == 0) { + config_warn(cfg_file, line, + "previous section did not have keys"); + } + + /* Create new section */ + current_section = flb_cf_section_create(cf, buf + 1, end - 1); + if (!current_section) { + continue; + } + current_group = NULL; + n_keys = 0; + continue; + } + else { + config_error(cfg_file, line, "bad header definition"); + goto error; + } + } + + /* No separator defined */ + if (!indent) { + i = 0; + + do { i++; } while (i < len && isblank(buf[i])); + + indent = flb_sds_create_len(buf, i); + indent_len = flb_sds_len(indent); + + /* Blank indented line */ + if (i == len) { + continue; + } + } + + /* Validate indentation level */ + ret = check_indent(buf, indent, &level); + if (ret == INDENT_ERROR) { + config_error(cfg_file, line, "invalid indentation level"); + goto error; + } + else { + if (ret == INDENT_OK && current_group) { + current_group = NULL; + } + indent_len = level; + } + + if (buf[indent_len] == '#' || indent_len == len) { + continue; + } + + /* get the key value separator */ + i = char_search(buf + indent_len, ' ', len - indent_len); + + /* key */ + key = buf + indent_len; + key_len = i; + + if (!key || i < 0) { + config_error(cfg_file, line, "undefined key"); + goto error; + } + + /* Check possible start of a group */ + if (key[0] == '[') { + end = char_search(key, ']', len - indent_len); + if (end == -1) { + config_error(cfg_file, line, "expected a valid group name: [..]"); + goto error; + } + + if (!current_section) { + config_warn(cfg_file, line, + "current group don't have a parent section"); + goto error; + } + + /* check if a previous group exists with one key */ + if (current_group && n_keys == 0) { + config_warn(cfg_file, line, "previous group did not have keys"); + goto error; + } + + /* Create new group */ + current_group = flb_cf_group_create(cf, current_section, + key + 1, end - 1); + if (!current_group) { + continue; + } + n_keys = 0; + + /* continue processing since we need key/value pairs */ + continue; + } + + /* val */ + val = buf + indent_len + i + 1; + val_len = len - indent_len - i - 1; + + if (!key || !val || i < 0) { + config_error(cfg_file, line, "each key must have a value"); + goto error; + } + + if (val_len == 0) { + config_error(cfg_file, line, "key has an empty value"); + goto error; + } + + /* register entry: key and val are copied as duplicated */ + kv = NULL; + if (current_group) { + kv = flb_cf_property_add(cf, ¤t_group->properties, + key, key_len, + val, val_len); + } + else if (current_section) { + kv = flb_cf_property_add(cf, ¤t_section->properties, + key, key_len, + val, val_len); + } + if (!kv) { + config_error(cfg_file, line, "could not allocate key value pair"); + goto error; + } + + /* Free temporary key and val */ + n_keys++; + } + + if (section && n_keys == 0) { + /* No key, no warning */ + } + + if (f) { + fclose(f); + } + + if (indent) { + flb_sds_destroy(indent); + indent = NULL; + } + flb_free(buf); + + /* Append this file to the list */ + file = flb_malloc(sizeof(struct local_file)); + if (!file) { + flb_errno(); + ctx->level--; + goto error; + } + file->path = flb_sds_create(cfg_file); + mk_list_add(&file->_head, &ctx->includes); + ctx->level--; + + return 0; + +error: + if (f) { + fclose(f); + } + if (indent) { + flb_sds_destroy(indent); + } + flb_free(buf); + return -1; +} + +struct flb_cf *flb_cf_fluentbit_create(struct flb_cf *cf, + char *file_path, char *buf, size_t size) +{ + int ret; + int created = FLB_FALSE; + struct local_ctx ctx; + + if (!cf) { + cf = flb_cf_create(); + if (!cf) { + return NULL; + } + created = FLB_TRUE; + } + + ret = local_init(&ctx, file_path); + if (ret != 0) { + if (cf && created) { + flb_cf_destroy(cf); + } + return NULL; + } + + ret = read_config(cf, &ctx, file_path, buf, size); + + local_exit(&ctx); + + if (ret == -1 && created) { + flb_cf_destroy(cf); + } + + return cf; +} diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c new file mode 100644 index 00000000000..a4d790a8433 --- /dev/null +++ b/src/config_format/flb_cf_yaml.c @@ -0,0 +1,579 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +enum section { + SECTION_ENV, + SECTION_SERVICE, + SECTION_PIPELINE, + SECTION_CUSTOM, + SECTION_INPUT, + SECTION_FILTER, + SECTION_OUTPUT, + SECTION_OTHER, +}; + +enum state { + STATE_START, /* start state */ + STATE_STREAM, /* start/end stream */ + STATE_DOCUMENT, /* start/end document */ + + STATE_SECTION, /* top level */ + STATE_SECTION_KEY, + STATE_SECTION_VAL, + + STATE_SERVICE, /* service section */ + STATE_OTHER, /* service section */ + + STATE_PIPELINE, /* pipeline groups customs inputs, filters and outputs */ + + STATE_PLUGIN_CUSTOM, /* custom plugins section */ + STATE_PLUGIN_INPUT, /* input plugins section */ + STATE_PLUGIN_FILTER, /* filter plugins section */ + STATE_PLUGIN_OUTPUT, /* output plugins section */ + + STATE_PLUGIN_TYPE, + STATE_PLUGIN_KEY_VALUE_PAIR, + STATE_PLUGIN_KEY, + STATE_PLUGIN_VAL, + + STATE_GROUP_KEY, + STATE_GROUP_VAL, + + /* environment variables */ + STATE_ENV, + + + STATE_STOP /* end state */ +}; + +struct parser_state { + /* file path */ + flb_sds_t file; + + /* tokens state */ + enum state state; + + /* active section (if any) */ + enum section section; + + /* temporary key value pair */ + flb_sds_t key; + flb_sds_t val; + + /* active section */ + struct flb_cf_section *cf_section; + + /* active group */ + struct flb_cf_group *cf_group; + + int service_set; +}; + +/* yaml_* functions return 1 on success and 0 on failure. */ +enum status { + YAML_SUCCESS = 1, + YAML_FAILURE = 0 +}; + +static int add_section_type(struct flb_cf *cf, struct parser_state *s) +{ + if (s->section == SECTION_CUSTOM) { + s->cf_section = flb_cf_section_create(cf, "CUSTOM", 0); + } + else if (s->section == SECTION_INPUT) { + s->cf_section = flb_cf_section_create(cf, "INPUT", 0); + } + else if (s->section == SECTION_FILTER) { + s->cf_section = flb_cf_section_create(cf, "FILTER", 0); + } + else if (s->section == SECTION_OUTPUT) { + s->cf_section = flb_cf_section_create(cf, "OUTPUT", 0); + } + + if (!s->cf_section) { + return -1; + } + + return 0; +} + +static void yaml_error_event(struct parser_state *s, yaml_event_t *event) +{ + flb_error("[config] YAML error found in file \"%s\", line %i, column %i: " + "unexpected event %d in state %d.", + s->file, event->start_mark.line + 1, event->start_mark.column, + event->type, s->state); +} + +static void yaml_error_definition(struct parser_state *s, yaml_event_t *event, + char *value) +{ + flb_error("[config] YAML error found in file \"%s\", line %i, column %i: " + "duplicated definition of '%s'", + s->file, event->start_mark.line + 1, event->start_mark.column, + value); +} + +static void yaml_error_plugin_category(struct parser_state *s, yaml_event_t *event, + char *value) +{ + flb_error("[config] YAML error found in file \"%s\", line %i, column %i: " + "the pipeline component '%s' is not valid. Try one of these values: " + "customs, inputs, filters or outputs.", + s->file, event->start_mark.line + 1, event->start_mark.column, + value); +} + +static int consume_event(struct flb_cf *cf, struct parser_state *s, + yaml_event_t *event) +{ + int len; + int ret; + char *value; + struct mk_list *list; + struct flb_kv *kv; + + switch (s->state) { + case STATE_START: + switch (event->type) { + case YAML_STREAM_START_EVENT: + s->state = STATE_STREAM; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_STREAM: + switch (event->type) { + case YAML_DOCUMENT_START_EVENT: + s->state = STATE_DOCUMENT; + break; + case YAML_STREAM_END_EVENT: + s->state = STATE_STOP; /* all done */ + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_DOCUMENT: + switch (event->type) { + case YAML_MAPPING_START_EVENT: + s->state = STATE_SECTION; + break; + case YAML_DOCUMENT_END_EVENT: + s->state = STATE_STREAM; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_PIPELINE: + switch (event->type) { + case YAML_SCALAR_EVENT: + value = (char *)event->data.scalar.value; + if (strcasecmp(value, "customs") == 0) { + s->state = STATE_PLUGIN_CUSTOM; + s->section = SECTION_CUSTOM; + } + else if (strcasecmp(value, "inputs") == 0) { + s->state = STATE_PLUGIN_INPUT; + s->section = SECTION_INPUT; + } + else if (strcasecmp(value, "filters") == 0) { + s->state = STATE_PLUGIN_FILTER; + s->section = SECTION_FILTER; + } + else if (strcasecmp(value, "outputs") == 0) { + s->state = STATE_PLUGIN_OUTPUT; + s->section = SECTION_OUTPUT; + } + else { + yaml_error_plugin_category(s, event, value); + return YAML_FAILURE; + } + break; + case YAML_MAPPING_START_EVENT: + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_SECTION; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + case STATE_SECTION: + s->section = SECTION_OTHER; + switch (event->type) { + case YAML_SCALAR_EVENT: + value = (char *)event->data.scalar.value; + if (strcasecmp(value, "env") == 0) { + s->state = STATE_ENV; + s->section = SECTION_ENV; + } + else if (strcasecmp(value, "pipeline") == 0) { + s->state = STATE_PIPELINE; + s->section = SECTION_PIPELINE; + } + else if (strcasecmp(value, "service") == 0) { + if (s->service_set) { + yaml_error_definition(s, event, value); + return YAML_FAILURE; + } + s->state = STATE_SERVICE; + s->section = SECTION_SERVICE; + s->service_set = 1; + s->cf_section = flb_cf_section_create(cf, value, 0); + if (!s->cf_section) { + return YAML_FAILURE; + } + } + else { + /* any other main section definition (e.g: similar to STATE_SERVICE) */ + s->state = STATE_OTHER; + s->cf_section = flb_cf_section_create(cf, value, 0); + if (!s->cf_section) { + return YAML_FAILURE; + } + } + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_DOCUMENT; + break; + case YAML_DOCUMENT_END_EVENT: + s->state = STATE_STREAM; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + /* service or others */ + case STATE_ENV: + case STATE_SERVICE: + case STATE_OTHER: + switch(event->type) { + case YAML_MAPPING_START_EVENT: + s->state = STATE_SECTION_KEY; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_SECTION; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_SECTION_KEY: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_SECTION_VAL; + value = (char *) event->data.scalar.value; + s->key = flb_sds_create(value); + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_SECTION; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_SECTION_VAL: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_SECTION_KEY; + value = (char *) event->data.scalar.value; + s->val = flb_sds_create(value); + + /* Check if the incoming k/v pair set a config environment variable */ + if (s->section == SECTION_ENV) { + list = &cf->env; + } + else { + list = &s->cf_section->properties; + } + + /* register key/value pair as a property */ + kv = flb_cf_property_add(cf, list /*&s->cf_section->properties*/, + s->key, flb_sds_len(s->key), + s->val, flb_sds_len(s->val)); + if (!kv) { + return YAML_FAILURE; + } + flb_sds_destroy(s->key); + flb_sds_destroy(s->val); + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + /* Plugin types */ + case STATE_PLUGIN_CUSTOM: + case STATE_PLUGIN_INPUT: + case STATE_PLUGIN_FILTER: + case STATE_PLUGIN_OUTPUT: + switch(event->type) { + case YAML_MAPPING_START_EVENT: + s->state = STATE_PLUGIN_TYPE; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_SECTION_KEY; + break; + case YAML_SCALAR_EVENT: + s->state = STATE_SECTION; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_PLUGIN_TYPE: + switch(event->type) { + case YAML_SCALAR_EVENT: + /* register the section by type */ + ret = add_section_type(cf, s); + if (ret == -1) { + return YAML_FAILURE; + } + + /* the 'type' is the plugin name, so we add it as a 'name' property */ + value = (char *) event->data.scalar.value; + len = strlen(value); + + /* register the type: name = abc */ + kv = flb_cf_property_add(cf, &s->cf_section->properties, + "name", 4, + value, len); + if (!kv) { + return YAML_FAILURE; + } + + /* next state are key value pairs */ + s->state = STATE_PLUGIN_KEY_VALUE_PAIR; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_PIPELINE; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_PLUGIN_KEY_VALUE_PAIR: + switch(event->type) { + case YAML_MAPPING_START_EVENT: + s->state = STATE_PLUGIN_KEY; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_PLUGIN_TYPE; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_PLUGIN_KEY: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_PLUGIN_VAL; + value = (char *) event->data.scalar.value; + s->key = flb_sds_create(value); + break; + case YAML_MAPPING_START_EVENT: + s->state = STATE_PLUGIN_TYPE; + break; + case YAML_MAPPING_END_EVENT: + s->state = STATE_PLUGIN_TYPE; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_PLUGIN_VAL: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_PLUGIN_KEY; + value = (char *) event->data.scalar.value; + s->val = flb_sds_create(value); + + /* register key/value pair as a property */ + flb_cf_property_add(cf, &s->cf_section->properties, + s->key, flb_sds_len(s->key), + s->val, flb_sds_len(s->val)); + flb_sds_destroy(s->key); + flb_sds_destroy(s->val); + break; + case YAML_MAPPING_START_EVENT: /* start a new group */ + s->state = STATE_GROUP_KEY; + s->cf_group = flb_cf_group_create(cf, s->cf_section, + s->key, flb_sds_len(s->key)); + flb_sds_destroy(s->key); + if (!s->cf_group) { + return YAML_FAILURE; + } + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_GROUP_KEY: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_GROUP_VAL; + value = (char *) event->data.scalar.value; + s->key = flb_sds_create(value); + break; + case YAML_MAPPING_END_EVENT: + s->cf_group = NULL; + s->state = STATE_PLUGIN_KEY; + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_GROUP_VAL: + switch(event->type) { + case YAML_SCALAR_EVENT: + s->state = STATE_GROUP_KEY; + value = (char *) event->data.scalar.value; + s->val = flb_sds_create(value); + + /* register key/value pair as a property */ + flb_cf_property_add(cf, &s->cf_group->properties, + s->key, flb_sds_len(s->key), + s->val, flb_sds_len(s->val)); + flb_sds_destroy(s->key); + flb_sds_destroy(s->val); + break; + default: + yaml_error_event(s, event); + return YAML_FAILURE; + } + break; + + case STATE_STOP: + break; + } + + return YAML_SUCCESS; +} + +static int read_config(struct flb_cf *cf, void *ctx, char *cfg_file) +{ + int code = 0; + int status; + struct parser_state state; + yaml_parser_t parser; + yaml_event_t event; + FILE *fh; + + fh = fopen(cfg_file, "r"); + if (!fh) { + flb_errno(); + return -1; + } + + memset(&state, '\0', sizeof(state)); + state.file = flb_sds_create(cfg_file); + if (!state.file) { + fclose(fh); + return -1; + } + + yaml_parser_initialize(&parser); + yaml_parser_set_input_file(&parser, fh); + + do { + status = yaml_parser_parse(&parser, &event); + if (status == YAML_FAILURE) { + flb_error("[config] invalid YAML format"); + code = -1; + goto done; + } + status = consume_event(cf, &state, &event); + if (status == YAML_FAILURE) { + code = -1; + goto done; + } + yaml_event_delete(&event); + } while (state.state != STATE_STOP); + +done: + if (code == -1) { + yaml_event_delete(&event); + } + flb_sds_destroy(state.file); + yaml_parser_delete(&parser); + fclose(fh); + + return code; +} + +struct flb_cf *flb_cf_yaml_create(struct flb_cf *cf, char *file_path, + char *buf, size_t size) +{ + int ret; + + if (!cf) { + cf = flb_cf_create(); + if (!cf) { + return NULL; + } + } + + ret = read_config(cf, NULL, file_path); + if (ret == -1) { + flb_cf_destroy(cf); + return NULL; + } + + return cf; +} diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c new file mode 100644 index 00000000000..7d4f4469d74 --- /dev/null +++ b/src/config_format/flb_config_format.c @@ -0,0 +1,495 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +int flb_cf_file_read() +{ + return 0; +} + +struct flb_cf *flb_cf_create() +{ + struct flb_cf *ctx; + + ctx = flb_calloc(1, sizeof(struct flb_cf)); + if (!ctx) { + flb_errno(); + return NULL; + } + + /* env vars */ + mk_list_init(&ctx->env); + + /* meta commands */ + mk_list_init(&ctx->metas); + + /* parsers */ + mk_list_init(&ctx->parsers); + mk_list_init(&ctx->multiline_parsers); + + /* custom plugins */ + mk_list_init(&ctx->customs); + + /* pipeline */ + mk_list_init(&ctx->inputs); + mk_list_init(&ctx->filters); + mk_list_init(&ctx->outputs); + + /* other sections */ + mk_list_init(&ctx->others); + + /* general list for sections */ + mk_list_init(&ctx->sections); + + return ctx; +} + +void flb_cf_destroy(struct flb_cf *cf) +{ + flb_kv_release(&cf->env); + flb_kv_release(&cf->metas); + flb_cf_section_destroy_all(cf); + flb_free(cf); +} + +static enum section_type get_section_type(char *name, int len) +{ + if (strncasecmp(name, "SERVICE", len) == 0) { + return FLB_CF_SERVICE; + } + else if (strncasecmp(name, "PARSER", len) == 0) { + return FLB_CF_PARSER; + } + else if (strncasecmp(name, "MULTILINE_PARSER", len) == 0) { + return FLB_CF_MULTILINE_PARSER; + } + else if (strncasecmp(name, "CUSTOM", len) == 0 || + strncasecmp(name, "CUSTOMS", len) == 0) { + return FLB_CF_CUSTOM; + } + else if (strncasecmp(name, "INPUT", len) == 0 || + strncasecmp(name, "INPUTS", len) == 0) { + return FLB_CF_INPUT; + } + else if (strncasecmp(name, "FILTER", len) == 0 || + strncasecmp(name, "FILTERS", len) == 0) { + return FLB_CF_FILTER; + } + else if (strncasecmp(name, "OUTPUT", len) == 0 || + strncasecmp(name, "OUTPUTS", len) == 0) { + return FLB_CF_OUTPUT; + } + + return FLB_CF_OTHER; +} + +struct flb_kv *flb_cf_property_add(struct flb_cf *cf, + struct mk_list *kv_list, + char *k_buf, size_t k_len, + char *v_buf, size_t v_len) +{ + int ret; + struct flb_kv *kv; + + if (k_len == 0) { + k_len = strlen(k_buf); + } + if (v_len == 0) { + v_len = strlen(v_buf); + } + + kv = flb_kv_item_create_len(kv_list, k_buf, k_len, v_buf, v_len); + if (!kv) { + return NULL; + } + + /* sanitize key and value by removing empty spaces */ + ret = flb_sds_trim(kv->key); + if (ret == -1) { + flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_KEY); + flb_kv_item_destroy(kv); + return NULL; + } + + ret = flb_sds_trim(kv->val); + if (ret == -1) { + flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_VAL); + flb_kv_item_destroy(kv); + return NULL; + } + + return kv; +} + +char *flb_cf_section_property_get(struct flb_cf *cf, struct flb_cf_section *s, + char *key) +{ + (void) cf; + return (char *) flb_kv_get_key_value(key, &s->properties); +} + +struct flb_kv *flb_cf_meta_create(struct flb_cf *cf, char *meta, int len) +{ + int xlen; + char *p; + char *tmp; + struct flb_kv *kv; + + if (len <= 0) { + len = strlen(meta); + if (len == 0) { + return NULL; + } + } + + if (meta[0] != '@') { + flb_cf_error_set(cf, FLB_CF_ERROR_META_CHAR); + return NULL; + } + + p = meta; + tmp = strchr(p, ' '); + xlen = (tmp - p); + + /* create k/v pair */ + kv = flb_cf_property_add(cf, &cf->metas, + meta + 1, xlen - 1, + meta + xlen + 1, len - xlen - 1); + if (!kv) { + return NULL; + } + + return kv; +} + +struct flb_cf_group *flb_cf_group_create(struct flb_cf *cf, struct flb_cf_section *s, + char *name, int len) +{ + struct flb_cf_group *g; + + if (!name || strlen(name) == 0 || len < 1) { + return NULL; + } + + /* section context */ + g = flb_malloc(sizeof(struct flb_cf_group)); + if (!g) { + flb_errno(); + return NULL; + } + + /* initialize lists */ + flb_kv_init(&g->properties); + + /* determinate type by name */ + if (len <= 0) { + len = strlen(name); + } + + /* create a NULL terminated name */ + g->name = flb_sds_create_len(name, len); + if (!g->name) { + flb_free(g); + return NULL; + } + + /* link to global section */ + mk_list_add(&g->_head, &s->groups); + + return g; +} + +void flb_cf_group_destroy(struct flb_cf_group *g) +{ + if (g->name) { + flb_sds_destroy(g->name); + } + + flb_kv_release(&g->properties); + mk_list_del(&g->_head); + flb_free(g); +} + +struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int len) +{ + int type; + struct flb_cf_section *s; + + if (!name) { + return NULL; + } + + /* determinate type by name */ + if (len <= 0) { + len = strlen(name); + } + + /* get the section type */ + type = get_section_type(name, len); + + /* check if 'service' already exists */ + if (type == FLB_CF_SERVICE && cf->service) { + return cf->service; + } + + /* section context */ + s = flb_malloc(sizeof(struct flb_cf_section)); + if (!s) { + flb_errno(); + return NULL; + } + + /* initialize lists */ + flb_kv_init(&s->properties); + mk_list_init(&s->groups); + + /* create a NULL terminated name */ + s->name = flb_sds_create_len(name, len); + if (!s->name) { + flb_free(s); + return NULL; + } + s->type = type; + + if (type == FLB_CF_SERVICE && !cf->service) { + cf->service = s; + } + + /* link to global section */ + mk_list_add(&s->_head, &cf->sections); + + /* link to list per type */ + if (type == FLB_CF_PARSER) { + mk_list_add(&s->_head_section, &cf->parsers); + } + else if (type == FLB_CF_MULTILINE_PARSER) { + mk_list_add(&s->_head_section, &cf->multiline_parsers); + } + else if (type == FLB_CF_CUSTOM) { + mk_list_add(&s->_head_section, &cf->customs); + } + else if (type == FLB_CF_INPUT) { + mk_list_add(&s->_head_section, &cf->inputs); + } + else if (type == FLB_CF_FILTER) { + mk_list_add(&s->_head_section, &cf->filters); + } + else if (type == FLB_CF_OUTPUT) { + mk_list_add(&s->_head_section, &cf->outputs); + } + else if (type == FLB_CF_OTHER) { + mk_list_add(&s->_head_section, &cf->others); + } + + return s; +} + +/* returns the first match of a section that it name matches 'name' parameter */ +struct flb_cf_section *flb_cf_section_get_by_name(struct flb_cf *cf, char *name) +{ + struct mk_list *head; + struct flb_cf_section *s; + + mk_list_foreach(head, &cf->sections) { + s = mk_list_entry(head, struct flb_cf_section, _head); + if (strcasecmp(s->name, name) == 0) { + return s; + } + } + + return NULL; +} + +void flb_cf_section_destroy(struct flb_cf *cf, struct flb_cf_section *s) +{ + struct mk_list *tmp; + struct mk_list *head; + struct flb_cf_group *g; + + if (s->name) { + flb_sds_destroy(s->name); + s->name = NULL; + } + flb_kv_release(&s->properties); + + /* groups */ + mk_list_foreach_safe(head, tmp, &s->groups) { + g = mk_list_entry(head, struct flb_cf_group, _head); + flb_cf_group_destroy(g); + } + + /* unlink */ + mk_list_del(&s->_head); + + if (s->type != FLB_CF_SERVICE) { + mk_list_del(&s->_head_section); + } + + flb_free(s); +} + +static void section_destroy_list(struct flb_cf *cf, struct mk_list *list) +{ + struct mk_list *tmp; + struct mk_list *head; + struct flb_cf_section *s; + + mk_list_foreach_safe(head, tmp, list) { + s = mk_list_entry(head, struct flb_cf_section, _head); + flb_cf_section_destroy(cf, s); + } +} + +void flb_cf_section_destroy_all(struct flb_cf *cf) +{ + section_destroy_list(cf, &cf->sections); +} + +/* + * Helpers + * ------- + */ + +static char *section_type_str(int type) +{ + switch (type) { + case FLB_CF_SERVICE: + return "SERVICE"; + case FLB_CF_PARSER: + return "PARSER"; + case FLB_CF_MULTILINE_PARSER: + return "MULTILINE_PARSER"; + case FLB_CF_CUSTOM: + return "CUSTOM"; + case FLB_CF_INPUT: + return "INPUT"; + case FLB_CF_FILTER: + return "FILTER"; + case FLB_CF_OUTPUT: + return "OUTPUT"; + case FLB_CF_OTHER: + return "OTHER"; + default: + return "error / unknown"; + } + + return NULL; +} + +static void dump_section(struct flb_cf_section *s) +{ + struct mk_list *head; + struct mk_list *p_head; + struct flb_kv *kv; + struct flb_cf_group *g; + + printf("> section:\n name: %s\n type: %s\n", + s->name, section_type_str(s->type)); + + if (mk_list_size(&s->properties) > 0) { + printf(" properties:\n"); + mk_list_foreach(p_head, &s->properties) { + kv = mk_list_entry(p_head, struct flb_kv, _head); + printf(" - %-15s: %s\n", kv->key, kv->val); + } + } + else { + printf(" properties: NONE\n"); + } + + if (mk_list_size(&s->groups) <= 0) { + printf(" groups : NONE\n"); + return; + } + + mk_list_foreach(head, &s->groups) { + g = mk_list_entry(head, struct flb_cf_group, _head); + printf(" > group:\n name: %s\n", g->name); + + if (mk_list_size(&g->properties) > 0) { + printf(" properties:\n"); + mk_list_foreach(p_head, &g->properties) { + kv = mk_list_entry(p_head, struct flb_kv, _head); + printf(" - %-11s: %s\n", kv->key, kv->val); + } + } + else { + printf(" properties: NONE\n"); + } + } +} + +static void dump_section_list(struct mk_list *list) +{ + struct mk_list *head; + struct flb_cf_section *s; + + mk_list_foreach(head, list) { + s = mk_list_entry(head, struct flb_cf_section, _head); + dump_section(s); + } +} + +void flb_cf_dump(struct flb_cf *cf) +{ + dump_section_list(&cf->sections); +} + +struct flb_cf *flb_cf_create_from_file(struct flb_cf *cf, char *file) +{ + int format = FLB_CF_FLUENTBIT; + char *ptr; + + if (!file) { + return NULL; + } + + ptr = strrchr(file, '.'); + if (!ptr) { + format = FLB_CF_FLUENTBIT; + } + else { + if (strcasecmp(ptr, ".conf") == 0) { + format = FLB_CF_FLUENTBIT; + } +#ifdef FLB_HAVE_LIBYAML + else if (strcasecmp(ptr, ".yaml") == 0 || strcasecmp(ptr, ".yml") == 0) { + format = FLB_CF_YAML; + } +#endif + } + + if (format == FLB_CF_FLUENTBIT) { + cf = flb_cf_fluentbit_create(cf, file, NULL, 0); + } +#ifdef FLB_HAVE_LIBYAML + else if (format == FLB_CF_YAML) { + cf = flb_cf_yaml_create(cf, file, NULL, 0); + } +#endif + + return cf; + } + diff --git a/src/flb_config.c b/src/flb_config.c index 4697bf8b126..20d63f8d3ff 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -41,6 +41,7 @@ #include #include #include +#include #include const char *FLB_CONF_ENV_LOGLEVEL = "FLB_LOG_LEVEL"; @@ -166,6 +167,8 @@ struct flb_config *flb_config_init() { int ret; struct flb_config *config; + struct flb_cf *cf; + struct flb_cf_section *section; config = flb_calloc(1, sizeof(struct flb_config)); if (!config) { @@ -183,6 +186,22 @@ struct flb_config *flb_config_init() /* Is the engine (event loop) actively running ? */ config->is_running = FLB_TRUE; + /* Initialize config_format context */ + cf = flb_cf_create(); + if (!cf) { + return NULL; + } + config->cf_main = cf; + + section = flb_cf_section_create(cf, "service", 0); + if (!section) { + flb_cf_destroy(cf); + return NULL; + } + + /* config_format for parsers */ + config->cf_parsers = flb_cf_create(); + /* Flush */ config->flush = FLB_CONFIG_FLUSH_SECS; config->daemon = FLB_FALSE; @@ -366,7 +385,8 @@ void flb_config_exit(struct flb_config *config) mk_event_timeout_destroy(config->evl, &collector->event); mk_event_closesocket(collector->fd_timer); } - } else { + } + else { mk_event_del(config->evl, &collector->event); } @@ -444,6 +464,13 @@ void flb_config_exit(struct flb_config *config) } flb_plugins_unregister(config); + + if (config->cf_main) { + flb_cf_destroy(config->cf_main); + } + if (config->cf_parsers) { + flb_cf_destroy(config->cf_parsers); + } flb_free(config); } diff --git a/src/flb_config_static.c b/src/flb_config_static.c index eca662293a0..700b3ee9e67 100644 --- a/src/flb_config_static.c +++ b/src/flb_config_static.c @@ -19,349 +19,26 @@ */ #include -#include +#include #include -#include -#include - -static int is_file_included(struct mk_rconf *conf, const char *path) -{ - struct mk_list *head; - struct mk_rconf_file *file; - - mk_list_foreach(head, &conf->includes) { - file = mk_list_entry(head, struct mk_rconf_file, _head); - if (strcmp(file->path, path) == 0) { - return MK_TRUE; - } - } - - return MK_FALSE; -} - -static int rconf_meta_add(struct mk_rconf *conf, char *buf, int len) -{ - int xlen; - char *p; - char *tmp; - struct mk_rconf_entry *meta; - - if (buf[0] != '@') { - return -1; - } - - meta = mk_mem_alloc(sizeof(struct mk_rconf_entry)); - if (!meta) { - perror("malloc"); - return -1; - } - - p = buf; - tmp = strchr(p, ' '); - xlen = (tmp - p); - meta->key = mk_string_copy_substr(buf, 1, xlen); - mk_string_trim(&meta->key); - - meta->val = mk_string_copy_substr(buf, xlen + 1, len); - mk_string_trim(&meta->val); - - mk_list_add(&meta->_head, &conf->metas); - return 0; -} - -static void rconf_section_entry_add(struct mk_rconf *conf, - const char *key, const char *val) -{ - struct mk_rconf_section *section; - struct mk_rconf_entry *new; - struct mk_list *head = &conf->sections; - - if (mk_list_is_empty(&conf->sections) == 0) { - mk_err("Error: there are not sections available on %s!", conf->file); - return; - } - - /* Last section */ - section = mk_list_entry_last(head, struct mk_rconf_section, _head); - - /* Alloc new entry */ - new = mk_mem_alloc(sizeof(struct mk_rconf_entry)); - new->key = mk_string_dup(key); - new->val = mk_string_dup(val); - - mk_list_add(&new->_head, §ion->entries); -} - -struct mk_rconf_section *rconf_section_add(struct mk_rconf *conf, - char *name) -{ - struct mk_rconf_section *new; - - /* Alloc section node */ - new = mk_mem_alloc(sizeof(struct mk_rconf_section)); - new->name = mk_string_dup(name); - mk_list_init(&new->entries); - mk_list_add(&new->_head, &conf->sections); - - return new; -} - /* - * Helper function to simulate a fgets(2) but instead of using - * a real file stream uses the data buffer provided. - */ -static int static_fgets(char *out, size_t size, const char *data, size_t *off) -{ - size_t len; - const char *start = data + *off; - char *end; - - end = strchr(start, '\n'); - - if (!end || *off >= size) { - len = size - *off - 1; - memcpy(out, start, len); - out[len] = '\0'; - *off += len + 1; - return 0; - } - - len = end - start; - if (len >= size) { - len = size - 1; - } - memcpy(out, start, len); - out[len] = '\0'; - *off += len + 1; - - return 1; -} - -static int flb_config_static_read(struct mk_rconf *conf, - const char *fname, const char *data) -{ - int i; - int len; - int ret; - int line = 0; - int indent_len = -1; - int n_keys = 0; - char *buf; - char *section = NULL; - char *indent = NULL; - char *key, *val; - size_t off; - struct mk_rconf_file *file; - struct mk_rconf_section *current = NULL; - - /* Check this file have not been included before */ - ret = is_file_included(conf, fname); - if (ret == MK_TRUE) { - mk_err("[config] file already included %s", fname); - return -1; - } - - conf->level++; - - /* Allocate temporary buffer to read file content */ - buf = mk_mem_alloc(MK_RCONF_KV_SIZE); - if (!buf) { - perror("malloc"); - return -1; - } - - /* looking for configuration directives */ - off = 0; - while (static_fgets(buf, MK_RCONF_KV_SIZE, data, &off)) { - len = strlen(buf); - if (buf[len - 1] == '\n') { - buf[--len] = 0; - if (len && buf[len - 1] == '\r') { - buf[--len] = 0; - } - } - - /* Line number */ - line++; - - if (!buf[0]) { - continue; - } - - /* Skip commented lines */ - if (buf[0] == '#') { - continue; - } - - if (len > 9 && strncasecmp(buf, "@INCLUDE ", 9) == 0) { - if (strchr(buf + 9, '*') != NULL) { - //ret = mk_rconf_read_glob(conf, buf + 9); - } - else { - //ret = flb_config_static_read(conf, - // const char *fname, const char *data) - //ret = mk_rconf_read(conf, buf + 9); - } - if (ret == -1) { - conf->level--; - if (indent) { - mk_mem_free(indent); - } - mk_mem_free(buf); - return -1; - } - continue; - } - else if (buf[0] == '@' && len > 3) { - ret = rconf_meta_add(conf, buf, len); - if (ret == -1) { - if (indent) { - mk_mem_free(indent); - } - mk_mem_free(buf); - return -1; - } - continue; - } - - /* Section definition */ - if (buf[0] == '[') { - int end = -1; - end = mk_string_char_search(buf, ']', len); - if (end > 0) { - /* - * Before to add a new section, lets check the previous - * one have at least one key set - */ - if (current && n_keys == 0) { - flb_warn("[static conf: %s] " - "Section do not have keys", fname); - } - - /* Create new section */ - section = mk_string_copy_substr(buf, 1, end); - current = rconf_section_add(conf, section); - mk_mem_free(section); - n_keys = 0; - continue; - } - else { - flb_error("[static conf: %s] Bad header definition", - fname); - } - } - - /* No separator defined */ - if (!indent) { - i = 0; - - do { i++; } while (i < len && isblank(buf[i])); - - indent = mk_string_copy_substr(buf, 0, i); - indent_len = strlen(indent); - - /* Blank indented line */ - if (i == len) { - continue; - } - } - - /* Validate indentation level */ - if (strncmp(buf, indent, indent_len) != 0 || - isblank(buf[indent_len]) != 0) { - //mk_config_error(path, line, "Invalid indentation level"); - } - - if (buf[indent_len] == '#' || indent_len == len) { - continue; - } - - /* Get key and val */ - i = mk_string_char_search(buf + indent_len, ' ', len - indent_len); - key = mk_string_copy_substr(buf + indent_len, 0, i); - val = mk_string_copy_substr(buf + indent_len + i, 1, len - indent_len - i); - - if (!key || !val || i < 0) { - //mk_config_error(path, line, "Each key must have a value"); - } - - /* Trim strings */ - mk_string_trim(&key); - mk_string_trim(&val); - - if (strlen(val) == 0) { - //mk_config_error(path, line, "Key has an empty value"); - } - - /* Register entry: key and val are copied as duplicated */ - rconf_section_entry_add(conf, key, val); - - /* Free temporary key and val */ - mk_mem_free(key); - mk_mem_free(val); - - n_keys++; - } - - if (section && n_keys == 0) { - /* No key, no warning */ - } - - /* - struct mk_config_section *s; - struct mk_rconf_entry *e; - - s = conf->section; - while(s) { - printf("\n[%s]", s->name); - e = s->entry; - while(e) { - printf("\n %s = %s", e->key, e->val); - e = e->next; - } - s = s->next; - } - fflush(stdout); - */ - if (indent) { - mk_mem_free(indent); - } - mk_mem_free(buf); - - /* Append this file to the list */ - file = mk_mem_alloc(sizeof(struct mk_rconf_file)); - if (!file) { - perror("malloc"); - conf->level--; - return -1; - } - - file->path = mk_string_dup(fname); - mk_list_add(&file->_head, &conf->includes); - conf->level--; - return 0; -} - -/* - * If Fluent Bit have static configuration support, this function - * allows to lookup, parse and create configuration file contexts - * from the entries generated by CMake at build-time. + * If Fluent Bit has static configuration support, this function allows to lookup, + * parse and create configuration file contexts from the entries generated by + * CMake at build-time. * - * The routine is a modified version from mk_rconf_read() but with - * the differences that it does all operations on top of the global - * entries at flb_config_files[] defined at: + * This routing passes a 'virtual' file name that should be registered into the + * static array 'flb_config_files'. Learn more about it at: * * include/fluent-bit/conf/flb_static_conf.h * */ -struct mk_rconf *flb_config_static_open(const char *file) +struct flb_cf *flb_config_static_open(const char *file) { int i; - int ret; - const char *k; - const char *v; - struct mk_rconf *conf = NULL; + const char *k = NULL; + const char *v = NULL; + struct flb_cf *cf; /* Iterate static array and lookup the file name */ for (i = 0; i < flb_config_files_size; i++) { @@ -378,24 +55,10 @@ struct mk_rconf *flb_config_static_open(const char *file) return NULL; } - /* Alloc configuration node */ - conf = mk_mem_alloc(sizeof(struct mk_rconf)); - conf->created = time(NULL); - conf->file = mk_string_dup(file); - conf->level = -1; - mk_list_init(&conf->sections); - mk_list_init(&conf->includes); - mk_list_init(&conf->metas); - - /* Set the absolute path for the entrypoint file */ - conf->root_path = mk_string_dup("/"); - - /* Read entrypoint */ - ret = flb_config_static_read(conf, k, v); - if (ret == -1) { - mk_rconf_free(conf); + cf = flb_cf_fluentbit_create(NULL, (char *) file, (char *) v, 0); + if (!cf) { return NULL; } - return conf; + return cf; } diff --git a/src/flb_lib.c b/src/flb_lib.c index 040f7dc78b2..ba156681c3b 100644 --- a/src/flb_lib.c +++ b/src/flb_lib.c @@ -704,7 +704,15 @@ int flb_stop(flb_ctx_t *ctx) int ret; pthread_t tid; + tid = ctx->config->worker; + if (ctx->status == FLB_LIB_NONE || ctx->status == FLB_LIB_ERROR) { + /* + * There is a chance the worker thread is still active while + * the service exited for some reason (plugin action). Always + * wait and double check that the child thread is not running. + */ + pthread_join(tid, NULL); return 0; } @@ -718,9 +726,11 @@ int flb_stop(flb_ctx_t *ctx) flb_debug("[lib] sending STOP signal to the engine"); - tid = ctx->config->worker; flb_engine_exit(ctx->config); ret = pthread_join(tid, NULL); + if (ret != 0) { + flb_errno(); + } flb_debug("[lib] Fluent Bit engine stopped"); return ret; diff --git a/src/flb_parser.c b/src/flb_parser.c index 0445678f7fe..e4ea16d300f 100644 --- a/src/flb_parser.c +++ b/src/flb_parser.c @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include #include @@ -424,21 +426,20 @@ static int proc_types_str(const char *types_str, struct flb_parser_types **types return i; } -static flb_sds_t get_parser_key(char *key, - struct flb_config *config, - struct mk_rconf_section *section) +static flb_sds_t get_parser_key(struct flb_config *config, + struct flb_cf *cf, struct flb_cf_section *s, + char *key) + { char *tmp; flb_sds_t val; - tmp = mk_rconf_section_get_key(section, key, MK_RCONF_STR); + tmp = flb_cf_section_property_get(cf, s, key); if (!tmp) { return NULL; } val = flb_env_var_translate(config->env, tmp); - flb_free(tmp); - if (!val) { return NULL; } @@ -451,8 +452,8 @@ static flb_sds_t get_parser_key(char *key, return val; } -/* Config file: read [PARSER] definitions */ -static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, +/* Config file: read 'parser' definitions */ +static int parser_conf_file(const char *cfg, struct flb_cf *cf, struct flb_config *config) { flb_sds_t name; @@ -469,11 +470,11 @@ static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, int types_len; struct mk_list *head; struct mk_list *decoders = NULL; - struct mk_rconf_section *section; + struct flb_cf_section *s; struct flb_parser_types *types = NULL; - /* Read all [PARSER] sections */ - mk_list_foreach(head, &fconf->sections) { + /* Read all 'parser' sections */ + mk_list_foreach(head, &cf->parsers) { name = NULL; format = NULL; regex = NULL; @@ -483,68 +484,67 @@ static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, types_str = NULL; tmp_str = NULL; - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "PARSER") != 0) { - continue; - } + /* retrieve the section context */ + s = mk_list_entry(head, struct flb_cf_section, _head_section); - /* Name */ - name = get_parser_key("Name", config, section); + /* name */ + name = get_parser_key(config, cf, s, "name"); if (!name) { flb_error("[parser] no parser 'name' found in file '%s'", cfg); goto fconf_error; } - /* Format */ - format = get_parser_key("Format", config, section); + /* format */ + format = get_parser_key(config, cf, s, "format"); if (!format) { flb_error("[parser] no parser 'format' found for '%s' in file '%s'", name, cfg); goto fconf_error; } - /* Regex (if 'format' == 'regex') */ - regex = get_parser_key("Regex", config, section); + /* regex (if 'format' == 'regex') */ + regex = get_parser_key(config, cf, s, "regex"); if (!regex && strcmp(format, "regex") == 0) { - flb_error("[parser] no parser 'regex' found for '%s' in file '%s", name, cfg); + flb_error("[parser] no parser 'regex' found for '%s' in file '%s", + name, cfg); goto fconf_error; } - /* Skip_Empty_Values */ + /* skip_empty_values */ skip_empty = FLB_TRUE; - tmp_str = get_parser_key("Skip_Empty_Values", config, section); + tmp_str = get_parser_key(config, cf, s, "skip_empty_values"); if (tmp_str) { skip_empty = flb_utils_bool(tmp_str); flb_sds_destroy(tmp_str); } - /* Time_Format */ - time_fmt = get_parser_key("Time_Format", config, section); + /* time_format */ + time_fmt = get_parser_key(config, cf, s, "time_format"); - /* Time_Key */ - time_key = get_parser_key("Time_Key", config, section); + /* time_key */ + time_key = get_parser_key(config, cf, s, "time_key"); - /* Time_Keep */ + /* time_keep */ time_keep = FLB_FALSE; - tmp_str = get_parser_key("Time_Keep", config, section); + tmp_str = get_parser_key(config, cf, s, "time_keep"); if (tmp_str) { time_keep = flb_utils_bool(tmp_str); flb_sds_destroy(tmp_str); } - /* Time_Strict */ + /* time_strict */ time_strict = FLB_TRUE; - tmp_str = get_parser_key("Time_Strict", config, section); + tmp_str = get_parser_key(config, cf, s, "time_strict"); if (tmp_str) { time_strict = flb_utils_bool(tmp_str); flb_sds_destroy(tmp_str); } - /* Time_Offset (UTC offset) */ - time_offset = get_parser_key("Time_Offset", config, section); + /* time_offset (UTC offset) */ + time_offset = get_parser_key(config, cf, s, "time_offset"); - /* Types */ - types_str = get_parser_key("Types", config, section); + /* types */ + types_str = get_parser_key(config, cf, s, "types"); if (types_str) { types_len = proc_types_str(types_str, &types); } @@ -553,7 +553,7 @@ static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, } /* Decoders */ - decoders = flb_parser_decoder_list_create(section); + decoders = flb_parser_decoder_list_create(s); /* Create the parser context */ if (!flb_parser_create(name, format, regex, skip_empty, @@ -612,20 +612,20 @@ static int parser_conf_file(const char *cfg, struct mk_rconf *fconf, } static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, - struct mk_rconf_section *section, + struct flb_cf_section *section, struct flb_config *config) { int ret; char *to_state = NULL; struct mk_list list; struct mk_list *head; - struct mk_rconf_entry *entry; + struct flb_kv *entry; struct flb_slist_entry *from_state; struct flb_slist_entry *regex_pattern; struct flb_slist_entry *tmp; - mk_list_foreach(head, §ion->entries) { - entry = mk_list_entry(head, struct mk_rconf_entry, _head); + mk_list_foreach(head, §ion->properties) { + entry = mk_list_entry(head, struct flb_kv, _head); /* only process 'rule' keys */ if (strcasecmp(entry->key, "rule") != 0) { @@ -692,8 +692,8 @@ static int multiline_load_regex_rules(struct flb_ml_parser *ml_parser, } -/* Config file: read [MULTILINE_PARSER] definitions */ -static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, +/* config file: read 'multiline_parser' sections */ +static int multiline_parser_conf_file(const char *cfg, struct flb_cf *cf, struct flb_config *config) { int ret; @@ -709,11 +709,11 @@ static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, int flush_timeout; struct flb_parser *parser_ctx; struct mk_list *head; - struct mk_rconf_section *section; + struct flb_cf_section *s; struct flb_ml_parser *ml_parser; - /* Read all [PARSER] sections */ - mk_list_foreach(head, &fconf->sections) { + /* read all 'multiline_parser' sections */ + mk_list_foreach(head, &cf->multiline_parsers) { ml_parser = NULL; name = NULL; type = -1; @@ -726,20 +726,17 @@ static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, flush_timeout = -1; tmp = NULL; - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "MULTILINE_PARSER") != 0) { - continue; - } + s = mk_list_entry(head, struct flb_cf_section, _head_section); /* name */ - name = get_parser_key("name", config, section); + name = get_parser_key(config, cf, s, "name"); if (!name) { flb_error("[multiline_parser] no 'name' defined in file '%s'", cfg); goto fconf_error; } /* type */ - tmp = get_parser_key("type", config, section); + tmp = get_parser_key(config, cf, s, "type"); if (!tmp) { flb_error("[multiline_parser] no 'type' defined in file '%s'", cfg); goto fconf_error; @@ -755,29 +752,29 @@ static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, } /* match_string */ - match_string = get_parser_key("match_string", config, section); + match_string = get_parser_key(config, cf, s, "match_string"); /* negate */ - tmp = get_parser_key("negate", config, section); + tmp = get_parser_key(config, cf, s, "negate"); if (tmp) { negate = flb_utils_bool(tmp); flb_sds_destroy(tmp); } /* key_content */ - key_content = get_parser_key("key_content", config, section); + key_content = get_parser_key(config, cf, s, "key_content"); /* key_pattern */ - key_pattern = get_parser_key("key_pattern", config, section); + key_pattern = get_parser_key(config, cf, s, "key_pattern"); /* key_group */ - key_group = get_parser_key("key_group", config, section); + key_group = get_parser_key(config, cf, s, "key_group"); /* parser */ - parser = get_parser_key("parser", config, section); + parser = get_parser_key(config, cf, s, "parser"); /* flush_timeout */ - tmp = get_parser_key("flush_timeout", config, section); + tmp = get_parser_key(config, cf, s, "flush_timeout"); if (tmp) { flush_timeout = atoi(tmp); } @@ -795,7 +792,7 @@ static int multiline_parser_conf_file(const char *cfg, struct mk_rconf *fconf, /* if type is regex, process rules */ if (type == FLB_ML_REGEX) { - ret = multiline_load_regex_rules(ml_parser, section, config); + ret = multiline_load_regex_rules(ml_parser, s, config); if (ret != 0) { goto fconf_error; } @@ -832,9 +829,9 @@ int flb_parser_conf_file(const char *file, struct flb_config *config) { int ret; char tmp[PATH_MAX + 1]; - const char *cfg = NULL; - struct mk_rconf *fconf; + char *cfg = NULL; struct stat st; + struct flb_cf *cf; #ifndef FLB_HAVE_STATIC_CONF ret = stat(file, &st); @@ -851,33 +848,36 @@ int flb_parser_conf_file(const char *file, struct flb_config *config) } } else { - cfg = file; + cfg = (char *) file; } - fconf = mk_rconf_open(cfg); + cf = flb_cf_create_from_file(config->cf_parsers, cfg); #else - fconf = flb_config_static_open(file); + cf = flb_config_static_open(file); + if (cf) { + if (config->cf_parsers) { + flb_cf_destroy(config->cf_parsers); + } + config->cf_parsers = cf; + } #endif - if (!fconf) { + if (!cf) { return -1; } - /* process [PARSER]'s sections */ - ret = parser_conf_file(cfg, fconf, config); + /* process 'parser' sections */ + ret = parser_conf_file(cfg, cf, config); if (ret == -1) { - mk_rconf_free(fconf); return -1; } - /* processs [MULTILINE_PARSER]'s sections */ - ret = multiline_parser_conf_file(cfg, fconf, config); + /* processs 'multiline_parser' sections */ + ret = multiline_parser_conf_file(cfg, cf, config); if (ret == -1) { - mk_rconf_free(fconf); return -1; } - mk_rconf_free(fconf); return 0; } diff --git a/src/flb_parser_decoder.c b/src/flb_parser_decoder.c index 2f37730d0f5..1588e453fa7 100644 --- a/src/flb_parser_decoder.c +++ b/src/flb_parser_decoder.c @@ -26,6 +26,8 @@ #include #include #include +#include + #include #define TYPE_OUT_STRING 0 /* unstructured text */ @@ -596,13 +598,12 @@ static struct flb_parser_dec *get_decoder_key_context(const char *key_name, int return dec; } -struct mk_list *flb_parser_decoder_list_create(struct mk_rconf_section *section) +struct mk_list *flb_parser_decoder_list_create(struct flb_cf_section *section) { int c = 0; int type; int backend; int size; - struct mk_rconf_entry *entry; struct mk_list *head; struct mk_list *list = NULL; struct mk_list *split; @@ -611,6 +612,7 @@ struct mk_list *flb_parser_decoder_list_create(struct mk_rconf_section *section) struct flb_split_entry *action; struct flb_parser_dec *dec; struct flb_parser_dec_rule *dec_rule; + struct flb_kv *entry; /* Global list to be referenced by parent parser definition */ list = flb_malloc(sizeof(struct mk_list)); @@ -620,15 +622,14 @@ struct mk_list *flb_parser_decoder_list_create(struct mk_rconf_section *section) } mk_list_init(list); - - mk_list_foreach(head, §ion->entries) { - entry = mk_list_entry(head, struct mk_rconf_entry, _head); + mk_list_foreach(head, §ion->properties) { + entry = mk_list_entry(head, struct flb_kv, _head); /* Lookup for specific Decode rules */ - if (strcasecmp(entry->key, "Decode_Field") == 0) { + if (strcasecmp(entry->key, "decode_field") == 0) { type = FLB_PARSER_DEC_DEFAULT; } - else if (strcasecmp(entry->key, "Decode_Field_As") == 0) { + else if (strcasecmp(entry->key, "decode_field_as") == 0) { type = FLB_PARSER_DEC_AS; } else { diff --git a/src/flb_plugin.c b/src/flb_plugin.c index 015bdd47fc9..47e83b10f13 100644 --- a/src/flb_plugin.c +++ b/src/flb_plugin.c @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include #include #include @@ -327,12 +329,12 @@ int flb_plugin_load_config_file(const char *file, struct flb_config *config) int ret; char tmp[PATH_MAX + 1]; const char *cfg = NULL; - struct mk_rconf *fconf; - struct mk_rconf_section *section; - struct mk_rconf_entry *entry; struct mk_list *head; struct mk_list *head_e; struct stat st; + struct flb_cf *cf; + struct flb_cf_section *section; + struct flb_kv *entry; #ifndef FLB_HAVE_STATIC_CONF ret = stat(file, &st); @@ -353,38 +355,39 @@ int flb_plugin_load_config_file(const char *file, struct flb_config *config) } flb_debug("[plugin] opening configuration file %s", cfg); - fconf = mk_rconf_open(cfg); + + cf = flb_cf_create_from_file(NULL, cfg); #else - fconf = flb_config_static_open(file); + cf = flb_config_static_open(file); #endif - if (!fconf) { + if (!cf) { return -1; } - /* Read all [PLUGINS] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "PLUGINS") != 0) { + /* read all 'plugins' sections */ + mk_list_foreach(head, &cf->sections) { + section = mk_list_entry(head, struct flb_cf_section, _head); + if (strcasecmp(section->name, "plugins") != 0) { continue; } - mk_list_foreach(head_e, §ion->entries) { - entry = mk_list_entry(head_e, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "Path") != 0) { + mk_list_foreach(head_e, §ion->properties) { + entry = mk_list_entry(head_e, struct flb_kv, _head); + if (strcasecmp(entry->key, "path") != 0) { continue; } /* Load plugin with router function */ ret = flb_plugin_load_router(entry->val, config); if (ret == -1) { - mk_rconf_free(fconf); + flb_cf_destroy(cf); return -1; } } } - mk_rconf_free(fconf); + flb_cf_destroy(cf); return 0; } diff --git a/src/flb_sds.c b/src/flb_sds.c index fb99a63a6ef..9953b8adacc 100644 --- a/src/flb_sds.c +++ b/src/flb_sds.c @@ -29,7 +29,9 @@ #include #include #include + #include +#include static flb_sds_t sds_alloc(size_t size) { @@ -138,6 +140,67 @@ flb_sds_t flb_sds_cat(flb_sds_t s, const char *str, int len) return s; } + +/* + * remove empty spaces on left/right from sds buffer 's' and return the new length + * of the content. + */ +int flb_sds_trim(flb_sds_t s) +{ + unsigned int i; + unsigned int len; + char *left = 0, *right = 0; + char *buf; + + if (!s) { + return -1; + } + + len = flb_sds_len(s); + if (len == 0) { + return 0; + } + + buf = s; + left = buf; + + /* left spaces */ + while (left) { + if (isspace(*left)) { + left++; + } + else { + break; + } + } + + right = buf + (len - 1); + /* Validate right v/s left */ + if (right < left) { + buf[0] = '\0'; + return -1; + } + + /* Move back */ + while (right != buf){ + if (isspace(*right)) { + right--; + } + else { + break; + } + } + + len = (right - left) + 1; + for (i=0; i #include #include +#include +#include #include #include @@ -108,7 +110,8 @@ struct flb_upstream_node *flb_upstream_ha_node_get(struct flb_upstream_ha *ctx) } static struct flb_upstream_node *create_node(int id, - struct mk_rconf_section *s, + struct flb_cf *cf, + struct flb_cf_section *s, struct flb_config *config) { int i; @@ -131,7 +134,7 @@ static struct flb_upstream_node *create_node(int id, char *tls_key_file = NULL; char *tls_key_passwd = NULL; struct mk_list *head; - struct mk_rconf_entry *entry; + struct flb_kv *entry; struct flb_hash *ht; const char *known_keys[] = {"name", "host", "port", "tls", "tls.vhost", "tls.verify", "tls.debug", @@ -141,7 +144,7 @@ static struct flb_upstream_node *create_node(int id, struct flb_upstream_node *node; /* name */ - name = mk_rconf_section_get_key(s, "name", MK_RCONF_STR); + name = flb_cf_section_property_get(cf, s, "name"); if (!name) { flb_error("[upstream_ha] no 'name' has been set on node #%i", id + 1); @@ -149,7 +152,7 @@ static struct flb_upstream_node *create_node(int id, } /* host */ - host = mk_rconf_section_get_key(s, "host", MK_RCONF_STR); + host = flb_cf_section_property_get(cf, s, "host"); if (!host) { flb_error("[upstream_ha] no 'host' has been set on node #%i", id + 1); @@ -157,7 +160,7 @@ static struct flb_upstream_node *create_node(int id, } /* port */ - port = mk_rconf_section_get_key(s, "port", MK_RCONF_STR); + port = flb_cf_section_property_get(cf, s, "port"); if (!port) { flb_error("[upstream_ha] no 'port' has been set on node #%i", id + 1); @@ -165,44 +168,40 @@ static struct flb_upstream_node *create_node(int id, } /* tls */ - tmp = mk_rconf_section_get_key(s, "tls", MK_RCONF_STR); + tmp = flb_cf_section_property_get(cf, s, "tls"); if (tmp) { tls = flb_utils_bool(tmp); - flb_free(tmp); } /* tls.verify */ - tmp = mk_rconf_section_get_key(s, "tls.verify", MK_RCONF_STR); + tmp = flb_cf_section_property_get(cf, s, "tls.verify"); if (tmp) { tls_verify = flb_utils_bool(tmp); - flb_free(tmp); } /* tls.debug */ - tmp = mk_rconf_section_get_key(s, "tls.debug", MK_RCONF_STR); + tmp = flb_cf_section_property_get(cf, s, "tls.debug"); if (tmp) { tls_debug = atoi(tmp); - flb_free(tmp); } /* tls.vhost */ - tls_vhost = mk_rconf_section_get_key(s, "tls.vhost", MK_RCONF_STR); + tls_vhost = flb_cf_section_property_get(cf, s, "tls.vhost"); /* tls.ca_path */ - tls_ca_path = mk_rconf_section_get_key(s, "tls.ca_path", MK_RCONF_STR); + tls_ca_path = flb_cf_section_property_get(cf, s, "tls.ca_path"); /* tls.ca_file */ - tls_ca_file = mk_rconf_section_get_key(s, "tls.ca_file", MK_RCONF_STR); + tls_ca_file = flb_cf_section_property_get(cf, s, "tls.ca_file"); /* tls.crt_file */ - tls_crt_file = mk_rconf_section_get_key(s, "tls.crt_file", MK_RCONF_STR); + tls_crt_file = flb_cf_section_property_get(cf, s, "tls.crt_file"); /* tls.key_file */ - tls_key_file = mk_rconf_section_get_key(s, "tls.key_file", MK_RCONF_STR); + tls_key_file = flb_cf_section_property_get(cf, s, "tls.key_file"); /* tls.key_file */ - tls_key_passwd = mk_rconf_section_get_key(s, "tls.key_passwd", - MK_RCONF_STR); + tls_key_passwd = flb_cf_section_property_get(cf, s, "tls.key_passwd"); /* * Create hash table to store unknown key/values that might be used @@ -211,16 +210,15 @@ static struct flb_upstream_node *create_node(int id, ht = flb_hash_create(FLB_HASH_EVICT_NONE, 32, 256); if (!ht) { flb_error("[upstream_ha] error creating hash table"); - node = NULL; - goto error; + return NULL; } /* * Iterate mk_rconf section internals, find all unknown keys and add * them to the hash table associated to the node. */ - mk_list_foreach(head, &s->entries) { - entry = mk_list_entry(head, struct mk_rconf_entry, _head); + mk_list_foreach(head, &s->properties) { + entry = mk_list_entry(head, struct flb_kv, _head); /* If this is a known entry, just skip it */ skip = FLB_FALSE; @@ -234,8 +232,8 @@ static struct flb_upstream_node *create_node(int id, continue; } - klen = strlen(entry->key); - vlen = strlen(entry->val); + klen = flb_sds_len(entry->key); + vlen = flb_sds_len(entry->val); /* Always store keys in lowercase */ for (i = 0; i < klen; i++) { @@ -255,11 +253,6 @@ static struct flb_upstream_node *create_node(int id, tls_debug, tls_vhost, tls_ca_path, tls_ca_file, tls_crt_file, tls_key_file, tls_key_passwd, ht, config); - error: - flb_free(name); - flb_free(host); - flb_free(port); - return node; } @@ -272,13 +265,12 @@ struct flb_upstream_ha *flb_upstream_ha_from_file(const char *file, const char *cfg = NULL; char *tmp; char path[PATH_MAX + 1]; - struct mk_rconf_section *u_section; - struct mk_rconf_section *n_section; - struct mk_rconf *fconf; struct stat st; struct mk_list *head; struct flb_upstream_ha *ups; struct flb_upstream_node *node; + struct flb_cf *cf = NULL; + struct flb_cf_section *section; #ifndef FLB_HAVE_STATIC_CONF ret = stat(file, &st); @@ -297,70 +289,66 @@ struct flb_upstream_ha *flb_upstream_ha_from_file(const char *file, cfg = file; } flb_debug("[upstream_ha] opening file %s", cfg); - fconf = mk_rconf_open(cfg); + cf = flb_cf_create_from_file(NULL, (char *) cfg); #else - fconf = flb_config_static_open(file); + //DISABLED/FIXME fconf = flb_config_static_open(file); #endif - if (!fconf) { + if (!cf) { return NULL; } - /* First section must be [UPSTREAM] */ - u_section = mk_list_entry_first(&fconf->sections, - struct mk_rconf_section, _head); - if (strcasecmp(u_section->name, "UPSTREAM") != 0) { - flb_error("[upstream_ha] invalid first section name, " - "expected UPSTREAM"); - mk_rconf_free(fconf); + /* 'upstream' sections are under enum section_type FLB_CF_OTHER */ + section = flb_cf_section_get_by_name(cf, "upstream"); + if (!section) { + flb_error("[upstream_ha] section name 'upstream' could not be found"); + flb_cf_destroy(cf); return NULL; } - /* Get Upstream name */ - tmp = mk_rconf_section_get_key(u_section, "name", MK_RCONF_STR); + /* upstream name */ + tmp = flb_cf_section_property_get(cf, section, "name"); if (!tmp) { - flb_error("[upstream_ha] missing name for upstream at %s", file); - mk_rconf_free(fconf); + flb_error("[upstream_ha] missing name for upstream at %s", cfg); + flb_cf_destroy(cf); return NULL; } ups = flb_upstream_ha_create(tmp); if (!ups) { flb_error("[upstream_ha] cannot create context"); - mk_rconf_free(fconf); + flb_cf_destroy(cf); return NULL; } - /* Register [NODE] sections */ - mk_list_foreach(head, &fconf->sections) { - n_section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(n_section->name, "NODE") != 0) { + /* 'node' sections */ + mk_list_foreach(head, &cf->sections) { + section = mk_list_entry(head, struct flb_cf_section, _head); + if (strcasecmp(section->name, "node") != 0) { continue; } /* Read section info and create a Node context */ - node = create_node(c, n_section, config); + node = create_node(c, cf, section, config); if (!node) { flb_error("[upstream_ha] cannot register node on upstream '%s'", tmp); - mk_rconf_free(fconf); flb_upstream_ha_destroy(ups); - flb_free(tmp); + flb_cf_destroy(cf); return NULL; } flb_upstream_ha_node_add(ups, node); c++; } - flb_free(tmp); if (c == 0) { flb_error("[upstream_ha] no nodes defined"); - mk_rconf_free(fconf); flb_upstream_ha_destroy(ups); + flb_cf_destroy(cf); return NULL; } - mk_rconf_free(fconf); + flb_cf_destroy(cf); return ups; } diff --git a/src/flb_utils.c b/src/flb_utils.c index d76a83e42f0..ff02221e337 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -61,55 +61,55 @@ void flb_utils_error(int err) switch (err) { case FLB_ERR_CFG_FILE: - msg = "Could not open configuration file"; + msg = "could not open configuration file"; break; case FLB_ERR_CFG_FILE_FORMAT: - msg = "Configuration file contains format errors"; + msg = "configuration file contains format errors"; break; case FLB_ERR_CFG_FILE_STOP: - msg = "Configuration file contains errors"; + msg = "configuration file contains errors"; break; case FLB_ERR_CFG_FLUSH: - msg = "Invalid flush value"; + msg = "invalid flush value"; break; case FLB_ERR_CFG_FLUSH_CREATE: - msg = "Could not create timer for flushing"; + msg = "could not create timer for flushing"; break; case FLB_ERR_CFG_FLUSH_REGISTER: - msg = "Could not register timer for flushing"; + msg = "could not register timer for flushing"; break; case FLB_ERR_INPUT_INVALID: - msg = "Invalid input type"; + msg = "invalid input type"; break; case FLB_ERR_INPUT_UNDEF: - msg = "No Input(s) have been defined"; + msg = "no input(s) have been defined"; break; case FLB_ERR_INPUT_UNSUP: - msg = "Unsupported Input"; + msg = "unsupported Input"; break; case FLB_ERR_OUTPUT_UNDEF: - msg = "You must specify an output target"; + msg = "you must specify an output target"; break; case FLB_ERR_OUTPUT_INVALID: - msg = "Invalid output target"; + msg = "invalid output target"; break; case FLB_ERR_OUTPUT_UNIQ: - msg = "Just one output type is supported"; + msg = "just one output type is supported"; break; case FLB_ERR_FILTER_INVALID: - msg = "Invalid filter plugin"; + msg = "invalid filter plugin"; break; case FLB_ERR_CFG_PARSER_FILE: - msg = "Could not open parser configuration file"; + msg = "could not open parser configuration file"; break; case FLB_ERR_JSON_INVAL: - msg = "Invalid JSON string"; + msg = "invalid JSON string"; break; case FLB_ERR_JSON_PART: - msg = "Truncated JSON string"; + msg = "truncated JSON string"; break; case FLB_ERR_CORO_STACK_SIZE: - msg = "Invalid coroutine stack size"; + msg = "invalid coroutine stack size"; break; } @@ -125,10 +125,7 @@ void flb_utils_error(int err) } else { - fprintf(stderr, - "%sError%s: %s. Aborting\n\n", - ANSI_BOLD ANSI_RED, ANSI_RESET, msg); - + flb_error("%s, aborting.", msg); #ifdef FLB_HAVE_AWS_ERROR_REPORTER if (is_error_reporting_enabled()) { flb_aws_error_reporter_write(error_reporter, msg); diff --git a/src/fluent-bit.c b/src/fluent-bit.c index 76103050b33..088ab61b614 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -53,6 +53,8 @@ #include #include #include +#include +#include #ifdef FLB_HAVE_MTRACE #include @@ -419,30 +421,53 @@ static void help_format_text(void *help_buf, size_t help_size) static void flb_help_plugin(int rc, int format, struct flb_config *config, int type, - struct flb_input_instance *in, - struct flb_filter_instance *filter, - struct flb_output_instance *out) - - + struct flb_cf *cf, + struct flb_cf_section *s) { - struct flb_config_map *m = NULL; struct flb_config_map *opt = NULL; void *help_buf; size_t help_size; + char *name; + struct flb_input_instance *i = NULL; + struct flb_filter_instance *f = NULL; + struct flb_output_instance *o = NULL; flb_banner(); + name = flb_cf_section_property_get(cf, s, "name"); + if (!name) { + exit(EXIT_FAILURE); + } + if (type == PLUGIN_INPUT) { - opt = in->p->config_map; - flb_help_input(in, &help_buf, &help_size); + i = flb_input_new(config, name, 0, FLB_TRUE); + if (!i) { + fprintf(stderr, "invalid input plugin '%s'", name); + return; + } + opt = i->p->config_map; + flb_help_input(i, &help_buf, &help_size); + flb_input_instance_destroy(i); } else if (type == PLUGIN_FILTER) { - opt = filter->p->config_map; - flb_help_filter(filter, &help_buf, &help_size); + f = flb_filter_new(config, name, 0); + if (!f) { + fprintf(stderr, "invalid filter plugin '%s'", name); + return; + } + opt = f->p->config_map; + flb_help_filter(f, &help_buf, &help_size); + flb_filter_instance_destroy(f); } else if (type == PLUGIN_OUTPUT) { - opt = m = out->p->config_map; - flb_help_output(out, &help_buf, &help_size); + o = flb_output_new(config, name, 0, FLB_TRUE); + if (!o) { + fprintf(stderr, "invalid output plugin '%s'", name); + return; + } + opt = o->p->config_map; + flb_help_output(o, &help_buf, &help_size); + flb_output_instance_destroy(o); } if (!opt) { @@ -455,8 +480,8 @@ static void flb_help_plugin(int rc, int format, else if (format == FLB_HELP_JSON) { help_format_json(help_buf, help_size); } - flb_free(help_buf); + flb_free(help_buf); exit(rc); } @@ -579,13 +604,13 @@ static void flb_signal_init() signal(SIGFPE, &flb_signal_handler); } -static int custom_set_property(struct flb_custom_instance *in, char *kv) +static int set_property(struct flb_cf *cf, struct flb_cf_section *s, char *kv) { - int ret; int len; int sep; char *key; char *value; + struct flb_kv *tmp; len = strlen(kv); sep = mk_string_char_search(kv, '=', len); @@ -600,102 +625,13 @@ static int custom_set_property(struct flb_custom_instance *in, char *kv) return -1; } - ret = flb_custom_set_property(in, key, value); - if (ret == -1) { - fprintf(stderr, "[error] setting up '%s' plugin property '%s'\n", - in->p->name, key); + tmp = flb_cf_property_add(cf, &s->properties, key, 0, value, 0); + if (!tmp) { + fprintf(stderr, "[error] setting up section '%s' plugin property '%s'\n", + s->name, key); } - mk_mem_free(key); - return ret; -} - -static int input_set_property(struct flb_input_instance *in, char *kv) -{ - int ret; - int len; - int sep; - char *key; - char *value; - - len = strlen(kv); - sep = mk_string_char_search(kv, '=', len); - if (sep == -1) { - return -1; - } - - key = mk_string_copy_substr(kv, 0, sep); - value = kv + sep + 1; - - if (!key) { - return -1; - } - - ret = flb_input_set_property(in, key, value); - if (ret == -1) { - fprintf(stderr, "[error] setting up '%s' plugin property '%s'\n", - in->p->name, key); - } - - mk_mem_free(key); - return ret; -} - -static int output_set_property(struct flb_output_instance *out, char *kv) -{ - int ret; - int len; - int sep; - char *key; - char *value; - len = strlen(kv); - sep = mk_string_char_search(kv, '=', len); - if (sep == -1) { - return -1; - } - - key = mk_string_copy_substr(kv, 0, sep); - value = kv + sep + 1; - - if (!key) { - return -1; - } - - ret = flb_output_set_property(out, key, value); - mk_mem_free(key); - return ret; -} - -static int filter_set_property(struct flb_filter_instance *filter, char *kv) -{ - int ret; - int len; - int sep; - char *key; - char *value; - - len = strlen(kv); - sep = mk_string_char_search(kv, '=', len); - if (sep == -1) { - return -1; - } - - key = mk_string_copy_substr(kv, 0, sep); - value = kv + sep + 1; - - if (!key) { - return -1; - } - - ret = flb_filter_set_property(filter, key, value); - mk_mem_free(key); - return ret; -} - -static void flb_service_conf_err(struct mk_rconf_section *section, char *key) -{ - fprintf(stderr, "Invalid configuration value at %s.%s\n", - section->name, key); + return 0; } static int flb_service_conf_path_set(struct flb_config *config, char *file) @@ -723,247 +659,212 @@ static int flb_service_conf_path_set(struct flb_config *config, char *file) return 0; } -static int flb_service_conf(struct flb_config *config, char *file) +static int service_configure_plugin(struct flb_config *config, + struct flb_cf *cf, enum section_type type) { - int ret = -1; + int ret; char *tmp; char *name; + char *s_type; + struct mk_list *list; struct mk_list *head; struct mk_list *h_prop; - struct mk_rconf *fconf = NULL; - struct mk_rconf_entry *entry; - struct mk_rconf_section *section; - struct flb_custom_instance *custom; - struct flb_input_instance *in; - struct flb_output_instance *out; - struct flb_filter_instance *filter; - -#ifdef FLB_HAVE_STATIC_CONF - fconf = flb_config_static_open(file); -#else - fconf = mk_rconf_open(file); -#endif + struct flb_kv *kv; + struct flb_cf_section *s; + void *ins; - if (!fconf) { - return -1; + if (type == FLB_CF_CUSTOM) { + s_type = "custom"; + list = &cf->customs; } - - /* Process all meta commands */ - mk_list_foreach(head, &fconf->metas) { - entry = mk_list_entry(head, struct mk_rconf_entry, _head); - flb_meta_run(config, entry->key, entry->val); + else if (type == FLB_CF_INPUT) { + s_type = "input"; + list = &cf->inputs; } - - /* Set configuration root path */ - flb_service_conf_path_set(config, file); - - /* Validate sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - - if (strcasecmp(section->name, "SERVICE") == 0 || - strcasecmp(section->name, "CUSTOM") == 0 || - strcasecmp(section->name, "INPUT") == 0 || - strcasecmp(section->name, "FILTER") == 0 || - strcasecmp(section->name, "OUTPUT") == 0) { - - /* continue on valid sections */ - continue; - } - - /* Extra sanity checks */ - if (strcasecmp(section->name, "PARSER") == 0 || - strcasecmp(section->name, "MULTILINE_PARSER") == 0) { - fprintf(stderr, - "Sections [MULTILINE_PARSER] and [PARSER] are not valid in " - "the main configuration file. It belongs to \n" - "the 'parsers_file' configuration files.\n"); - } - else { - fprintf(stderr, - "Error: unexpected section [%s] in the main " - "configuration file.\n", section->name); - } - exit(EXIT_FAILURE); + else if (type == FLB_CF_FILTER) { + s_type = "filter"; + list = &cf->filters; } - - /* Read main [SERVICE] section */ - section = mk_rconf_section_get(fconf, "SERVICE"); - if (section) { - /* Iterate properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - /* Set the property */ - flb_config_set_property(config, entry->key, entry->val); - } + else if (type == FLB_CF_OUTPUT) { + s_type = "output"; + list = &cf->outputs; + } + else { + return -1; } - /* Read all [CUSTOM] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "CUSTOM") != 0) { - continue; - } - - /* Get the input plugin name */ - name = s_get_key(section, "name", MK_RCONF_STR); + mk_list_foreach(head, list) { + s = mk_list_entry(head, struct flb_cf_section, _head_section); + name = flb_cf_section_property_get(cf, s, "name"); if (!name) { - flb_service_conf_err(section, "name"); - goto flb_service_conf_end; + flb_error("[config] section '%s' is missing the 'name' property", + s_type); + return -1; } - flb_debug("[service] loading custom plugin: %s", name); - - /* Create an instace of the plugin */ + /* translate the variable */ tmp = flb_env_var_translate(config->env, name); - custom = flb_custom_new(config, tmp, NULL); - mk_mem_free(name); - if (!custom) { - fprintf(stderr, "Custom plugin '%s' cannot be loaded\n", tmp); - flb_sds_destroy(tmp); - goto flb_service_conf_end; + + /* create an instance of the plugin */ + ins = NULL; + if (type == FLB_CF_CUSTOM) { + ins = flb_custom_new(config, tmp, NULL); + } + else if (type == FLB_CF_INPUT) { + ins = flb_input_new(config, tmp, NULL, FLB_TRUE); + } + else if (type == FLB_CF_FILTER) { + ins = flb_filter_new(config, tmp, NULL); + } + else if (type == FLB_CF_OUTPUT) { + ins = flb_output_new(config, tmp, NULL, FLB_TRUE); } flb_sds_destroy(tmp); - /* Iterate other properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "name") == 0) { + /* validate the instance creation */ + if (!ins) { + flb_error("[config] section '%s' tried to instance a plugin name " + "that don't exists", name); + return -1; + } + + /* + * iterate section properties and populate instance by using specific + * api function. + */ + mk_list_foreach(h_prop, &s->properties) { + kv = mk_list_entry(h_prop, struct flb_kv, _head); + if (strcasecmp(kv->key, "name") == 0) { continue; } - /* Set the property */ - ret = flb_custom_set_property(custom, entry->key, entry->val); + ret = -1; + if (type == FLB_CF_CUSTOM) { + ret = flb_custom_set_property(ins, kv->key, kv->val); + } + else if (type == FLB_CF_INPUT) { + ret = flb_input_set_property(ins, kv->key, kv->val); + } + else if (type == FLB_CF_FILTER) { + ret = flb_filter_set_property(ins, kv->key, kv->val); + } + else if (type == FLB_CF_OUTPUT) { + ret = flb_output_set_property(ins, kv->key, kv->val); + } + if (ret == -1) { - fprintf(stderr, "Error setting up %s plugin property '%s'\n", - custom->name, entry->key); - goto flb_service_conf_end; + flb_error("[config] could not configure property '%s' on " + "%s plugin with section name '%s'", + kv->key, s_type, name); } } } - /* Read all [INPUT] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "INPUT") != 0) { - continue; - } + return 0; +} - /* Get the input plugin name */ - name = s_get_key(section, "Name", MK_RCONF_STR); - if (!name) { - flb_service_conf_err(section, "Name"); - goto flb_service_conf_end; - } +static struct flb_cf *service_configure(struct flb_cf *cf, + struct flb_config *config, char *file) +{ + int ret = -1; + struct flb_cf_section *s; + struct flb_kv *kv; + struct mk_list *head; - flb_debug("[service] loading input: %s", name); +#ifdef FLB_HAVE_STATIC_CONF + cf = flb_config_static_open(file); +#else + if (file) { + cf = flb_cf_create_from_file(cf, file); + } +#endif - /* Create an instace of the plugin */ - tmp = flb_env_var_translate(config->env, name); - in = flb_input_new(config, tmp, NULL, FLB_TRUE); - mk_mem_free(name); - if (!in) { - fprintf(stderr, "Input plugin '%s' cannot be loaded\n", tmp); - flb_sds_destroy(tmp); - goto flb_service_conf_end; - } - flb_sds_destroy(tmp); + if (!cf) { + return NULL; + } - /* Iterate other properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "Name") == 0) { - continue; - } + config->cf_main = cf; - /* Set the property */ - ret = flb_input_set_property(in, entry->key, entry->val); - if (ret == -1) { - fprintf(stderr, "Error setting up %s plugin property '%s'\n", - in->name, entry->key); - goto flb_service_conf_end; - } - } + /* Set configuration root path */ + if (file) { + flb_service_conf_path_set(config, file); } - /* Read all [OUTPUT] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "OUTPUT") != 0) { - continue; - } - - /* Get the output plugin name */ - name = s_get_key(section, "Name", MK_RCONF_STR); - if (!name) { - flb_service_conf_err(section, "Name"); - goto flb_service_conf_end; + /* Process config environment vars */ + mk_list_foreach(head, &cf->env) { + kv = mk_list_entry(head, struct flb_kv, _head); + ret = flb_env_set(config->env, kv->key, kv->val); + if (ret == -1) { + fprintf(stderr, "could not set config environment variable '%s'\n", + kv->key); + exit(EXIT_FAILURE); } + } - /* Create an instace of the plugin */ - tmp = flb_env_var_translate(config->env, name); - out = flb_output_new(config, tmp, NULL, FLB_TRUE); - mk_mem_free(name); - if (!out) { - fprintf(stderr, "Output plugin '%s' cannot be loaded\n", tmp); - flb_sds_destroy(tmp); - goto flb_service_conf_end; - } - flb_sds_destroy(tmp); + /* Process all meta commands */ + mk_list_foreach(head, &cf->metas) { + kv = mk_list_entry(head, struct flb_kv, _head); + flb_meta_run(config, kv->key, kv->val); + } - /* Iterate other properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "Name") == 0) { - continue; - } + /* Validate sections */ + mk_list_foreach(head, &cf->sections) { + s = mk_list_entry(head, struct flb_cf_section, _head); - /* Set the property */ - flb_output_set_property(out, entry->key, entry->val); - } - } + if (strcasecmp(s->name, "env") == 0 || + strcasecmp(s->name, "service") == 0 || + strcasecmp(s->name, "custom") == 0 || + strcasecmp(s->name, "input") == 0 || + strcasecmp(s->name, "filter") == 0 || + strcasecmp(s->name, "output") == 0) { - /* Read all [FILTER] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "FILTER") != 0) { + /* continue on valid sections */ continue; } - /* Get the filter plugin name */ - name = s_get_key(section, "Name", MK_RCONF_STR); - if (!name) { - flb_service_conf_err(section, "Name"); - goto flb_service_conf_end; - } - /* Create an instace of the plugin */ - tmp = flb_env_var_translate(config->env, name); - filter = flb_filter_new(config, tmp, NULL); - flb_sds_destroy(tmp); - mk_mem_free(name); - if (!filter) { - flb_service_conf_err(section, "Name"); - goto flb_service_conf_end; - } - /* Iterate other properties */ - mk_list_foreach(h_prop, §ion->entries) { - entry = mk_list_entry(h_prop, struct mk_rconf_entry, _head); - if (strcasecmp(entry->key, "Name") == 0) { - continue; - } + /* Extra sanity checks */ + if (strcasecmp(s->name, "parser") == 0 || + strcasecmp(s->name, "multiline_parser") == 0) { + fprintf(stderr, + "Sections 'multiline_parser' and 'parser' are not valid in " + "the main configuration file. It belongs to \n" + "the 'parsers_file' configuration files.\n"); + exit(EXIT_FAILURE); + } + } - /* Set the property */ - flb_filter_set_property(filter, entry->key, entry->val); + /* Read main 'service' section */ + s = cf->service; + if (s) { + /* Iterate properties */ + mk_list_foreach(head, &s->properties) { + kv = mk_list_entry(head, struct flb_kv, _head); + flb_config_set_property(config, kv->key, kv->val); } } - ret = 0; + ret = service_configure_plugin(config, cf, FLB_CF_CUSTOM); + if (ret == -1) { + goto error; + } -flb_service_conf_end: - if (fconf != NULL) { - mk_rconf_free(fconf); + ret = service_configure_plugin(config, cf, FLB_CF_INPUT); + if (ret == -1) { + goto error; } - return ret; + ret = service_configure_plugin(config, cf, FLB_CF_FILTER); + if (ret == -1) { + goto error; + } + ret = service_configure_plugin(config, cf, FLB_CF_OUTPUT); + if (ret == -1) { + goto error; + } + + return cf; + +error: + return NULL; } int flb_main(int argc, char **argv) @@ -976,10 +877,12 @@ int flb_main(int argc, char **argv) /* local variables to handle config options */ char *cfg_file = NULL; - struct flb_custom_instance *custom = NULL; - struct flb_input_instance *in = NULL; - struct flb_output_instance *out = NULL; - struct flb_filter_instance *filter = NULL; + + /* config format context */ + struct flb_cf *cf; + struct flb_cf *tmp; + struct flb_cf_section *service; + struct flb_cf_section *s; #ifdef FLB_HAVE_LIBBACKTRACE flb_stacktrace_init(argv[0], &flb_st); @@ -1039,6 +942,8 @@ int flb_main(int argc, char **argv) exit(EXIT_FAILURE); } config = ctx->config; + cf = config->cf_main; + service = cf->service; #ifndef FLB_HAVE_STATIC_CONF @@ -1050,13 +955,16 @@ int flb_main(int argc, char **argv) switch (opt) { case 'b': - config->storage_path = flb_strdup(optarg); + flb_cf_property_add(cf, &service->properties, + "storage.path", 0, optarg, 0); break; case 'c': cfg_file = flb_strdup(optarg); break; #ifdef FLB_HAVE_FORK case 'd': + flb_cf_property_add(cf, &service->properties, + "daemon", 0, "on", 0); config->daemon = FLB_TRUE; break; #endif @@ -1070,35 +978,36 @@ int flb_main(int argc, char **argv) } break; case 'f': - config->flush = atof(optarg); + flb_cf_property_add(cf, &service->properties, + "flush", 0, optarg, 0); break; case 'C': - custom = flb_custom_new(config, optarg, NULL); - if (!custom) { + s = flb_cf_section_create(cf, "custom", 0); + if (!s) { flb_utils_error(FLB_ERR_CUSTOM_INVALID); } + flb_cf_property_add(cf, &s->properties, "name", 0, optarg, 0); last_plugin = PLUGIN_CUSTOM; break; case 'i': - in = flb_input_new(config, optarg, NULL, FLB_TRUE); - if (!in) { + s = flb_cf_section_create(cf, "input", 0); + if (!s) { flb_utils_error(FLB_ERR_INPUT_INVALID); } + flb_cf_property_add(cf, &s->properties, "name", 0, optarg, 0); last_plugin = PLUGIN_INPUT; break; case 'm': - if (last_plugin == PLUGIN_FILTER) { - flb_filter_set_property(filter, "match", optarg); - } - else if (last_plugin == PLUGIN_OUTPUT) { - flb_output_set_property(out, "match", optarg); + if (last_plugin == PLUGIN_FILTER || last_plugin == PLUGIN_OUTPUT) { + flb_cf_property_add(cf, &s->properties, "match", 0, optarg, 0); } break; case 'o': - out = flb_output_new(config, optarg, NULL, FLB_TRUE); - if (!out) { + s = flb_cf_section_create(cf, "output", 0); + if (!s) { flb_utils_error(FLB_ERR_OUTPUT_INVALID); } + flb_cf_property_add(cf, &s->properties, "name", 0, optarg, 0); last_plugin = PLUGIN_OUTPUT; break; #ifdef FLB_HAVE_PARSER @@ -1110,35 +1019,25 @@ int flb_main(int argc, char **argv) break; #endif case 'F': - filter = flb_filter_new(config, optarg, NULL); - if (!filter) { + s = flb_cf_section_create(cf, "filter", 0); + if (!s) { flb_utils_error(FLB_ERR_FILTER_INVALID); } + flb_cf_property_add(cf, &s->properties, "name", 0, optarg, 0); last_plugin = PLUGIN_FILTER; break; case 'l': - config->log_file = flb_strdup(optarg); + flb_cf_property_add(cf, &service->properties, + "log_file", 0, optarg, 0); break; case 'p': - if (last_plugin == PLUGIN_INPUT) { - ret = input_set_property(in, optarg); - if (ret != 0) { - exit(EXIT_FAILURE); - } - } - else if (last_plugin == PLUGIN_OUTPUT) { - output_set_property(out, optarg); - } - else if (last_plugin == PLUGIN_FILTER) { - filter_set_property(filter, optarg); - } - else if (last_plugin == PLUGIN_CUSTOM) { - custom_set_property(custom, optarg); + if (s) { + set_property(cf, s, optarg); } break; case 't': - if (in) { - flb_input_set_property(in, "tag", optarg); + if (s) { + flb_cf_property_add(cf, &s->properties, "tag", 0, optarg, 0); } break; #ifdef FLB_HAVE_STREAM_PROCESSOR @@ -1153,7 +1052,7 @@ int flb_main(int argc, char **argv) else { flb_help_plugin(EXIT_SUCCESS, FLB_HELP_TEXT, config, - last_plugin, in, filter, out); + last_plugin, cf, s); } break; case 'J': @@ -1162,12 +1061,12 @@ int flb_main(int argc, char **argv) } else { flb_help_plugin(EXIT_SUCCESS, FLB_HELP_JSON, config, - last_plugin, in, filter, out); + last_plugin, cf, s); } break; #ifdef FLB_HAVE_HTTP_SERVER case 'H': - config->http_server = FLB_TRUE; + flb_cf_property_add(cf, &service->properties, "http_server", 0, "on", 0); break; case 'L': if (config->http_listen) { @@ -1226,25 +1125,33 @@ int flb_main(int argc, char **argv) /* Validate config file */ #ifndef FLB_HAVE_STATIC_CONF + if (cfg_file) { if (access(cfg_file, R_OK) != 0) { + flb_free(cfg_file); flb_utils_error(FLB_ERR_CFG_FILE); } + } - /* Load the service configuration file */ - ret = flb_service_conf(config, cfg_file); - if (ret != 0) { - flb_utils_error(FLB_ERR_CFG_FILE_STOP); - } - flb_free(cfg_file); + /* Load the service configuration file */ + tmp = service_configure(cf, config, cfg_file); + flb_free(cfg_file); + if (!tmp) { + flb_utils_error(FLB_ERR_CFG_FILE_STOP); } #else - ret = flb_service_conf(config, "fluent-bit.conf"); - if (ret != 0) { + tmp = service_configure(cf, config, "fluent-bit.conf"); + if (!tmp) { flb_utils_error(FLB_ERR_CFG_FILE_STOP); } + + /* destroy previous context and override */ + flb_cf_destroy(cf); + config->cf_main = tmp; + cf = tmp; #endif + /* Check co-routine stack size */ if (config->coro_stack_size < getpagesize()) { flb_utils_error(FLB_ERR_CORO_STACK_SIZE); @@ -1285,12 +1192,14 @@ int flb_main(int argc, char **argv) while (ctx->status == FLB_LIB_OK && exit_signal == 0) { sleep(1); } + if (exit_signal) { flb_signal_exit(exit_signal); } ret = config->exit_status_code; - flb_destroy(ctx); + flb_stop(ctx); + flb_destroy(ctx); return ret; } diff --git a/src/stream_processor/flb_sp.c b/src/stream_processor/flb_sp.c index 18ee3cb8a40..6cd82efdabf 100644 --- a/src/stream_processor/flb_sp.c +++ b/src/stream_processor/flb_sp.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -64,10 +65,10 @@ static int sp_config_file(struct flb_config *config, struct flb_sp *sp, const char *cfg = NULL; char tmp[PATH_MAX + 1]; struct stat st; - struct mk_rconf *fconf; - struct mk_rconf_section *section; struct mk_list *head; struct flb_sp_task *task; + struct flb_cf *cf; + struct flb_cf_section *section; #ifndef FLB_HAVE_STATIC_CONF ret = stat(file, &st); @@ -87,34 +88,34 @@ static int sp_config_file(struct flb_config *config, struct flb_sp *sp, cfg = file; } - fconf = mk_rconf_open(cfg); + cf = flb_cf_create_from_file(NULL, cfg); #else - fconf = flb_config_static_open(file); + cf = flb_config_static_open(file); #endif - if (!fconf) { + if (!cf) { return -1; } - /* Read all [STREAM_TASK] sections */ - mk_list_foreach(head, &fconf->sections) { - section = mk_list_entry(head, struct mk_rconf_section, _head); - if (strcasecmp(section->name, "STREAM_TASK") != 0) { + /* Read all 'stream_task' sections */ + mk_list_foreach(head, &cf->sections) { + section = mk_list_entry(head, struct flb_cf_section, _head); + if (strcasecmp(section->name, "stream_task") != 0) { continue; } name = NULL; exec = NULL; - /* Name */ - name = mk_rconf_section_get_key(section, "Name", MK_RCONF_STR); + /* name */ + name = flb_cf_section_property_get(cf, section, "name"); if (!name) { flb_error("[sp] task 'name' not found in file '%s'", cfg); goto fconf_error; } - /* Exec */ - exec = mk_rconf_section_get_key(section, "Exec", MK_RCONF_STR); + /* exec */ + exec = flb_cf_section_property_get(cf, section, "exec"); if (!exec) { flb_error("[sp] task '%s' don't have an 'exec' command", name); goto fconf_error; @@ -125,18 +126,13 @@ static int sp_config_file(struct flb_config *config, struct flb_sp *sp, if (!task) { goto fconf_error; } - - flb_free(name); - flb_free(exec); } - mk_rconf_free(fconf); + flb_cf_destroy(cf); return 0; fconf_error: - flb_free(name); - flb_free(exec); - + flb_cf_destroy(cf); return -1; } diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 5075b4a4aee..194d57c1147 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -24,6 +24,20 @@ set(UNIT_TESTS_FILES typecast.c ) +# Config format +set(UNIT_TESTS_FILES + ${UNIT_TESTS_FILES} + config_format.c + config_format_fluentbit.c +) + +if(FLB_HAVE_LIBYAML) + set(UNIT_TESTS_FILES + ${UNIT_TESTS_FILES} + config_format_yaml.c + ) +endif() + if (NOT WIN32) set(UNIT_TESTS_FILES ${UNIT_TESTS_FILES} diff --git a/tests/internal/config_format.c b/tests/internal/config_format.c new file mode 100644 index 00000000000..53660d692fc --- /dev/null +++ b/tests/internal/config_format.c @@ -0,0 +1,90 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include + +#include "flb_tests_internal.h" + +void test_api() +{ + struct flb_cf *cf; + struct flb_cf_section *s_tmp; + struct flb_cf_section *service; + struct flb_cf_group *g_tmp; + struct flb_kv *meta; + struct flb_kv *kv; + + /* create context */ + cf = flb_cf_create(); + TEST_CHECK(cf != NULL); + + /* create service section */ + service = flb_cf_section_create(cf, "SERVICE", 7); + TEST_CHECK(service != NULL); + + /* add a property */ + kv = flb_cf_property_add(cf, &service->properties, "key", 3, "val", 3); + TEST_CHECK(kv != NULL); + + /* add a property with empty spaces on left/right */ + kv = flb_cf_property_add(cf, &service->properties, " key ", 5, " val ", 7); + TEST_CHECK(kv != NULL); + + /* property: check key */ + TEST_CHECK(flb_sds_len(kv->key) == 3); + TEST_CHECK(strcmp(kv->key, "key") == 0); + + /* property: check val */ + TEST_CHECK(flb_sds_len(kv->key) == 3); + TEST_CHECK(strcmp(kv->key, "key") == 0); + + /* add an invalid property */ + kv = flb_cf_property_add(cf, &service->properties, " ", 3, "", 0); + TEST_CHECK(kv == NULL); + + /* try to add another 'SERVICE' section, it should return the same one */ + s_tmp = flb_cf_section_create(cf, "SERVICE", 7); + TEST_CHECK(s_tmp == service); + + /* add a valid section */ + s_tmp = flb_cf_section_create(cf, "INPUT", 5); + TEST_CHECK(s_tmp != NULL); + + TEST_CHECK(mk_list_size(&cf->inputs) == 1); + + /* add property to the section recently created */ + kv = flb_cf_property_add(cf, &s_tmp->properties, "key", 3, "val", 3); + TEST_CHECK(kv != NULL); + + /* groups: add groups to the last section created */ + g_tmp = flb_cf_group_create(cf, s_tmp, "FLUENT GROUP", 12); + TEST_CHECK(g_tmp != NULL); + + /* add properties to the group */ + kv = flb_cf_property_add(cf, &g_tmp->properties, "key", 3, "val", 3); + TEST_CHECK(kv != NULL); + + /* groups: invalid group */ + g_tmp = flb_cf_group_create(cf, s_tmp, "", 0); + TEST_CHECK(g_tmp == NULL); + + /* Meta commands */ + meta = flb_cf_meta_create(cf, "@SET a=1 ", 20); + TEST_CHECK(meta != NULL); + TEST_CHECK(flb_sds_len(meta->key) == 3 && strcmp(meta->key, "SET") == 0); + TEST_CHECK(flb_sds_len(meta->val) == 3 && strcmp(meta->val, "a=1") == 0); + + /* invalid meta */ + meta = flb_cf_meta_create(cf, "@a=1 ", 5); + TEST_CHECK(meta == NULL); + + /* destroy context */ + flb_cf_destroy(cf); +} + +TEST_LIST = { + { "api" , test_api}, + { 0 } +}; diff --git a/tests/internal/config_format_fluentbit.c b/tests/internal/config_format_fluentbit.c new file mode 100644 index 00000000000..99d07fd9c09 --- /dev/null +++ b/tests/internal/config_format_fluentbit.c @@ -0,0 +1,63 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include + +#include "flb_tests_internal.h" + +#define FLB_000 FLB_TESTS_DATA_PATH "/data/config_format/fluent-bit.conf" + +/* data/config_format/fluent-bit.conf */ +void test_basic() +{ + struct mk_list *head; + struct flb_cf *cf; + struct flb_cf_section *s; + struct flb_cf_group *g; + + cf = flb_cf_fluentbit_create(NULL, FLB_000, NULL, 0); + TEST_CHECK(cf != NULL); + + /* Total number of sections */ + TEST_CHECK(mk_list_size(&cf->sections) == 8); + + /* SERVICE check */ + TEST_CHECK(cf->service != NULL); + if (cf->service) { + TEST_CHECK(mk_list_size(&cf->service->properties) == 3); + } + + /* Meta commands */ + TEST_CHECK(mk_list_size(&cf->metas) == 2); + + /* Check number sections per list */ + TEST_CHECK(mk_list_size(&cf->parsers) == 1); + TEST_CHECK(mk_list_size(&cf->multiline_parsers) == 1); + TEST_CHECK(mk_list_size(&cf->customs) == 1); + TEST_CHECK(mk_list_size(&cf->inputs) == 1); + TEST_CHECK(mk_list_size(&cf->filters) == 1); + TEST_CHECK(mk_list_size(&cf->outputs) == 1); + TEST_CHECK(mk_list_size(&cf->others) == 1); + + /* groups */ + s = flb_cf_section_get_by_name(cf, "input"); + TEST_CHECK(s != NULL); + TEST_CHECK(mk_list_size(&s->groups) == 2); + + mk_list_foreach(head, &s->groups) { + g = mk_list_entry(head, struct flb_cf_group, _head); + TEST_CHECK(mk_list_size(&g->properties) == 2); + } + + printf("\n"); + flb_cf_dump(cf); + + flb_cf_destroy(cf); +} + +TEST_LIST = { + { "basic" , test_basic}, + { 0 } +}; diff --git a/tests/internal/config_format_yaml.c b/tests/internal/config_format_yaml.c new file mode 100644 index 00000000000..77efd4904f9 --- /dev/null +++ b/tests/internal/config_format_yaml.c @@ -0,0 +1,64 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include + +#include "flb_tests_internal.h" + +#define FLB_000 FLB_TESTS_DATA_PATH "/data/config_format/fluent-bit.yaml" + +/* data/config_format/fluent-bit.yaml */ +void test_basic() +{ + struct mk_list *head; + struct flb_cf *cf; + struct flb_cf_section *s; + struct flb_cf_group *g; + + cf = flb_cf_yaml_create(NULL, FLB_000, NULL, 0); + TEST_CHECK(cf != NULL); + if (!cf) { + exit(EXIT_FAILURE); + } + + /* Total number of sections */ + TEST_CHECK(mk_list_size(&cf->sections) == 10); + + /* SERVICE check */ + TEST_CHECK(cf->service != NULL); + if (cf->service) { + TEST_CHECK(mk_list_size(&cf->service->properties) == 3); + } + + /* Check number sections per list */ + TEST_CHECK(mk_list_size(&cf->parsers) == 0); + TEST_CHECK(mk_list_size(&cf->multiline_parsers) == 0); + TEST_CHECK(mk_list_size(&cf->customs) == 1); + TEST_CHECK(mk_list_size(&cf->inputs) == 4); + TEST_CHECK(mk_list_size(&cf->filters) == 1); + TEST_CHECK(mk_list_size(&cf->outputs) == 2); + TEST_CHECK(mk_list_size(&cf->others) == 1); + + /* groups */ + s = flb_cf_section_get_by_name(cf, "input"); + TEST_CHECK(s != NULL); + TEST_CHECK(mk_list_size(&s->groups) == 1); + + mk_list_foreach(head, &s->groups) { + g = mk_list_entry(head, struct flb_cf_group, _head); + TEST_CHECK(mk_list_size(&g->properties) == 2); + } + + printf("\n"); + flb_cf_dump(cf); + flb_cf_destroy(cf); + + +} + +TEST_LIST = { + { "basic" , test_basic}, + { 0 } +}; diff --git a/tests/internal/data/config_format/fluent-bit.conf b/tests/internal/data/config_format/fluent-bit.conf new file mode 100644 index 00000000000..72d6c18cd90 --- /dev/null +++ b/tests/internal/data/config_format/fluent-bit.conf @@ -0,0 +1,39 @@ +@SET a=1 +@SET b=2 + +[SERVICE] + flush 1 + log_level info + http_server on + +[PARSER] + name test_api + +[MULTILINE_PARSER] + name abc + +[CUSTOM] + name calyptia + +[INPUT] + name tail + path /var/log/containers/*.log + + [GROUP 1] + key1 aa + key2 bb + + [GROUP 2] + key3 cc + key4 dd + +[FILTER] + name stdout + match * + +[OUTPUT] + name null + match * + +[UNKNOWN] + name blah diff --git a/tests/internal/data/config_format/fluent-bit.yaml b/tests/internal/data/config_format/fluent-bit.yaml new file mode 100644 index 00000000000..61e5a94c6f3 --- /dev/null +++ b/tests/internal/data/config_format/fluent-bit.yaml @@ -0,0 +1,60 @@ +# environment variable definition +env: + flush_interval: 1 + +service: + flush: ${flush_interval} + log_level: info + http_server: on + +pipeline: + inputs: + dummy: + samples: 1 + tag: dummy_tag + group1: + key1: aa + key2: bb + +pipeline: + customs: + calyptia: + api_key: abcdefg + + inputs: + tail: + path: "/var/log/containers/*.log" + parser: docker + group1: + key1: aa + key2: bb + group2: + key3: cc + key4: dd + + tail: + path: "/var/log/messages" + parser: syslog + + forward: + listen: 0.0.0.0 + + filters: + grep: + exclude: $kubernetes['labels']['app'] myapp + + outputs: + stdout: + match: "*" + retry_limit: false + + http: + match: "*" + host: 192.168.3.2 + port: 9444 + tls: on + tls.verify: off + retry_limit: false + +unknown: + a: b diff --git a/tests/internal/fuzzers/config_fuzzer.c b/tests/internal/fuzzers/config_fuzzer.c index 300fe8581c8..4288e80b778 100644 --- a/tests/internal/fuzzers/config_fuzzer.c +++ b/tests/internal/fuzzers/config_fuzzer.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "flb_fuzz_header.h" /* A sample of configurations */