diff --git a/CMakeLists.txt b/CMakeLists.txt index 99109309405..26842d80cbe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,12 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux") add_definitions(-DFLB_SYSTEM_LINUX) endif() +# Define macro to identify Android system +if(CMAKE_SYSTEM_NAME MATCHES "Android") + set(FLB_SYSTEM_ANDROID On) + add_definitions(-DFLB_SYSTEM_ANDROID) +endif() + # Update CFLAGS if (MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS) @@ -621,6 +627,23 @@ if(FLB_METRICS) FLB_DEFINITION(FLB_HAVE_METRICS) endif() +# Solaris detection +check_c_source_compiles(" + int main() { +#if !defined(__SVR4) || \ + !defined (__sun) + THIS SHOULD + CAUSE THIS CODE + NOT TO COMPILE +#endif + return 0; + }" FLB_SYSTEM_SOLARIS) + +if (FLB_SYSTEM_SOLARIS) + set(FLB_SYSTEM_SOLARIS On) + add_definitions(-DFLB_SYSTEM_SOLARIS) +endif() + # WASM if(FLB_WASM) if (FLB_SYSTEM_LINUX) diff --git a/include/fluent-bit/flb_file.h b/include/fluent-bit/flb_file.h index 0fbb357bcfb..61eb6acbe30 100644 --- a/include/fluent-bit/flb_file.h +++ b/include/fluent-bit/flb_file.h @@ -22,8 +22,81 @@ #define FLB_FILE_H #include +#include +#include -flb_sds_t flb_file_read(const char *path); -// TODO int flb_file_write(const char *path, flb_sds_t contents); +#ifdef FLB_SYSTEM_WINDOWS +#include +#else +#include +#endif + +#define FLB_FILE_GLOB_ABORT_ON_ERROR (((uint64_t) 1) << 0) +#define FLB_FILE_GLOB_MARK_DIRECTORIES (((uint64_t) 1) << 1) +#define FLB_FILE_GLOB_DO_NOT_SORT (((uint64_t) 1) << 2) +#define FLB_FILE_GLOB_EXPAND_TILDE (((uint64_t) 1) << 3) + +#define FLB_FILE_GLOB_ERROR_SUCCESS 0 +#define FLB_FILE_GLOB_ERROR_ABORTED 1 +#define FLB_FILE_GLOB_ERROR_NO_MEMORY 2 +#define FLB_FILE_GLOB_ERROR_NO_FILE 3 +#define FLB_FILE_GLOB_ERROR_NO_ACCESS 4 +#define FLB_FILE_GLOB_ERROR_NO_MATCHES 5 +#define FLB_FILE_GLOB_ERROR_NO_MORE_RESULTS 6 +#define FLB_FILE_GLOB_ERROR_OVERSIZED_PATH 7 +#define FLB_FILE_GLOB_ERROR_INVALID_ARGUMENT 8 + +struct flb_file_glob_inner_context; + +struct flb_file_glob_context { + struct flb_file_glob_inner_context *inner_context; + uint64_t flags; + char *path; +}; + +struct flb_file_stat { + uint64_t device; + uint64_t inode; + uint16_t mode; + int64_t modification_time; + int16_t hard_link_count; + int64_t size; +}; + +int flb_file_glob_start(struct flb_file_glob_context *context, + const char *path, + uint64_t flags); + +void flb_file_glob_clean(struct flb_file_glob_context *context); + +int flb_file_glob_fetch(struct flb_file_glob_context *context, + char **result); + +flb_file_handle flb_file_open(const char *path, + unsigned int flags); + +void flb_file_close(flb_file_handle handle); + +ssize_t flb_file_read(flb_file_handle handle, + void *output_buffer, + size_t byte_count); + +int64_t flb_file_lseek(flb_file_handle handle, + int64_t offset, + int reference_point); + +int flb_file_stat(const char *path, + struct flb_file_stat *output_buffer); + +int flb_file_lstat(const char *path, + struct flb_file_stat *output_buffer); + +int flb_file_fstat(flb_file_handle handle, + struct flb_file_stat *output_buffer); + +char *flb_file_get_path(flb_file_handle handle); + +char *flb_file_basename(const char *path); +flb_sds_t flb_file_read_contents(const char *path); #endif diff --git a/plugins/in_tail/win32/interface.h b/include/fluent-bit/flb_file_unix.h similarity index 51% rename from plugins/in_tail/win32/interface.h rename to include/fluent-bit/flb_file_unix.h index 73b2ef233d1..d01ed48192a 100644 --- a/plugins/in_tail/win32/interface.h +++ b/include/fluent-bit/flb_file_unix.h @@ -2,7 +2,8 @@ /* Fluent Bit * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,28 +18,26 @@ * limitations under the License. */ -#ifndef FLB_TAIL_WIN32_INTERFACE_H -#define FLB_TAIL_WIN32_INTERFACE_H +#ifndef FLB_FILE_UNIX_H +#define FLB_FILE_UNIX_H -struct win32_stat { - uint64_t st_ino; - uint16_t st_mode; - int64_t st_mtime; - int16_t st_nlink; - int64_t st_size; -}; +#define FLB_FILE_IFMT S_IFMT +#define FLB_FILE_IFIFO S_IFIFO +#define FLB_FILE_IFCHR S_IFCHR +#define FLB_FILE_IFDIR S_IFDIR +#define FLB_FILE_IFBLK S_IFBLK +#define FLB_FILE_IFREG S_IFREG +#define FLB_FILE_IFLNK S_IFLNK -int win32_stat(const char *path, struct win32_stat *wst); -int win32_lstat(const char *path, struct win32_stat *wst); -int win32_fstat(int fd, struct win32_stat *wst); +#define FLB_FILE_ISDIR(m) S_ISDIR(m) +#define FLB_FILE_ISCHR(m) S_ISCHR(m) +#define FLB_FILE_ISFIFO(m) S_ISFIFO(m) +#define FLB_FILE_ISREG(m) S_ISREG(m) +#define FLB_FILE_ISLNK(m) S_ISLNK(m) -int win32_open(const char *path, int flags); +#define FLB_FILE_INVALID_HANDLE (-1) +#define FLB_FILE_MAX_PATH_LENGTH PATH_MAX -#define WIN32_S_IFDIR 0x1000 -#define WIN32_S_IFCHR 0x2000 -#define WIN32_S_IFIFO 0x4000 -#define WIN32_S_IFREG 0x8000 -#define WIN32_S_IFLNK 0xc000 -#define WIN32_S_IFMT 0xf000 +typedef int flb_file_handle; #endif diff --git a/include/fluent-bit/flb_file_win32.h b/include/fluent-bit/flb_file_win32.h new file mode 100644 index 00000000000..88fd5745a57 --- /dev/null +++ b/include/fluent-bit/flb_file_win32.h @@ -0,0 +1,44 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FILE_WIN32_H +#define FLB_FILE_WIN32_H + +#define FLB_FILE_IFMT 0170000 +#define FLB_FILE_IFIFO 0010000 +#define FLB_FILE_IFCHR 0020000 +#define FLB_FILE_IFDIR 0040000 +#define FLB_FILE_IFBLK 0060000 +#define FLB_FILE_IFREG 0100000 +#define FLB_FILE_IFLNK 0120000 + +#define FLB_FILE_ISTYPE(m, t) (((m) & FLB_FILE_IFMT) == t) +#define FLB_FILE_ISDIR(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFDIR)) +#define FLB_FILE_ISCHR(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFCHR)) +#define FLB_FILE_ISFIFO(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFIFO)) +#define FLB_FILE_ISREG(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFREG)) +#define FLB_FILE_ISLNK(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFLNK)) + +#define FLB_FILE_INVALID_HANDLE (INVALID_HANDLE_VALUE) +#define FLB_FILE_MAX_PATH_LENGTH MAX_PATH + +typedef HANDLE flb_file_handle; + +#endif diff --git a/plugins/in_node_exporter_metrics/ne_textfile_linux.c b/plugins/in_node_exporter_metrics/ne_textfile_linux.c index d65e01441da..05f5f82a210 100644 --- a/plugins/in_node_exporter_metrics/ne_textfile_linux.c +++ b/plugins/in_node_exporter_metrics/ne_textfile_linux.c @@ -143,7 +143,8 @@ static int textfile_update(struct flb_ne *ctx) mk_list_foreach(head, &list) { entry = mk_list_entry(head, struct flb_slist_entry, _head); /* Update metrics from text file */ - contents = flb_file_read(entry->str); + contents = flb_file_read_contents(entry->str); + if (contents == NULL) { flb_plg_debug(ctx->ins, "skip invalid file of prometheus: %s", entry->str); diff --git a/plugins/in_tail/CMakeLists.txt b/plugins/in_tail/CMakeLists.txt index 31d865218d5..f2790a955bc 100644 --- a/plugins/in_tail/CMakeLists.txt +++ b/plugins/in_tail/CMakeLists.txt @@ -26,11 +26,6 @@ if(FLB_PARSER) endif() if(MSVC) - set(src - ${src} - win32/stat.c - win32/io.c - ) FLB_PLUGIN(in_tail "${src}" "Shlwapi") else() FLB_PLUGIN(in_tail "${src}" "") diff --git a/plugins/in_tail/tail.c b/plugins/in_tail/tail.c index 34a0fec3dbd..8bf08ae4f5f 100644 --- a/plugins/in_tail/tail.c +++ b/plugins/in_tail/tail.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "tail.h" #include "tail_fs.h" @@ -67,7 +68,7 @@ static int in_tail_collect_pending(struct flb_input_instance *ins, struct mk_list *head; struct flb_tail_config *ctx = in_context; struct flb_tail_file *file; - struct stat st; + struct flb_file_stat st; uint64_t pre; uint64_t total_processed = 0; @@ -77,7 +78,8 @@ static int in_tail_collect_pending(struct flb_input_instance *ins, if (file->watch_fd == -1 || (file->offset >= file->size)) { - ret = fstat(file->fd, &st); + /* Gather current file size */ + ret = flb_file_fstat(file->fd, &st); if (ret == -1) { flb_errno(); @@ -85,11 +87,11 @@ static int in_tail_collect_pending(struct flb_input_instance *ins, continue; } - file->size = st.st_size; + file->size = st.size; file->pending_bytes = (file->size - file->offset); } else { - memset(&st, 0, sizeof(struct stat)); + memset(&st, 0, sizeof(struct flb_file_stat)); } if (file->pending_bytes <= 0) { @@ -325,10 +327,10 @@ static int in_tail_watcher_callback(struct flb_input_instance *ins, int in_tail_collect_event(void *file, struct flb_config *config) { int ret; - struct stat st; + struct flb_file_stat st; struct flb_tail_file *f = file; - ret = fstat(f->fd, &st); + ret = flb_file_fstat(f->fd, &st); if (ret == -1) { flb_tail_file_remove(f); return 0; diff --git a/plugins/in_tail/tail_file.c b/plugins/in_tail/tail_file.c index 2385f062627..7eee16780ef 100644 --- a/plugins/in_tail/tail_file.c +++ b/plugins/in_tail/tail_file.c @@ -44,10 +44,6 @@ #include "tail_multiline.h" #include "tail_scan.h" -#ifdef FLB_SYSTEM_WINDOWS -#include "win32.h" -#endif - #include static inline void consume_bytes(char *buf, int bytes, int length) @@ -55,36 +51,26 @@ static inline void consume_bytes(char *buf, int bytes, int length) memmove(buf, buf + bytes, length - bytes); } -static uint64_t stat_get_st_dev(struct stat *st) -{ -#ifdef FLB_SYSTEM_WINDOWS - /* do you want to contribute with a way to extract volume serial number ? */ - return 0; -#else - return st->st_dev; -#endif -} - -static int stat_to_hash_bits(struct flb_tail_config *ctx, struct stat *st, +static int stat_to_hash_bits(struct flb_tail_config *ctx, struct flb_file_stat *st, uint64_t *out_hash) { int len; - uint64_t st_dev; char tmp[64]; - st_dev = stat_get_st_dev(st); - - len = snprintf(tmp, sizeof(tmp) - 1, "%" PRIu64 ":%" PRIu64, - st_dev, (uint64_t)st->st_ino); + len = snprintf(tmp, + sizeof(tmp) - 1, + "%" PRIu64 ":%" PRIu64, + (uint64_t) st->device, + (uint64_t) st->inode); *out_hash = cfl_hash_64bits(tmp, len); + return 0; } -static int stat_to_hash_key(struct flb_tail_config *ctx, struct stat *st, +static int stat_to_hash_key(struct flb_tail_config *ctx, struct flb_file_stat *st, flb_sds_t *key) { - uint64_t st_dev; flb_sds_t tmp; flb_sds_t buf; @@ -93,15 +79,17 @@ static int stat_to_hash_key(struct flb_tail_config *ctx, struct stat *st, return -1; } - st_dev = stat_get_st_dev(st); - tmp = flb_sds_printf(&buf, "%" PRIu64 ":%" PRIu64, - st_dev, (uint64_t)st->st_ino); + tmp = flb_sds_printf(&buf, + "%" PRIu64 ":%" PRIu64, + (uint64_t) st->device, + (uint64_t) st->inode); if (!tmp) { flb_sds_destroy(buf); return -1; } *key = buf; + return 0; } @@ -793,7 +781,7 @@ static int tag_compose(char *tag, char *fname, char *out_buf, size_t *out_size, return 0; } -static inline int flb_tail_file_exists(struct stat *st, +static inline int flb_tail_file_exists(struct flb_file_stat *st, struct flb_tail_config *ctx) { int ret; @@ -835,14 +823,14 @@ static int set_file_position(struct flb_tail_config *ctx, ret = flb_tail_db_file_set(file, ctx); if (ret == 0) { if (file->offset > 0) { - ret = lseek(file->fd, file->offset, SEEK_SET); + ret = flb_file_lseek(file->fd, file->offset, SEEK_SET); if (ret == -1) { flb_errno(); return -1; } } else if (ctx->read_from_head == FLB_FALSE) { - ret = lseek(file->fd, 0, SEEK_END); + ret = flb_file_lseek(file->fd, 0, SEEK_END); if (ret == -1) { flb_errno(); return -1; @@ -861,7 +849,7 @@ static int set_file_position(struct flb_tail_config *ctx, } /* tail... */ - ret = lseek(file->fd, 0, SEEK_END); + ret = flb_file_lseek(file->fd, 0, SEEK_END); if (ret == -1) { flb_errno(); return -1; @@ -910,10 +898,12 @@ static int ml_flush_callback(struct flb_ml_parser *parser, return 0; } -int flb_tail_file_append(char *path, struct stat *st, int mode, +int flb_tail_file_append(char *path, + struct flb_file_stat *st, + int mode, struct flb_tail_config *ctx) { - int fd; + flb_file_handle fd; int ret; uint64_t stream_id; uint64_t ts; @@ -924,10 +914,10 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, char *name; size_t tag_len; struct flb_tail_file *file; - struct stat lst; + struct flb_file_stat lst; flb_sds_t inode_str; - if (!S_ISREG(st->st_mode)) { + if (!FLB_FILE_ISREG(st->mode)) { return -1; } @@ -935,8 +925,8 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, return -1; } - fd = open(path, O_RDONLY); - if (fd == -1) { + fd = flb_file_open(path, O_RDONLY); + if (fd == FLB_FILE_INVALID_HANDLE) { flb_errno(); flb_plg_error(ctx->ins, "cannot open %s", path); return -1; @@ -953,11 +943,11 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, file->fd = fd; /* On non-windows environments check if the original path is a link */ - ret = lstat(path, &lst); + ret = flb_file_lstat(path, &lst); if (ret == 0) { - if (S_ISLNK(lst.st_mode)) { + if (FLB_FILE_ISLNK(lst.mode)) { file->is_link = FLB_TRUE; - file->link_inode = lst.st_ino; + file->link_inode = lst.inode; } } @@ -977,9 +967,9 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, } file->hash_key = hash_key; - file->inode = st->st_ino; + file->inode = st->inode; file->offset = 0; - file->size = st->st_size; + file->size = st->size; file->buf_len = 0; file->parsed = 0; file->config = ctx; @@ -1191,7 +1181,8 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, } flb_free(file); } - close(fd); + + flb_file_close(fd); return -1; } @@ -1242,8 +1233,8 @@ void flb_tail_file_remove(struct flb_tail_file *file) flb_tail_fs_remove(ctx, file); /* avoid deleting file with -1 fd */ - if (file->fd != -1) { - close(file->fd); + if (file->fd != FLB_FILE_INVALID_HANDLE) { + flb_file_close(file->fd); } if (file->tag_buf) { flb_free(file->tag_buf); @@ -1297,17 +1288,17 @@ static int adjust_counters(struct flb_tail_config *ctx, struct flb_tail_file *fi { int ret; int64_t offset; - struct stat st; + struct flb_file_stat st; - ret = fstat(file->fd, &st); + ret = flb_file_fstat(file->fd, &st); if (ret == -1) { flb_errno(); return FLB_TAIL_ERROR; } /* Check if the file was truncated */ - if (file->offset > st.st_size) { - offset = lseek(file->fd, 0, SEEK_SET); + if (file->offset > st.size) { + offset = flb_file_lseek(file->fd, 0, SEEK_SET); if (offset == -1) { flb_errno(); return FLB_TAIL_ERROR; @@ -1326,8 +1317,8 @@ static int adjust_counters(struct flb_tail_config *ctx, struct flb_tail_file *fi #endif } else { - file->size = st.st_size; - file->pending_bytes = (st.st_size - file->offset); + file->size = st.size; + file->pending_bytes = (st.size - file->offset); } return FLB_TAIL_OK; @@ -1399,7 +1390,7 @@ int flb_tail_file_chunk(struct flb_tail_file *file) capacity = (file->buf_size - file->buf_len) - 1; } - bytes = read(file->fd, file->buf_data + file->buf_len, capacity); + bytes = flb_file_read(file->fd, file->buf_data + file->buf_len, capacity); if (bytes > 0) { /* we read some data, let the content processor take care of it */ file->buf_len += bytes; @@ -1463,7 +1454,7 @@ int flb_tail_file_is_rotated(struct flb_tail_config *ctx, { int ret; char *name; - struct stat st; + struct flb_file_stat st; /* * Do not double-check already rotated files since the caller of this @@ -1475,7 +1466,7 @@ int flb_tail_file_is_rotated(struct flb_tail_config *ctx, /* Check if the 'original monitored file' is a link and rotated */ if (file->is_link == FLB_TRUE) { - ret = lstat(file->name, &st); + ret = flb_file_lstat(file->name, &st); if (ret == -1) { /* Broken link or missing file */ if (errno == ENOENT) { @@ -1493,14 +1484,14 @@ int flb_tail_file_is_rotated(struct flb_tail_config *ctx, } else { /* The file name is there, check if the same that we have */ - if (st.st_ino != file->link_inode) { + if (st.inode != file->link_inode) { return FLB_TRUE; } } } /* Retrieve the real file name, operating system lookup */ - name = flb_tail_file_name(file); + name = flb_file_get_path(file->fd); if (!name) { flb_plg_error(ctx->ins, "inode=%"PRIu64" cannot detect if file was rotated: %s", @@ -1510,7 +1501,7 @@ int flb_tail_file_is_rotated(struct flb_tail_config *ctx, /* Get stats from the file name */ - ret = stat(name, &st); + ret = flb_file_stat(name, &st); if (ret == -1) { flb_errno(); flb_free(name); @@ -1518,7 +1509,7 @@ int flb_tail_file_is_rotated(struct flb_tail_config *ctx, } /* Compare inodes and names */ - if (file->inode == st.st_ino && + if (file->inode == st.inode && flb_tail_target_file_name_cmp(name, file) == 0) { flb_free(name); return FLB_FALSE; @@ -1528,6 +1519,7 @@ int flb_tail_file_is_rotated(struct flb_tail_config *ctx, file->inode, file->name, name); flb_free(name); + return FLB_TRUE; } @@ -1535,18 +1527,18 @@ int flb_tail_file_is_rotated(struct flb_tail_config *ctx, int flb_tail_file_to_event(struct flb_tail_file *file) { int ret; - struct stat st; + struct flb_file_stat st; struct flb_tail_config *ctx = file->config; /* Check if the file promoted have pending bytes */ - ret = fstat(file->fd, &st); + ret = flb_file_fstat(file->fd, &st); if (ret != 0) { flb_errno(); return -1; } - if (file->offset < st.st_size) { - file->pending_bytes = (st.st_size - file->offset); + if (file->offset < st.size) { + file->pending_bytes = (st.size - file->offset); tail_signal_pending(file->config); } else { @@ -1579,103 +1571,6 @@ int flb_tail_file_to_event(struct flb_tail_file *file) return 0; } -/* - * Given an open file descriptor, return the filename. This function is a - * bit slow and it aims to be used only when a file is rotated. - */ -char *flb_tail_file_name(struct flb_tail_file *file) -{ - int ret; - char *buf; -#ifdef __linux__ - ssize_t s; - char tmp[128]; -#elif defined(__APPLE__) - char path[PATH_MAX]; -#elif defined(FLB_SYSTEM_WINDOWS) - HANDLE h; -#elif defined(FLB_SYSTEM_FREEBSD) - struct kinfo_file *file_entries; - int file_count; - int file_index; -#endif - - buf = flb_malloc(PATH_MAX); - if (!buf) { - flb_errno(); - return NULL; - } - -#ifdef __linux__ - ret = snprintf(tmp, sizeof(tmp) - 1, "/proc/%i/fd/%i", getpid(), file->fd); - if (ret == -1) { - flb_errno(); - flb_free(buf); - return NULL; - } - - s = readlink(tmp, buf, PATH_MAX); - if (s == -1) { - flb_free(buf); - flb_errno(); - return NULL; - } - buf[s] = '\0'; - -#elif __APPLE__ - int len; - - ret = fcntl(file->fd, F_GETPATH, path); - if (ret == -1) { - flb_errno(); - flb_free(buf); - return NULL; - } - - len = strlen(path); - memcpy(buf, path, len); - buf[len] = '\0'; - -#elif defined(FLB_SYSTEM_WINDOWS) - int len; - - h = (HANDLE) _get_osfhandle(file->fd); - if (h == INVALID_HANDLE_VALUE) { - flb_errno(); - flb_free(buf); - return NULL; - } - - /* This function returns the length of the string excluding "\0" - * and the resulting path has a "\\?\" prefix. - */ - len = GetFinalPathNameByHandleA(h, buf, PATH_MAX, FILE_NAME_NORMALIZED); - if (len == 0 || len >= PATH_MAX) { - flb_free(buf); - return NULL; - } - - if (strstr(buf, "\\\\?\\")) { - memmove(buf, buf + 4, len + 1); - } -#elif defined(FLB_SYSTEM_FREEBSD) - if ((file_entries = kinfo_getfile(getpid(), &file_count)) == NULL) { - flb_free(buf); - return NULL; - } - - for (file_index=0; file_index < file_count; file_index++) { - if (file_entries[file_index].kf_fd == file->fd) { - strncpy(buf, file_entries[file_index].kf_path, PATH_MAX - 1); - buf[PATH_MAX - 1] = 0; - break; - } - } - free(file_entries); -#endif - return buf; -} - int flb_tail_file_name_dup(char *path, struct flb_tail_file *file) { file->name = flb_strdup(path); @@ -1689,7 +1584,7 @@ int flb_tail_file_name_dup(char *path, struct flb_tail_file *file) flb_free(file->real_name); } - file->real_name = flb_tail_file_name(file); + file->real_name = flb_file_get_path(file->fd); if (!file->real_name) { flb_errno(); flb_free(file->name); @@ -1708,11 +1603,11 @@ int flb_tail_file_rotated(struct flb_tail_file *file) char *name; char *i_name; char *tmp; - struct stat st; + struct flb_file_stat st; struct flb_tail_config *ctx = file->config; /* Get the new file name */ - name = flb_tail_file_name(file); + name = flb_file_get_path(file->fd); if (!name) { return -1; } @@ -1751,8 +1646,8 @@ int flb_tail_file_rotated(struct flb_tail_file *file) #endif /* Check if a new file has been created */ - ret = stat(tmp, &st); - if (ret == 0 && st.st_ino != file->inode) { + ret = flb_file_stat(tmp, &st); + if (ret == 0 && st.inode != file->inode) { if (flb_tail_file_exists(&st, ctx) == FLB_FALSE) { ret = flb_tail_file_append(tmp, &st, FLB_TAIL_STATIC, ctx); if (ret == -1) { @@ -1774,16 +1669,17 @@ static int check_purge_deleted_file(struct flb_tail_config *ctx, struct flb_tail_file *file, time_t ts) { int ret; - struct stat st; + int64_t mtime; + struct flb_file_stat st; - ret = fstat(file->fd, &st); + ret = flb_file_fstat(file->fd, &st); if (ret == -1) { flb_plg_debug(ctx->ins, "error stat(2) %s, removing", file->name); flb_tail_file_remove(file); return FLB_TRUE; } - if (st.st_nlink == 0) { + if (st.hard_link_count == 0) { flb_plg_debug(ctx->ins, "purge: monitored file has been deleted: %s", file->name); #ifdef FLB_HAVE_SQLDB @@ -1811,21 +1707,26 @@ int flb_tail_file_purge(struct flb_input_instance *ins, struct flb_tail_file *file; struct flb_tail_config *ctx = context; time_t now; - struct stat st; + struct flb_file_stat st; /* Rotated files */ now = time(NULL); mk_list_foreach_safe(head, tmp, &ctx->files_rotated) { file = mk_list_entry(head, struct flb_tail_file, _rotate_head); if ((file->rotated + ctx->rotate_wait) <= now) { - ret = fstat(file->fd, &st); + ret = flb_file_fstat(file->fd, &st); if (ret == 0) { flb_plg_debug(ctx->ins, "inode=%"PRIu64" purge rotated file %s " \ "(offset=%"PRId64" / size = %"PRIu64")", - file->inode, file->name, file->offset, (uint64_t)st.st_size); + file->inode, + file->name, + file->offset, + (uint64_t) st.size); + if (file->pending_bytes > 0 && flb_input_buf_paused(ins)) { - flb_plg_warn(ctx->ins, "purged rotated file while data " + flb_plg_warn(ctx->ins, + "purged rotated file while data " "ingestion is paused, consider increasing " "rotate_wait"); } diff --git a/plugins/in_tail/tail_file.h b/plugins/in_tail/tail_file.h index 796224c37c9..7d5caa0b76b 100644 --- a/plugins/in_tail/tail_file.h +++ b/plugins/in_tail/tail_file.h @@ -25,92 +25,41 @@ #include #include +#include #include "tail.h" #include "tail_fs.h" #include "tail_config.h" #include "tail_file_internal.h" -#ifdef FLB_SYSTEM_WINDOWS -#include "win32.h" -#endif - #ifdef FLB_HAVE_REGEX #define FLB_HASH_TABLE_SIZE 50 #endif -/* return the file modification time in seconds since epoch */ -static inline int64_t flb_tail_stat_mtime(struct stat *st) -{ -#if defined(FLB_HAVE_WINDOWS) - return (int64_t) st->st_mtime; -#elif defined(__APPLE__) && !defined(_POSIX_C_SOURCE) - return (int64_t) st->st_mtimespec.tv_sec; -#elif (_POSIX_C_SOURCE >= 200809L || \ - defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ - defined(__BIONIC__) || (defined (__SVR4) && defined (__sun)) || \ - defined(__FreeBSD__) || defined (__linux__)) - return (int64_t) st->st_mtim.tv_sec; -#elif defined(_AIX) - return (int64_t) st->st_mtime; -#else - return (int64_t) st->st_mtime; -#endif - - /* backend unsupported: submit a PR :) */ - return -1; -} - static inline int flb_tail_target_file_name_cmp(char *name, struct flb_tail_file *file) { - int ret; - char *name_a = NULL; - char *name_b = NULL; - char *base_a = NULL; - char *base_b = NULL; - - name_a = flb_strdup(name); - if (!name_a) { - flb_errno(); - ret = -1; - goto out; - } + char *base_a; + char *base_b; + int ret; - base_a = flb_strdup(basename(name_a)); - if (!base_a) { - flb_errno(); - ret = -1; - goto out; - } + base_a = NULL; + base_b = NULL; + ret = -1; -#if defined(FLB_SYSTEM_WINDOWS) - name_b = flb_strdup(file->real_name); - if (!name_b) { - flb_errno(); - ret = -1; - goto out; - } + base_a = flb_file_basename(name); - base_b = basename(name_b); - ret = _stricmp(base_a, base_b); -#else - name_b = flb_strdup(file->real_name); - if (!name_b) { - flb_errno(); - ret = -1; - goto out; - } - base_b = basename(name_b); - ret = strcmp(base_a, base_b); -#endif + if (base_a != NULL) { + base_b = flb_file_basename(file->real_name); + + if (base_b != NULL) { + ret = strcasecmp(base_a, base_b); - out: - flb_free(name_a); - flb_free(name_b); - flb_free(base_a); + flb_free(base_b); + } - /* FYI: 'base_b' never points to a new allocation, no flb_free is needed */ + flb_free(base_a); + } return ret; } @@ -118,11 +67,10 @@ static inline int flb_tail_target_file_name_cmp(char *name, int flb_tail_file_name_dup(char *path, struct flb_tail_file *file); int flb_tail_file_to_event(struct flb_tail_file *file); int flb_tail_file_chunk(struct flb_tail_file *file); -int flb_tail_file_append(char *path, struct stat *st, int mode, +int flb_tail_file_append(char *path, struct flb_file_stat *st, int mode, struct flb_tail_config *ctx); void flb_tail_file_remove(struct flb_tail_file *file); int flb_tail_file_remove_all(struct flb_tail_config *ctx); -char *flb_tail_file_name(struct flb_tail_file *file); int flb_tail_file_is_rotated(struct flb_tail_config *ctx, struct flb_tail_file *file); int flb_tail_file_rotated(struct flb_tail_file *file); diff --git a/plugins/in_tail/tail_file_internal.h b/plugins/in_tail/tail_file_internal.h index 6d95c87c195..d7226702b94 100644 --- a/plugins/in_tail/tail_file_internal.h +++ b/plugins/in_tail/tail_file_internal.h @@ -21,6 +21,7 @@ #define FLB_TAIL_INTERNAL_H #include +#include #include #include #include @@ -37,7 +38,7 @@ struct flb_tail_file { /* Inotify */ int watch_fd; /* file lookup info */ - int fd; + flb_file_handle fd; int64_t size; int64_t offset; int64_t last_line; diff --git a/plugins/in_tail/tail_fs_inotify.c b/plugins/in_tail/tail_fs_inotify.c index 59d10ca08d5..d819618d76a 100644 --- a/plugins/in_tail/tail_fs_inotify.c +++ b/plugins/in_tail/tail_fs_inotify.c @@ -132,7 +132,7 @@ static int tail_fs_add(struct flb_tail_file *file, int check_rotated) * 6. conflict: inotify_add_watch() receives the path 'a.log', */ - name = flb_tail_file_name(file); + name = flb_file_get_path(file->fd); if (!name) { flb_plg_error(ctx->ins, "inode=%"PRIu64" cannot get real filename for inotify", file->inode); diff --git a/plugins/in_tail/tail_fs_stat.c b/plugins/in_tail/tail_fs_stat.c index 6b312c9bd43..3738e16547b 100644 --- a/plugins/in_tail/tail_fs_stat.c +++ b/plugins/in_tail/tail_fs_stat.c @@ -20,6 +20,7 @@ #define _DEFAULT_SOURCE #include +#include #include #include @@ -30,16 +31,12 @@ #include "tail_config.h" #include "tail_signal.h" -#ifdef FLB_SYSTEM_WINDOWS -#include "win32.h" -#endif - struct fs_stat { /* last time check */ time_t checked; /* previous status */ - struct stat st; + struct flb_file_stat st; }; static int tail_fs_event(struct flb_input_instance *ins, @@ -51,7 +48,7 @@ static int tail_fs_event(struct flb_input_instance *ins, struct flb_tail_config *ctx = in_context; struct flb_tail_file *file = NULL; struct fs_stat *fst; - struct stat st; + struct flb_file_stat st; time_t t; t = time(NULL); @@ -62,17 +59,17 @@ static int tail_fs_event(struct flb_input_instance *ins, fst = file->fs_backend; /* Check current status of the file */ - ret = fstat(file->fd, &st); + ret = flb_file_fstat(file->fd, &st); if (ret == -1) { flb_errno(); continue; } /* Check if the file was modified */ - if ((fst->st.st_mtime != st.st_mtime) || - (fst->st.st_size != st.st_size)) { + if ((fst->st.modification_time != st.modification_time) || + (fst->st.size != st.size)) { /* Update stat info and trigger the notification */ - memcpy(&fst->st, &st, sizeof(struct stat)); + memcpy(&fst->st, &st, sizeof(struct flb_file_stat)); fst->checked = t; in_tail_collect_event(file, config); } @@ -92,14 +89,14 @@ static int tail_fs_check(struct flb_input_instance *ins, struct flb_tail_config *ctx = in_context; struct flb_tail_file *file = NULL; struct fs_stat *fst; - struct stat st; + struct flb_file_stat st; /* Lookup watched file */ mk_list_foreach_safe(head, tmp, &ctx->files_event) { file = mk_list_entry(head, struct flb_tail_file, _head); fst = file->fs_backend; - ret = fstat(file->fd, &st); + ret = flb_file_fstat(file->fd, &st); if (ret == -1) { flb_plg_debug(ctx->ins, "error stat(2) %s, removing", file->name); flb_tail_file_remove(file); @@ -107,7 +104,7 @@ static int tail_fs_check(struct flb_input_instance *ins, } /* Check if the file have been deleted */ - if (st.st_nlink == 0) { + if (st.hard_link_count == 0) { flb_plg_debug(ctx->ins, "file has been deleted: %s", file->name); #ifdef FLB_HAVE_SQLDB if (ctx->db) { @@ -120,8 +117,8 @@ static int tail_fs_check(struct flb_input_instance *ins, } /* Check if the file was truncated */ - if (file->offset > st.st_size) { - offset = lseek(file->fd, 0, SEEK_SET); + if (file->offset > st.size) { + offset = flb_file_lseek(file->fd, 0, SEEK_SET); if (offset == -1) { flb_errno(); return -1; @@ -130,7 +127,7 @@ static int tail_fs_check(struct flb_input_instance *ins, flb_plg_debug(ctx->ins, "file truncated %s", file->name); file->offset = offset; file->buf_len = 0; - memcpy(&fst->st, &st, sizeof(struct stat)); + memcpy(&fst->st, &st, sizeof(struct flb_file_stat)); #ifdef FLB_HAVE_SQLDB /* Update offset in database file */ @@ -140,8 +137,8 @@ static int tail_fs_check(struct flb_input_instance *ins, #endif } - if (file->offset < st.st_size) { - file->pending_bytes = (st.st_size - file->offset); + if (file->offset < st.size) { + file->pending_bytes = (st.size - file->offset); tail_signal_pending(ctx); } else { @@ -150,7 +147,7 @@ static int tail_fs_check(struct flb_input_instance *ins, /* Discover the current file name for the open file descriptor */ - name = flb_tail_file_name(file); + name = flb_file_get_path(file->fd); if (!name) { flb_plg_debug(ctx->ins, "could not resolve %s, removing", file->name); flb_tail_file_remove(file); @@ -227,7 +224,7 @@ int flb_tail_fs_stat_add(struct flb_tail_file *file) } fst->checked = time(NULL); - ret = stat(file->name, &fst->st); + ret = flb_file_stat(file->name, &fst->st); if (ret == -1) { flb_errno(); flb_free(fst); diff --git a/plugins/in_tail/tail_scan_glob.c b/plugins/in_tail/tail_scan_glob.c index b330b7c3b23..ed7f19335e2 100644 --- a/plugins/in_tail/tail_scan_glob.c +++ b/plugins/in_tail/tail_scan_glob.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "tail.h" #include "tail_file.h" @@ -32,88 +33,6 @@ #include "tail_scan.h" #include "tail_config.h" -/* Define missing GLOB_TILDE if not exists */ -#ifndef GLOB_TILDE -#define GLOB_TILDE 1<<2 /* use GNU Libc value */ -#define UNSUP_TILDE 1 - -/* we need these extra headers for path resolution */ -#include -#include -#include - -static char *expand_tilde(const char *path) -{ - int len; - char user[256]; - char *p = NULL; - char *dir = NULL; - char *tmp = NULL; - struct passwd *uinfo = NULL; - - if (path[0] == '~') { - p = strchr(path, '/'); - - if (p) { - /* check case '~/' */ - if ((p - path) == 1) { - dir = getenv("HOME"); - if (!dir) { - return path; - } - } - else { - /* - * it refers to a different user: ~user/abc, first step grab - * the user name. - */ - len = (p - path) - 1; - memcpy(user, path + 1, len); - user[len] = '\0'; - - /* use getpwnam() to resolve user information */ - uinfo = getpwnam(user); - if (!uinfo) { - return path; - } - - dir = uinfo->pw_dir; - } - } - else { - dir = getenv("HOME"); - if (!dir) { - return path; - } - } - - if (p) { - tmp = flb_malloc(PATH_MAX); - if (!tmp) { - flb_errno(); - return NULL; - } - snprintf(tmp, PATH_MAX - 1, "%s%s", dir, p); - } - else { - dir = getenv("HOME"); - if (!dir) { - return path; - } - - tmp = flb_strdup(dir); - if (!tmp) { - return path; - } - } - - return tmp; - } - - return path; -} -#endif - static int tail_is_excluded(char *path, struct flb_tail_config *ctx) { struct mk_list *head; @@ -133,146 +52,105 @@ static int tail_is_excluded(char *path, struct flb_tail_config *ctx) return FLB_FALSE; } -static inline int do_glob(const char *pattern, int flags, - void *not_used, glob_t *pglob) -{ - int ret; - int new_flags; - char *tmp = NULL; - int tmp_needs_free = FLB_FALSE; - (void) not_used; - - /* Save current values */ - new_flags = flags; - - if (flags & GLOB_TILDE) { -#ifdef UNSUP_TILDE - /* - * Some libc libraries like Musl do not support GLOB_TILDE for tilde - * expansion. A workaround is to use wordexp(3) but looking at it - * implementation in Musl it looks quite expensive: - * - * http://git.musl-libc.org/cgit/musl/tree/src/misc/wordexp.c - * - * the workaround is to do our own tilde expansion in a temporary buffer. - */ - - /* Look for a tilde */ - tmp = expand_tilde(pattern); - if (tmp != pattern) { - /* the path was expanded */ - pattern = tmp; - tmp_needs_free = FLB_TRUE; - } - - /* remove unused flag */ - new_flags &= ~GLOB_TILDE; -#endif - } - - /* invoke glob with new parameters */ - ret = glob(pattern, new_flags, NULL, pglob); - - /* remove temporary buffer, if allocated by expand_tilde above. - * Note that this buffer is only used for libc implementations - * that do not support the GLOB_TILDE flag, like musl. */ - if ((tmp != NULL) && (tmp_needs_free == FLB_TRUE)) { - flb_free(tmp); - } - - return ret; -} - /* Scan a path, register the entries and return how many */ static int tail_scan_path(const char *path, struct flb_tail_config *ctx) { - int i; int ret; - int count = 0; - glob_t globbuf; + int count; time_t now; int64_t mtime; - struct stat st; + struct flb_file_stat st; + struct flb_file_glob_context glob_context; + uint64_t glob_flags; + char *file_path; flb_plg_debug(ctx->ins, "scanning path %s", path); - /* Safe reset for globfree() */ - globbuf.gl_pathv = NULL; + glob_flags = FLB_FILE_GLOB_ABORT_ON_ERROR | FLB_FILE_GLOB_EXPAND_TILDE; - /* Scan the given path */ - ret = do_glob(path, GLOB_TILDE | GLOB_ERR, NULL, &globbuf); - if (ret != 0) { - switch (ret) { - case GLOB_NOSPACE: + ret = flb_file_glob_start(&glob_context, path, glob_flags ); + + if (ret != FLB_FILE_GLOB_ERROR_SUCCESS) { + if (ret == FLB_FILE_GLOB_ERROR_NO_MEMORY) { flb_plg_error(ctx->ins, "no memory space available"); - return -1; - case GLOB_ABORTED: + } + else if (ret == FLB_FILE_GLOB_ERROR_ABORTED) { flb_plg_error(ctx->ins, "read error, check permissions: %s", path); - return -1; - case GLOB_NOMATCH: - ret = stat(path, &st); - if (ret == -1) { - flb_plg_debug(ctx->ins, "cannot read info from: %s", path); - } - else { - ret = access(path, R_OK); - if (ret == -1 && errno == EACCES) { - flb_plg_error(ctx->ins, "NO read access for path: %s", path); - } - else { - flb_plg_debug(ctx->ins, "NO matches for path: %s", path); - } - } - return 0; } - } + else if (ret == FLB_FILE_GLOB_ERROR_NO_FILE) { + flb_plg_debug(ctx->ins, "cannot read info from: %s", path); + } + else if (ret == FLB_FILE_GLOB_ERROR_NO_ACCESS) { + flb_plg_error(ctx->ins, "no read access for path: %s", path); + } + else if (ret == FLB_FILE_GLOB_ERROR_NO_MATCHES) { + flb_plg_debug(ctx->ins, "no matches for path: %s", path); + } + else if (ret == FLB_FILE_GLOB_ERROR_OVERSIZED_PATH) { + flb_plg_debug(ctx->ins, "oversized path or entry: %s", path); + } + flb_file_glob_clean(&glob_context); + + return -1; + } - /* For every entry found, generate an output list */ now = time(NULL); - for (i = 0; i < globbuf.gl_pathc; i++) { - ret = stat(globbuf.gl_pathv[i], &st); - if (ret == 0 && S_ISREG(st.st_mode)) { + count = 0; + + while (flb_file_glob_fetch(&glob_context, &file_path) == + FLB_FILE_GLOB_ERROR_SUCCESS) { + ret = flb_file_stat(file_path, &st); + + if (ret == 0 && FLB_FILE_ISREG(st.mode)) { /* Check if this file is blacklisted */ - if (tail_is_excluded(globbuf.gl_pathv[i], ctx) == FLB_TRUE) { - flb_plg_debug(ctx->ins, "excluded=%s", globbuf.gl_pathv[i]); + if (tail_is_excluded(file_path, ctx) == FLB_TRUE) { + flb_plg_debug(ctx->ins, "excluded=%s", file_path); continue; } if (ctx->ignore_older > 0) { - mtime = flb_tail_stat_mtime(&st); + mtime = st.modification_time; if (mtime > 0) { if ((now - ctx->ignore_older) > mtime) { flb_plg_debug(ctx->ins, "excluded=%s (ignore_older)", - globbuf.gl_pathv[i]); + file_path); continue; } } } /* Append file to list */ - ret = flb_tail_file_append(globbuf.gl_pathv[i], &st, + ret = flb_tail_file_append(file_path, &st, FLB_TAIL_STATIC, ctx); if (ret == 0) { - flb_plg_debug(ctx->ins, "scan_glob add(): %s, inode %li", - globbuf.gl_pathv[i], st.st_ino); + flb_plg_debug(ctx->ins, + "scan_glob add(): %s, inode %"PRIu64, + file_path, + (uint64_t) st.inode); + count++; } else { - flb_plg_debug(ctx->ins, "scan_blog add(): dismissed: %s, inode %li", - globbuf.gl_pathv[i], st.st_ino); + flb_plg_debug(ctx->ins, + "scan_blog add(): dismissed: %s, inode %"PRIu64, + file_path, + (uint64_t) st.inode); } } else { flb_plg_debug(ctx->ins, "skip (invalid) entry=%s", - globbuf.gl_pathv[i]); + file_path); } } - if (count > 0) { - tail_signal_manager(ctx); - } +/* + I removed the call to tail_signal_manager because flb_tail_file_append + does emit a signal when the file is appended with FLB_TAIL_STATIC as its + mode. +*/ + + flb_file_glob_clean(&glob_context); - globfree(&globbuf); return count; } diff --git a/plugins/in_tail/tail_scan_win32.c b/plugins/in_tail/tail_scan_win32.c index 94733f06536..1fa95fe9be6 100644 --- a/plugins/in_tail/tail_scan_win32.c +++ b/plugins/in_tail/tail_scan_win32.c @@ -26,6 +26,7 @@ #include #include #include +#include #include @@ -34,8 +35,6 @@ #include "tail_signal.h" #include "tail_config.h" -#include "win32.h" - static int tail_is_excluded(char *path, struct flb_tail_config *ctx) { struct mk_list *head; @@ -55,45 +54,6 @@ static int tail_is_excluded(char *path, struct flb_tail_config *ctx) return FLB_FALSE; } -/* - * This function is a thin wrapper over flb_tail_file_append(), - * adding normalization and sanity checks on top of it. - */ -static int tail_register_file(const char *target, struct flb_tail_config *ctx, - time_t ts) -{ - int64_t mtime; - struct stat st; - char path[MAX_PATH]; - - if (_fullpath(path, target, MAX_PATH) == NULL) { - flb_plg_error(ctx->ins, "cannot get absolute path of %s", target); - return -1; - } - - if (stat(path, &st) != 0 || !S_ISREG(st.st_mode)) { - return -1; - } - - if (ctx->ignore_older > 0) { - mtime = flb_tail_stat_mtime(&st); - if (mtime > 0) { - if ((ts - ctx->ignore_older) > mtime) { - flb_plg_debug(ctx->ins, "excluded=%s (ignore_older)", - target); - return -1; - } - } - } - - if (tail_is_excluded(path, ctx) == FLB_TRUE) { - flb_plg_trace(ctx->ins, "skip '%s' (excluded)", path); - return -1; - } - - return flb_tail_file_append(path, &st, FLB_TAIL_STATIC, ctx); -} - /* * Perform patern match on the given path string. This function * supports patterns with "nested" wildcards like below. @@ -105,141 +65,148 @@ static int tail_register_file(const char *target, struct flb_tail_config *ctx, */ static int tail_scan_pattern(const char *path, struct flb_tail_config *ctx) { - char *star, *p0, *p1; - char pattern[MAX_PATH]; - char buf[MAX_PATH]; - int ret; - int n_added = 0; - time_t now; - int64_t mtime; - HANDLE h; - WIN32_FIND_DATA data; - - if (strlen(path) > MAX_PATH - 1) { - flb_plg_error(ctx->ins, "path too long '%s'"); - return -1; - } + struct flb_file_glob_context glob_context; + char *filename; + int count; + time_t now; + int ret; + + ret = flb_file_glob_start(&glob_context, + path, + FLB_FILE_GLOB_ABORT_ON_ERROR); + + if (ret != FLB_FILE_GLOB_ERROR_SUCCESS) { + if (ret == FLB_FILE_GLOB_ERROR_NO_MEMORY) { + flb_plg_error(ctx->ins, "no memory space available"); + } + else if (ret == FLB_FILE_GLOB_ERROR_ABORTED) { + flb_plg_error(ctx->ins, "read error, check permissions: %s", path); + } + else if (ret == FLB_FILE_GLOB_ERROR_NO_FILE) { + flb_plg_debug(ctx->ins, "cannot read info from: %s", path); + } + else if (ret == FLB_FILE_GLOB_ERROR_NO_ACCESS) { + flb_plg_error(ctx->ins, "no read access for path: %s", path); + } + else if (ret == FLB_FILE_GLOB_ERROR_NO_MATCHES) { + flb_plg_debug(ctx->ins, "no matches for path: %s", path); + } - star = strchr(path, '*'); - if (star == NULL) { - return -1; - } + flb_file_glob_clean(&glob_context); - /* - * C:\data\tmp\input_*.conf - * 0<-----| - */ - p0 = star; - while (path <= p0 && *p0 != '\\') { - p0--; + return -1; } - /* - * C:\data\tmp\input_*.conf - * |---->1 - */ - p1 = star; - while (*p1 && *p1 != '\\') { - p1++; - } + now = time(NULL); + count = 0; - memcpy(pattern, path, (p1 - path)); - pattern[p1 - path] = '\0'; + while(flb_file_glob_fetch(&glob_context, + &filename) == FLB_FILE_GLOB_ERROR_SUCCESS) { - h = FindFirstFileA(pattern, &data); - if (h == INVALID_HANDLE_VALUE) { - return 0; /* none matched */ - } + /* Try to register the target file */ + ret = tail_register_file(filename, ctx, now); - now = time(NULL); - do { - /* Ignore the current and parent dirs */ - if (!strcmp(".", data.cFileName) || !strcmp("..", data.cFileName)) { - continue; + if (ret == 0) { + count++; } + } - /* Avoid an infinite loop */ - if (strchr(data.cFileName, '*')) { - continue; - } + flb_file_glob_clean(&glob_context); - /* Create a path (prefix + filename + suffix) */ - memcpy(buf, path, p0 - path + 1); - buf[p0 - path + 1] = '\0'; + return count; +} - if (strlen(buf) + strlen(data.cFileName) + strlen(p1) > MAX_PATH - 1) { - flb_plg_warn(ctx->ins, "'%s%s%s' is too long", buf, data.cFileName, p1); - continue; +static int tail_scan_path(const char *path, struct flb_tail_config *ctx) +{ + struct flb_file_glob_context glob_context; + char *file_path; + int64_t mtime; + int count; + time_t now; + int ret; + struct flb_file_stat st; + + ret = flb_file_glob_start(&glob_context, + path, + FLB_FILE_GLOB_ABORT_ON_ERROR); + + if (ret != FLB_FILE_GLOB_ERROR_SUCCESS) { + if (ret == FLB_FILE_GLOB_ERROR_NO_MEMORY) { + flb_plg_error(ctx->ins, "no memory space available"); } - strcat(buf, data.cFileName); - strcat(buf, p1); - - if (strchr(p1, '*')) { - ret = tail_scan_pattern(buf, ctx); /* recursive */ - if (ret >= 0) { - n_added += ret; - } - continue; + else if (ret == FLB_FILE_GLOB_ERROR_ABORTED) { + flb_plg_error(ctx->ins, "read error, check permissions: %s", path); } - - /* Try to register the target file */ - ret = tail_register_file(buf, ctx, now); - if (ret == 0) { - n_added++; + else if (ret == FLB_FILE_GLOB_ERROR_NO_FILE) { + flb_plg_debug(ctx->ins, "cannot read info from: %s", path); + } + else if (ret == FLB_FILE_GLOB_ERROR_NO_ACCESS) { + flb_plg_error(ctx->ins, "no read access for path: %s", path); + } + else if (ret == FLB_FILE_GLOB_ERROR_NO_MATCHES) { + flb_plg_debug(ctx->ins, "no matches for path: %s", path); + } + else if (ret == FLB_FILE_GLOB_ERROR_OVERSIZED_PATH) { + flb_plg_debug(ctx->ins, "oversized path or entry: %s", path); } - } while (FindNextFileA(h, &data) != 0); - FindClose(h); - return n_added; -} + flb_file_glob_clean(&glob_context); -static int tail_filepath(char *buf, int len, const char *basedir, const char *filename) -{ - char drive[_MAX_DRIVE]; - char dir[_MAX_DIR]; - char fname[_MAX_FNAME]; - char ext[_MAX_EXT]; - char tmp[MAX_PATH]; - int ret; - - ret = _splitpath_s(basedir, drive, _MAX_DRIVE, dir, _MAX_DIR, NULL, 0, NULL, 0); - if (ret) { return -1; } - ret = _splitpath_s(filename, NULL, 0, NULL, 0, fname, _MAX_FNAME, ext, _MAX_EXT); - if (ret) { - return -1; - } + now = time(NULL); + count = 0; - ret = _makepath_s(tmp, MAX_PATH, drive, dir, fname, ext); - if (ret) { - return -1; - } - if (_fullpath(buf, tmp, len) == NULL) { - return -1; - } + while(flb_file_glob_fetch(&glob_context, &file_path) == + FLB_FILE_GLOB_ERROR_SUCCESS) { - return 0; -} + ret = flb_file_stat(file_path, &st); -static int tail_scan_path(const char *path, struct flb_tail_config *ctx) -{ - int ret; - int n_added = 0; - time_t now; + if (ret == 0 && FLB_FILE_ISREG(st.mode)) { + /* Check if this file is blacklisted */ + if (tail_is_excluded(file_path, ctx) == FLB_TRUE) { + flb_plg_debug(ctx->ins, "excluded=%s", file_path); + continue; + } - if (strchr(path, '*')) { - return tail_scan_pattern(path, ctx); - } + if (ctx->ignore_older > 0) { + mtime = st.modification_time; + if (mtime > 0) { + if ((now - ctx->ignore_older) > mtime) { + flb_plg_debug(ctx->ins, "excluded=%s (ignore_older)", + file_path); + continue; + } + } + } - /* No wildcard involved. Let's just handle the file... */ - now = time(NULL); - ret = tail_register_file(path, ctx, now); - if (ret == 0) { - n_added++; + /* Append file to list */ + ret = flb_tail_file_append(file_path, &st, + FLB_TAIL_STATIC, ctx); + if (ret == 0) { + flb_plg_debug(ctx->ins, + "scan_glob add(): %s, inode %"PRIu64, + file_path, + (uint64_t) st.inode); + + count++; + } + else { + flb_plg_debug(ctx->ins, + "scan_blog add(): dismissed: %s, inode %"PRIu64, + file_path, + (uint64_t) st.inode); + } + } + else { + flb_plg_debug(ctx->ins, "skip (invalid) entry=%s", + file_path); + } } - return n_added; + flb_file_glob_clean(&glob_context); + + return count; } diff --git a/plugins/in_tail/win32.h b/plugins/in_tail/win32.h deleted file mode 100644 index a9414f89259..00000000000 --- a/plugins/in_tail/win32.h +++ /dev/null @@ -1,67 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This is the interface file that replaces POSIX functions - * with our own custom implementation. - */ - -#ifndef FLB_TAIL_WIN32_H -#define FLB_TAIL_WIN32_H - -#include "win32/interface.h" - -#undef open -#undef stat -#undef lstat -#undef fstat -#undef lseek - -#undef S_IFDIR -#undef S_IFCHR -#undef S_IFIFO -#undef S_IFREG -#undef S_IFLNK -#undef S_IFMT -#undef S_ISDIR -#undef S_ISCHR -#undef S_ISFIFO -#undef S_ISREG -#undef S_ISLNK - -#define open win32_open -#define stat win32_stat -#define lstat win32_lstat -#define fstat win32_fstat - -#define lseek _lseeki64 - -#define S_IFDIR WIN32_S_IFDIR -#define S_IFCHR WIN32_S_IFCHR -#define S_IFIFO WIN32_S_IFIFO -#define S_IFREG WIN32_S_IFREG -#define S_IFLNK WIN32_S_IFLNK -#define S_IFMT WIN32_S_IFMT - -#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) -#define S_ISCHR(m) (((m) & S_IFMT) == S_IFCHR) -#define S_ISIFO(m) (((m) & S_IFMT) == S_IFIFO) -#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) -#define S_ISLNK(m) (((m) & S_IFMT) == S_IFLNK) -#endif diff --git a/plugins/in_tail/win32/io.c b/plugins/in_tail/win32/io.c deleted file mode 100644 index 45928b04a6b..00000000000 --- a/plugins/in_tail/win32/io.c +++ /dev/null @@ -1,47 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include "interface.h" - -/* - * POSIX IO emulation tailored for in_tail's usage. - * - * open(2) that does not acquire an exclusive lock. - */ - -int win32_open(const char *path, int flags) -{ - HANDLE h; - h = CreateFileA(path, - GENERIC_READ, - FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, - NULL, /* lpSecurityAttributes */ - OPEN_EXISTING, /* dwCreationDisposition */ - 0, /* dwFlagsAndAttributes */ - NULL); /* hTemplateFile */ - if (h == INVALID_HANDLE_VALUE) { - return -1; - } - return _open_osfhandle((intptr_t) h, _O_RDONLY); -} diff --git a/plugins/in_tail/win32/stat.c b/plugins/in_tail/win32/stat.c deleted file mode 100644 index bce80274960..00000000000 --- a/plugins/in_tail/win32/stat.c +++ /dev/null @@ -1,332 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include "interface.h" - -/* - * NTFS stat(2) emulation tailored for in_tail's usage. - * - * (1) Support st_ino (inode) for Windows NTFS. - * (2) Support NTFS symlinks. - * (3) Support large files >= 2GB. - * - * To use it, include "win32.h" and it will transparently - * replace stat(), lstat() and fstat(). - */ - -#define UINT64(high, low) ((uint64_t) (high) << 32 | (low)) -#define WINDOWS_TICKS_TO_SECONDS_RATIO 10000000 -#define WINDOWS_EPOCH_TO_UNIX_EPOCH_DELTA 11644473600 - -/* - * FILETIME timestamps are represented in 100-nanosecond intervals, - * because of this, that's why we need to divide the number by 10000000 - * in order to convert it to seconds. - * - * While UNIX timestamps use January 1, 1970 as epoch Windows FILETIME - * timestamps use January 1, 1601. Because of this we need to subtract - * 11644473600 seconds to account for it. - * - * Note: Even though this does not account for leap seconds it should be - * accurate enough. - */ - -static uint64_t filetime_to_epoch(FILETIME *ft) -{ - ULARGE_INTEGER timestamp; - - if (ft == NULL) { - return 0; - } - - timestamp.HighPart = ft->dwHighDateTime; - timestamp.LowPart = ft->dwLowDateTime; - - timestamp.QuadPart /= WINDOWS_TICKS_TO_SECONDS_RATIO; - timestamp.QuadPart -= WINDOWS_EPOCH_TO_UNIX_EPOCH_DELTA; - - return timestamp.QuadPart; -} - -static void reset_errno() -{ - errno = 0; -} - -static void propagate_last_error_to_errno() -{ - DWORD error_code; - - error_code = GetLastError(); - - switch (error_code) { - case ERROR_INVALID_TARGET_HANDLE: - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - - case ERROR_TOO_MANY_OPEN_FILES: - errno = EMFILE; - break; - - case ERROR_INVALID_FLAG_NUMBER: - case ERROR_INVALID_PARAMETER: - errno = EINVAL; - break; - - case ERROR_NOT_ENOUGH_MEMORY: - case ERROR_OUTOFMEMORY: - errno = ENOMEM; - break; - - case ERROR_SHARING_VIOLATION: - case ERROR_LOCK_VIOLATION: - case ERROR_PATH_BUSY: - case ERROR_BUSY: - errno = EBUSY; - break; - - case ERROR_HANDLE_DISK_FULL: - case ERROR_DISK_FULL: - errno = ENOSPC; - break; - - case ERROR_INVALID_ADDRESS: - errno = EFAULT; - break; - - case ERROR_FILE_TOO_LARGE: - errno = EFBIG; - break; - - case ERROR_ALREADY_EXISTS: - case ERROR_FILE_EXISTS: - errno = EEXIST; - break; - - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: - case ERROR_INVALID_DRIVE: - case ERROR_BAD_PATHNAME: - case ERROR_INVALID_NAME: - case ERROR_BAD_UNIT: - errno = ENOENT; - break; - - case ERROR_SEEK_ON_DEVICE: - case ERROR_NEGATIVE_SEEK: - errno = ESPIPE; - break; - - case ERROR_ACCESS_DENIED: - errno = EACCES; - break; - - case ERROR_DIR_NOT_EMPTY: - errno = ENOTEMPTY; - break; - - case ERROR_BROKEN_PIPE: - errno = EPIPE; - break; - - case ERROR_GEN_FAILURE: - errno = EIO; - break; - - case ERROR_OPEN_FAILED: - errno = EIO; - break; - - case ERROR_SUCCESS: - errno = 0; - break; - - default: - /* This is just a canary, if you find this - * error then it means we need to expand the - * translation list. - */ - - errno = EOWNERDEAD; - break; - } -} - -static int get_mode(unsigned int attr) -{ - if (attr & FILE_ATTRIBUTE_DIRECTORY) { - return WIN32_S_IFDIR; - } - return WIN32_S_IFREG; -} - - - -static int is_symlink(const char *path) -{ - WIN32_FIND_DATA data; - HANDLE h; - - SetLastError(0); - reset_errno(); - - h = FindFirstFileA(path, &data); - - if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); - - return 0; - } - - FindClose(h); - - /* - * A NTFS symlink is a file with a bit of metadata ("reparse point"), - * So (1) check if the file has metadata and then (2) confirm that - * it is indeed a symlink. - */ - if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - if (data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) { - return 1; - } - } - - return 0; -} - -static int hstat(HANDLE h, struct win32_stat *wst) -{ - BY_HANDLE_FILE_INFORMATION info; - FILE_STANDARD_INFO std; - - SetLastError(0); - reset_errno(); - - if (!GetFileInformationByHandle(h, &info)) { - propagate_last_error_to_errno(); - - return -1; - } - - if (!GetFileInformationByHandleEx(h, FileStandardInfo, - &std, sizeof(std))) { - propagate_last_error_to_errno(); - - return -1; - } - - wst->st_nlink = std.NumberOfLinks; - if (std.DeletePending) { - wst->st_nlink = 0; - } - - wst->st_mode = get_mode(info.dwFileAttributes); - wst->st_size = UINT64(info.nFileSizeHigh, info.nFileSizeLow); - wst->st_ino = UINT64(info.nFileIndexHigh, info.nFileIndexLow); - wst->st_mtime = filetime_to_epoch(&info.ftLastWriteTime); - - return 0; -} - -int win32_stat(const char *path, struct win32_stat *wst) -{ - HANDLE h; - - SetLastError(0); - reset_errno(); - - h = CreateFileA(path, - GENERIC_READ, - FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, - NULL, /* lpSecurityAttributes */ - OPEN_EXISTING, /* dwCreationDisposition */ - 0, /* dwFlagsAndAttributes */ - NULL); /* hTemplateFile */ - - if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); - - return -1; - } - - if (hstat(h, wst)) { - CloseHandle(h); - return -1; - } - - CloseHandle(h); - return 0; -} - -int win32_lstat(const char *path, struct win32_stat *wst) -{ - HANDLE h; - - SetLastError(0); - reset_errno(); - - h = CreateFileA(path, - GENERIC_READ, - FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, - NULL, /* lpSecurityAttributes */ - OPEN_EXISTING, /* dwCreationDisposition */ - FILE_FLAG_OPEN_REPARSE_POINT, - NULL); /* hTemplateFile */ - - if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); - - return -1; - } - - if (hstat(h, wst)) { - CloseHandle(h); - return -1; - } - - if (is_symlink(path)) { - wst->st_mode = WIN32_S_IFLNK; - } - - CloseHandle(h); - return 0; -} - -int win32_fstat(int fd, struct win32_stat *wst) -{ - HANDLE h; - - SetLastError(0); - reset_errno(); - - h = (HANDLE) _get_osfhandle(fd); - - if (h == INVALID_HANDLE_VALUE) { - propagate_last_error_to_errno(); - - return -1; - } - - return hstat(h, wst); -} diff --git a/plugins/out_oracle_log_analytics/oci_logan_conf.c b/plugins/out_oracle_log_analytics/oci_logan_conf.c index a3980318465..c113fa2cd30 100644 --- a/plugins/out_oracle_log_analytics/oci_logan_conf.c +++ b/plugins/out_oracle_log_analytics/oci_logan_conf.c @@ -103,7 +103,7 @@ static int load_oci_credentials(struct flb_oci_logan *ctx) char* key = NULL; char* val; - content = flb_file_read(ctx->config_file_location); + content = flb_file_read_contents(ctx->config_file_location); if (content == NULL || flb_sds_len(content) == 0) { return -1; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b6233d9f721..641917199d9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -80,6 +80,36 @@ set(src flb_reload.c ) +# File access abstraction layer +if(FLB_SYSTEM_WINDOWS) + set(src + ${src} + flb_file_win32.c + ) +else() + if(FLB_SYSTEM_LINUX) + set(src + ${src} + flb_file_linux.c + ) + elseif(FLB_SYSTEM_MACOS) + set(src + ${src} + flb_file_macos.c + ) + elseif(FLB_SYSTEM_FREEBSD) + set(src + ${src} + flb_file_freebsd.c + ) + endif() + + set(src + ${src} + flb_file_unix.c + ) +endif() + # Config format set(src ${src} diff --git a/src/flb_file.c b/src/flb_file.c index 2225bc3c927..b4e11bb9bd0 100644 --- a/src/flb_file.c +++ b/src/flb_file.c @@ -18,14 +18,16 @@ * limitations under the License. */ +#include #include #include #include #include +#include #include -flb_sds_t flb_file_read(const char *path) +flb_sds_t flb_file_read_contents(const char *path) { long flen; FILE *f = NULL; diff --git a/src/flb_file_freebsd.c b/src/flb_file_freebsd.c new file mode 100644 index 00000000000..266f3f63b59 --- /dev/null +++ b/src/flb_file_freebsd.c @@ -0,0 +1,63 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +char *flb_file_get_path(flb_file_handle handle) +{ + char *buf; + struct kinfo_file *file_entries; + int file_count; + int file_index; + + buf = flb_calloc(sizeof(char), PATH_MAX); + + if (buf == NULL) { + flb_errno(); + return NULL; + } + + if ((file_entries = kinfo_getfile(getpid(), &file_count)) == NULL) { + flb_free(buf); + return NULL; + } + + for (file_index=0; file_index < file_count; file_index++) { + if (file_entries[file_index].kf_fd == handle) { + strncpy(buf, file_entries[file_index].kf_path, PATH_MAX - 1); + buf[PATH_MAX - 1] = 0; + break; + } + } + + free(file_entries); + + return buf; +} diff --git a/src/flb_file_linux.c b/src/flb_file_linux.c new file mode 100644 index 00000000000..55cfa310a57 --- /dev/null +++ b/src/flb_file_linux.c @@ -0,0 +1,62 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include + +char *flb_file_get_path(flb_file_handle handle) +{ + int ret; + char *buf; + ssize_t s; + char tmp[128]; + + buf = flb_calloc(sizeof(char), PATH_MAX); + + if (buf == NULL) { + flb_errno(); + return NULL; + } + + ret = snprintf(tmp, sizeof(tmp) - 1, "/proc/%i/fd/%i", getpid(), handle); + if (ret == -1) { + flb_errno(); + flb_free(buf); + return NULL; + } + + s = readlink(tmp, buf, PATH_MAX); + + if (s == -1) { + flb_free(buf); + flb_errno(); + return NULL; + } + + buf[s] = '\0'; + + return buf; +} diff --git a/src/flb_file_macos.c b/src/flb_file_macos.c new file mode 100644 index 00000000000..6aaa0f12735 --- /dev/null +++ b/src/flb_file_macos.c @@ -0,0 +1,59 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include + +char *flb_file_get_path(flb_file_handle handle) +{ + int ret; + char *buf; + char path[PATH_MAX]; + int len; + + buf = flb_calloc(sizeof(char), PATH_MAX); + + if (buf == NULL) { + flb_errno(); + return NULL; + } + + ret = fcntl(handle, F_GETPATH, path); + + if (ret == -1) { + flb_errno(); + flb_free(buf); + return NULL; + } + + len = strlen(path); + + memcpy(buf, path, len); + + buf[len] = '\0'; + + return buf; +} diff --git a/src/flb_file_unix.c b/src/flb_file_unix.c new file mode 100644 index 00000000000..85b55e291a7 --- /dev/null +++ b/src/flb_file_unix.c @@ -0,0 +1,386 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifndef GLOB_TILDE +static char *expand_tilde(const char *path, + int *expansion_attempted) +{ + int len; + char user[256]; + char *p = NULL; + char *dir = NULL; + char *tmp = NULL; + struct passwd *uinfo = NULL; + + if (expansion_attempted != NULL) { + *expansion_attempted = FLB_TRUE; + } + + if (path[0] == '~') { + p = strchr(path, '/'); + + if (p) { + /* check case '~/' */ + if ((p - path) == 1) { + dir = getenv("HOME"); + if (!dir) { + return flb_strdup(path); + } + } + else { + /* + * it refers to a different user: ~user/abc, first step grab + * the user name. + */ + len = (p - path) - 1; + memcpy(user, path + 1, len); + user[len] = '\0'; + + /* use getpwnam() to resolve user information */ + uinfo = getpwnam(user); + if (!uinfo) { + return flb_strdup(path); + } + + dir = uinfo->pw_dir; + } + } + else { + dir = getenv("HOME"); + if (!dir) { + return flb_strdup(path); + } + } + + if (p) { + tmp = flb_malloc(PATH_MAX); + if (!tmp) { + flb_errno(); + return NULL; + } + snprintf(tmp, PATH_MAX - 1, "%s%s", dir, p); + } + else { + dir = getenv("HOME"); + if (!dir) { + return flb_strdup(path); + } + + tmp = flb_strdup(dir); + if (!tmp) { + return flb_strdup(path); + } + } + + return tmp; + } + + return flb_strdup(path); +} +#else +static char *expand_tilde(const char *path, + int *expansion_attempted) +{ + if (expansion_attempted != NULL) { + *expansion_attempted = FLB_FALSE; + } + + return flb_strdup(path); +} +#endif + +static void convert_stat_buffer(struct flb_file_stat *output_buffer, + struct stat *input_buffer) +{ + output_buffer->device = (uint64_t) input_buffer->st_dev; + output_buffer->inode = (uint64_t) input_buffer->st_ino; + output_buffer->mode = (uint16_t) input_buffer->st_mode; + output_buffer->hard_link_count = (uint16_t) input_buffer->st_nlink; + output_buffer->size = (uint64_t) input_buffer->st_size; + +#if (defined(FLB_SYSTEM_MACOS) && !defined(_POSIX_C_SOURCE)) + output_buffer->modification_time = + (int64_t) input_buffer->st_mtimespec.tv_sec; + +#elif (defined(FLB_SYSTEM_LINUX) || \ + defined(FLB_SYSTEM_FREEBSD) || \ + defined(FLB_SYSTEM_ANDROID) || \ + defined(FLB_SYSTEM_SOLARIS) || \ + _POSIX_C_SOURCE >= 200809L || \ + defined(_BSD_SOURCE) || \ + defined(_SVID_SOURCE)) + + output_buffer->modification_time = + (int64_t) input_buffer->st_mtim.tv_sec; +#else + output_buffer->modification_time = + (int64_t) input_buffer->st_mtime; +#endif +} + +flb_file_handle flb_file_open(const char *path, unsigned int flags) +{ + return open(path, flags); +} + +void flb_file_close(flb_file_handle handle) +{ + if (handle != FLB_FILE_INVALID_HANDLE) { + close(handle); + } +} + +ssize_t flb_file_read(flb_file_handle handle, + void *output_buffer, + size_t byte_count) +{ + return read(handle, output_buffer, byte_count); +} + +int64_t flb_file_lseek(flb_file_handle handle, + int64_t offset, + int reference_point) +{ + return (int64_t) lseek(handle, (off_t) offset, reference_point); +} + +int flb_file_stat(const char *path, + struct flb_file_stat *output_buffer) +{ + struct stat stat_buffer; + int result; + + result = stat(path, &stat_buffer); + + if (result != -1) { + convert_stat_buffer(output_buffer, &stat_buffer); + } + + return result; +} + +int flb_file_lstat(const char *path, + struct flb_file_stat *output_buffer) +{ + struct stat stat_buffer; + int result; + + result = lstat(path, &stat_buffer); + + if (result != -1) { + convert_stat_buffer(output_buffer, &stat_buffer); + } + + return result; +} + +int flb_file_fstat(flb_file_handle handle, + struct flb_file_stat *output_buffer) +{ + struct stat stat_buffer; + int result; + + result = fstat(handle, &stat_buffer); + + if (result != -1) { + convert_stat_buffer(output_buffer, &stat_buffer); + } + + return result; +} + +char *flb_file_basename(const char *path) +{ + char *mutable_path; + char *result; + char *name; + + mutable_path = NULL; + result = NULL; + name = NULL; + + mutable_path = flb_strdup(path); + + if (mutable_path != NULL) { + name = basename(mutable_path); + + if (name != NULL) { + result = flb_strdup(name); + + if (result == NULL) { + flb_errno(); + } + } + else { + flb_errno(); + } + + flb_free(mutable_path); + } + else { + flb_errno(); + } + + return result; +} + +struct flb_file_glob_inner_context { + glob_t results; + size_t index; + uint64_t flags; +}; + +int flb_file_glob_start(struct flb_file_glob_context *context, + const char *path, + uint64_t flags) +{ + int tilde_expansion_attempted; + struct flb_file_stat path_stat; + int result; + + if (context == NULL) { + return -1; + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + + context->inner_context = + flb_calloc(1, sizeof(struct flb_file_glob_inner_context)); + + if (context->inner_context == NULL) { + return -2; + } + + context->inner_context->flags = 0; + context->flags = flags; + + if (flags & FLB_FILE_GLOB_ABORT_ON_ERROR) { + context->inner_context->flags |= GLOB_ERR; + } + + if (flags & FLB_FILE_GLOB_EXPAND_TILDE) { + tilde_expansion_attempted = FLB_FALSE; + + context->path = expand_tilde(path, &tilde_expansion_attempted); + + if (tilde_expansion_attempted == FLB_FALSE) { + context->inner_context->flags |= GLOB_TILDE; + } + } + else { + context->path = flb_strdup(path); + } + + if (context->path == NULL) { + flb_file_glob_clean(context); + + return -3; + } + + result = glob(context->path, + context->inner_context->flags, + NULL, + &context->inner_context->results); + + if (result == GLOB_ABORTED) { + result = FLB_FILE_GLOB_ERROR_ABORTED; + } + else if (result == GLOB_NOSPACE) { + result = FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + else if (result == GLOB_NOMATCH) { + result = flb_file_stat(context->path, &path_stat); + + if (result == -1) { + result = FLB_FILE_GLOB_ERROR_NO_FILE; + } + else { + result = access(context->path, R_OK); + + if (result == -1 && errno == EACCES) { + result = FLB_FILE_GLOB_ERROR_NO_ACCESS; + } + else { + result = FLB_FILE_GLOB_ERROR_NO_MATCHES; + } + } + } + else { + result = FLB_FILE_GLOB_ERROR_SUCCESS; + } + + return result; +} + +void flb_file_glob_clean(struct flb_file_glob_context *context) +{ + if (context != NULL) { + if (context->path != NULL) { + flb_free(context->path); + } + + if (context->inner_context != NULL) { + globfree(&context->inner_context->results); + + flb_free(context->inner_context); + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + } +} + +int flb_file_glob_fetch(struct flb_file_glob_context *context, + char **result) +{ + if (context == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + if (result == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + *result = NULL; + + if (context->inner_context->index >= + context->inner_context->results.gl_pathc) { + return FLB_FILE_GLOB_ERROR_NO_MORE_RESULTS; + } + + *result = context->inner_context->results.gl_pathv[ + context->inner_context->index]; + + context->inner_context->index++; + + return FLB_FILE_GLOB_ERROR_SUCCESS; +} diff --git a/src/flb_file_win32.c b/src/flb_file_win32.c new file mode 100644 index 00000000000..d54e28c10a9 --- /dev/null +++ b/src/flb_file_win32.c @@ -0,0 +1,911 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * NTFS stat(2) emulation tailored for in_tail's usage. + * + * (1) Support st_ino (inode) for Windows NTFS. + * (2) Support NTFS symlinks. + * (3) Support large files >= 2GB. + * + * To use it, include "win32.h" and it will transparently + * replace stat(), lstat() and fstat(). + */ + +#define WINDOWS_TICKS_TO_SECONDS_RATIO 10000000 +#define WINDOWS_EPOCH_TO_UNIX_EPOCH_DELTA 11644473600 + +/* + * FILETIME timestamps are represented in 100-nanosecond intervals, + * because of this, that's why we need to divide the number by 10000000 + * in order to convert it to seconds. + * + * While UNIX timestamps use January 1, 1970 as epoch Windows FILETIME + * timestamps use January 1, 1601. Because of this we need to subtract + * 11644473600 seconds to account for it. + * + * Note: Even though this does not account for leap seconds it should be + * accurate enough. + */ + +static uint64_t filetime_to_epoch(FILETIME *ft) +{ + ULARGE_INTEGER timestamp; + + if (ft == NULL) { + return 0; + } + + timestamp.HighPart = ft->dwHighDateTime; + timestamp.LowPart = ft->dwLowDateTime; + + timestamp.QuadPart /= WINDOWS_TICKS_TO_SECONDS_RATIO; + timestamp.QuadPart -= WINDOWS_EPOCH_TO_UNIX_EPOCH_DELTA; + + return timestamp.QuadPart; +} + +static void reset_errno() +{ + errno = 0; +} + +static void propagate_last_error_to_errno() +{ + DWORD error_code; + + error_code = GetLastError(); + + switch (error_code) { + case ERROR_INVALID_TARGET_HANDLE: + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + + case ERROR_TOO_MANY_OPEN_FILES: + errno = EMFILE; + break; + + case ERROR_INVALID_FLAG_NUMBER: + case ERROR_INVALID_PARAMETER: + errno = EINVAL; + break; + + case ERROR_NOT_ENOUGH_MEMORY: + case ERROR_OUTOFMEMORY: + errno = ENOMEM; + break; + + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + case ERROR_PATH_BUSY: + case ERROR_BUSY: + errno = EBUSY; + break; + + case ERROR_HANDLE_DISK_FULL: + case ERROR_DISK_FULL: + errno = ENOSPC; + break; + + case ERROR_INVALID_ADDRESS: + errno = EFAULT; + break; + + case ERROR_FILE_TOO_LARGE: + errno = EFBIG; + break; + + case ERROR_ALREADY_EXISTS: + case ERROR_FILE_EXISTS: + errno = EEXIST; + break; + + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_INVALID_DRIVE: + case ERROR_BAD_PATHNAME: + case ERROR_INVALID_NAME: + case ERROR_BAD_UNIT: + errno = ENOENT; + break; + + case ERROR_SEEK_ON_DEVICE: + errno = ESPIPE; + break; + + case ERROR_NEGATIVE_SEEK: + errno = EINVAL; + break; + + case ERROR_ACCESS_DENIED: + errno = EACCES; + break; + + case ERROR_DIR_NOT_EMPTY: + errno = ENOTEMPTY; + break; + + case ERROR_BROKEN_PIPE: + errno = EPIPE; + break; + + case ERROR_GEN_FAILURE: + errno = EIO; + break; + + case ERROR_OPEN_FAILED: + errno = EIO; + break; + + case ERROR_SUCCESS: + errno = 0; + break; + + default: + /* This is just a canary, if you find this + * error then it means we need to expand the + * translation list. + */ + + errno = EOWNERDEAD; + break; + } +} + +static int is_symlink(const char *path) +{ + WIN32_FIND_DATA data; + HANDLE h; + + SetLastError(0); + reset_errno(); + + h = FindFirstFileA(path, &data); + + if (h == INVALID_HANDLE_VALUE) { + propagate_last_error_to_errno(); + + return 0; + } + + FindClose(h); + + /* + * A NTFS symlink is a file with a bit of metadata ("reparse point"), + * So (1) check if the file has metadata and then (2) confirm that + * it is indeed a symlink. + */ + if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + if (data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) { + return 1; + } + } + + return 0; +} + +flb_file_handle flb_file_open(const char *path, unsigned int flags) +{ + DWORD creation_disposition; + DWORD sharing_disposition; + DWORD desired_access; + HANDLE handle; + + creation_disposition = OPEN_EXISTING; + sharing_disposition = FILE_SHARE_READ | + FILE_SHARE_WRITE | + FILE_SHARE_DELETE; + desired_access = 0; + + if (flags == O_RDONLY) { + desired_access |= FILE_READ_DATA; + } + else if (flags == O_WRONLY) { + desired_access |= FILE_WRITE_DATA; + } + else if (flags == O_RDWR) { + desired_access |= FILE_READ_DATA; + desired_access |= FILE_WRITE_DATA; + } + + if ((flags & O_APPEND) != 0) { + desired_access |= FILE_APPEND_DATA; + } + + if ((flags & O_CREAT) != 0) { + if ((flags & O_EXCL) != 0) { + creation_disposition = CREATE_NEW; + } + else { + if ((flags & O_TRUNC) != 0) { + creation_disposition = CREATE_ALWAYS; + } + else { + creation_disposition = OPEN_ALWAYS; + } + } + } + else if ((flags & O_TRUNC) != 0) { + creation_disposition = TRUNCATE_EXISTING; + } + + handle = CreateFileA(path, + desired_access, + sharing_disposition, + NULL, + creation_disposition, + 0, + NULL); + + if (handle == INVALID_HANDLE_VALUE) { + return FLB_FILE_INVALID_HANDLE; + } + + return handle; +} + +void flb_file_close(flb_file_handle handle) +{ + if (handle != FLB_FILE_INVALID_HANDLE) { + CloseHandle(handle); + } +} + +ssize_t flb_file_read(flb_file_handle handle, + void *output_buffer, + size_t byte_count) +{ + DWORD bytes_read; + DWORD result; + + bytes_read = 0; + + result = ReadFile(handle, + output_buffer, + byte_count, + &bytes_read, + NULL); + + if (result == 0) { + propagate_last_error_to_errno(); + + return -1; + } + + return (ssize_t) bytes_read; +} + +int64_t flb_file_lseek(flb_file_handle handle, + int64_t offset, + int reference_point) +{ + LONG distance_high; + LONG distance_low; + DWORD result; + + distance_high = (LONG) ((offset & 0xFFFFFFFF00000000) >> 32); + distance_low = (LONG) ((offset & 0x00000000FFFFFFFF)); + + if (reference_point == SEEK_SET) { + reference_point = FILE_BEGIN; + } + else if (reference_point == SEEK_CUR) { + reference_point = FILE_CURRENT; + } + else if (reference_point == SEEK_END) { + reference_point = FILE_END; + } + else { + return -1; + } + + result = SetFilePointer(handle, + distance_low, + &distance_high, + reference_point); + + if (result == INVALID_SET_FILE_POINTER) { + propagate_last_error_to_errno(); + + return -1; + } + + offset = (int64_t) (((uint64_t) distance_high) << 32); + offset |= (int64_t) (((uint64_t) result)); + + return offset; +} + +static int flb_file_hstat(HANDLE handle, + struct flb_file_stat *output_buffer) +{ + FILE_STANDARD_INFO standard_info; + BY_HANDLE_FILE_INFORMATION handle_info; + DWORD result; + + SetLastError(0); + reset_errno(); + + result = GetFileInformationByHandle(handle, &handle_info); + + if (result == 0) { + propagate_last_error_to_errno(); + + return -1; + } + + result = GetFileInformationByHandleEx(handle, + FileStandardInfo, + &standard_info, + sizeof(standard_info)); + + if (result == 0) { + propagate_last_error_to_errno(); + + return -1; + } + + memset(output_buffer, 0, sizeof(struct flb_file_stat)); + + if (standard_info.DeletePending == 0) { + output_buffer->hard_link_count = standard_info.NumberOfLinks; + } + else { + output_buffer->hard_link_count = 0; + } + + output_buffer->mode = 0; + + if ((handle_info.dwFileAttributes & + FILE_ATTRIBUTE_DIRECTORY) != 0) { + output_buffer->mode = FLB_FILE_IFDIR; + } + else if ((handle_info.dwFileAttributes & + FILE_ATTRIBUTE_REPARSE_POINT) != 0) { + output_buffer->mode = FLB_FILE_IFLNK; + } + else { + output_buffer->mode = FLB_FILE_IFREG; + } + + output_buffer->size = (uint64_t) handle_info.nFileSizeHigh; + output_buffer->size <<= 32; + output_buffer->size |= (uint64_t) handle_info.nFileSizeLow; + + output_buffer->inode = (uint64_t) handle_info.nFileIndexHigh; + output_buffer->inode <<= 32; + output_buffer->inode |= (uint64_t) handle_info.nFileIndexLow; + + output_buffer->modification_time = + filetime_to_epoch(&handle_info.ftLastWriteTime); + + return 0; +} + +int flb_file_stat(const char *path, + struct flb_file_stat *output_buffer) +{ + HANDLE handle; + int result; + + SetLastError(0); + reset_errno(); + + handle = CreateFileA(path, + GENERIC_READ, + FILE_SHARE_READ | + FILE_SHARE_WRITE | + FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + 0, + NULL); + + if (handle == INVALID_HANDLE_VALUE) { + propagate_last_error_to_errno(); + + return -1; + } + + result = flb_file_hstat(handle, output_buffer); + + CloseHandle(handle); + + return result; +} + +int flb_file_lstat(const char *path, + struct flb_file_stat *output_buffer) +{ + HANDLE handle; + int result; + + SetLastError(0); + reset_errno(); + + handle = CreateFileA(path, + GENERIC_READ, + FILE_SHARE_READ | + FILE_SHARE_WRITE | + FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_OPEN_REPARSE_POINT, + NULL); + + if (handle == INVALID_HANDLE_VALUE) { + propagate_last_error_to_errno(); + + return -1; + } + + result = flb_file_hstat(handle, output_buffer); + + CloseHandle(handle); + + if (result != 0) { + return -1; + } + + if (is_symlink(path)) { + output_buffer->mode = FLB_FILE_IFLNK; + } + + return 0; +} + +int flb_file_fstat(flb_file_handle handle, + struct flb_file_stat *output_buffer) +{ + return flb_file_hstat(handle, output_buffer); +} + +char *flb_file_get_path(flb_file_handle handle) +{ + char *buf; + int len; + + buf = flb_calloc(sizeof(char), PATH_MAX); + + if (buf == NULL) { + flb_errno(); + return NULL; + } + + /* This function returns the length of the string excluding "\0" + * and the resulting path has a "\\?\" prefix. + */ + len = GetFinalPathNameByHandleA(handle, buf, PATH_MAX, FILE_NAME_NORMALIZED); + + if (len == 0 || len >= PATH_MAX) { + flb_free(buf); + return NULL; + } + + if (strstr(buf, "\\\\?\\")) { + memmove(buf, buf + 4, len + 1); + } + + return buf; +} + +char *flb_file_basename(const char *path) +{ + char *mutable_path; + char *result; + char *name; + + mutable_path = NULL; + result = NULL; + name = NULL; + + mutable_path = flb_strdup(path); + + if (mutable_path != NULL) { + name = basename(mutable_path); + + if (name != NULL) { + result = flb_strdup(name); + + if (result == NULL) { + flb_errno(); + } + } + else { + flb_errno(); + } + + flb_free(mutable_path); + } + else { + flb_errno(); + } + + return result; +} + + +struct flb_file_glob_inner_entry { + char *path; + struct cfl_list _head; +}; + +struct flb_file_glob_inner_context { + struct flb_file_glob_inner_entry *current_entry; + struct cfl_list results; + size_t entries; + size_t index; + uint64_t flags; +}; + +static int limited_win32_glob_append_entry( + struct flb_file_glob_inner_context *context, + char *path, + uint16_t mode_filter) +{ + char entry_path_buffer[FLB_FILE_MAX_PATH_LENGTH]; + char *entry_path; + struct flb_file_stat entry_info; + int result; + struct flb_file_glob_inner_entry *entry; + + result = flb_file_stat(path, &entry_info); + + if (result != 0) { + result = FLB_FILE_GLOB_ERROR_NO_FILE; + } + else { + result = FLB_FILE_GLOB_ERROR_SUCCESS; + + if (mode_filter != 0) { + if (!FLB_FILE_ISTYPE(entry_info.mode, mode_filter)) { + result = FLB_FILE_GLOB_ERROR_NO_MATCHES; + } + } + } + + if (result == FLB_FILE_GLOB_ERROR_SUCCESS) { + entry_path = _fullpath(entry_path_buffer, + path, + FLB_FILE_MAX_PATH_LENGTH); + + if (entry_path == NULL) { + result = FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; + } + } + + if (result == FLB_FILE_GLOB_ERROR_SUCCESS) { + entry = flb_calloc(1, sizeof(struct flb_file_glob_inner_entry)); + + if (entry == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + entry->path = flb_strdup(entry_path); + + if (entry->path == NULL) { + flb_free(entry); + + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + cfl_list_append(&entry->_head, &context->results); + + context->entries++; + } + + return result; +} + +/* + * Perform patern match on the given path string. This function + * supports patterns with "nested" wildcards like below. + * + * tail_scan_pattern("C:\fluent-bit\*\*.txt", ctx); + * + * On success, the number of files found is returned (zero indicates + * "no file found"). On error, -1 is returned. + */ +static int limited_win32_glob(struct flb_file_glob_inner_context *context, + char *path) +{ + char *star, *p0, *p1; + char pattern[FLB_FILE_MAX_PATH_LENGTH]; + char buf[FLB_FILE_MAX_PATH_LENGTH]; + int ret; + int n_added = 0; + time_t now; + int64_t mtime; + HANDLE h; + WIN32_FIND_DATA data; + struct flb_file_glob_inner_entry *entry; + int transverse_directory; + struct flb_file_stat entry_info; + + if (strlen(path) >= FLB_FILE_MAX_PATH_LENGTH) { + return FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; + } + + star = strchr(path, '*'); + + if (star == NULL) { + return limited_win32_glob_append_entry(context, path, 0); + } + + /* + * C:\data\tmp\input_*.conf + * 0<-----| + */ + p0 = star; + while (path <= p0 && *p0 != '\\') { + p0--; + } + + /* + * C:\data\tmp\input_*.conf + * |---->1 + */ + p1 = star; + while (*p1 && *p1 != '\\') { + p1++; + } + + memcpy(pattern, path, (p1 - path)); + pattern[p1 - path] = '\0'; + + h = FindFirstFileA(pattern, &data); + + if (h == INVALID_HANDLE_VALUE) { + return FLB_FILE_GLOB_ERROR_NO_MATCHES; + } + + ret = FLB_FILE_GLOB_ERROR_SUCCESS; + + do { + /* Ignore the current and parent dirs */ + if (!strcmp(".", data.cFileName) || + !strcmp("..", data.cFileName)) { + continue; + } + + /* Avoid an infinite loop */ + if (strchr(data.cFileName, '*')) { + continue; + } + + /* Create a path (prefix + filename + suffix) */ + memcpy(buf, path, p0 - path + 1); + buf[p0 - path + 1] = '\0'; + + if ((strlen(buf) + + strlen(data.cFileName) + + strlen(p1)) >= FLB_FILE_MAX_PATH_LENGTH) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + ret = FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; + + break; + } + else { + continue; + } + } + + strcat(buf, data.cFileName); + + if (strchr(p1, '*')) { + transverse_directory = FLB_FALSE; + + if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + transverse_directory = FLB_TRUE; + } + else if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + ret = flb_file_stat(data.cFileName, + &entry_info); + + if (ret != 0) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + ret = FLB_FILE_GLOB_ERROR_NO_FILE; + + break; + } + } + + if (FLB_FILE_ISDIR(entry_info.mode)) { + transverse_directory = FLB_TRUE; + } + } + + if (transverse_directory) { + strcat(buf, p1); + + ret = limited_win32_glob(context, buf); + + if (ret != FLB_FILE_GLOB_ERROR_SUCCESS && + ret != FLB_FILE_GLOB_ERROR_NO_FILE && + ret != FLB_FILE_GLOB_ERROR_NO_MATCHES) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + break; + } + } + + continue; + } + } + + strcat(buf, p1); + + ret = limited_win32_glob_append_entry(context, buf, 0); + + if (ret != FLB_FILE_GLOB_ERROR_SUCCESS && + ret != FLB_FILE_GLOB_ERROR_NO_FILE) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + break; + } + } + + ret = FLB_FILE_GLOB_ERROR_SUCCESS; + } while (FindNextFileA(h, &data) != 0); + + FindClose(h); + + if (!(context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR)) { + ret = FLB_FILE_GLOB_ERROR_SUCCESS; + } + + return ret; +} + +int flb_file_glob_start(struct flb_file_glob_context *context, + const char *path, + uint64_t flags) +{ + + int tilde_expansion_attempted; + struct flb_file_stat path_stat; + int result; + + if (context == NULL) { + return -1; + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + + context->inner_context = + flb_calloc(1, sizeof(struct flb_file_glob_inner_context)); + + if (context->inner_context == NULL) { + return -2; + } + + cfl_list_init(&context->inner_context->results); + + context->inner_context->flags = 0; + context->flags = flags; + + if (flags & FLB_FILE_GLOB_ABORT_ON_ERROR) { + context->inner_context->flags |= FLB_FILE_GLOB_ABORT_ON_ERROR; + } + + context->path = flb_strdup(path); + + if (context->path == NULL) { + flb_file_glob_clean(context); + + return -3; + } + + return limited_win32_glob(context->inner_context, + context->path); +} + +void flb_file_glob_clean(struct flb_file_glob_context *context) +{ + struct cfl_list *iterator_backup; + struct cfl_list *iterator; + struct flb_file_glob_inner_entry *entry; + + if (context != NULL) { + if (context->path != NULL) { + flb_free(context->path); + } + + if (context->inner_context != NULL) { + cfl_list_foreach_safe(iterator, + iterator_backup, + &context->inner_context->results) { + entry = cfl_list_entry(iterator, + struct flb_file_glob_inner_entry, + _head); + + if (entry->path != NULL) { + flb_free(entry->path); + } + + cfl_list_del(&entry->_head); + + flb_free(entry); + } + + flb_free(context->inner_context); + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + } + +} + +int flb_file_glob_fetch(struct flb_file_glob_context *context, + char **result) +{ + + if (context == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + if (result == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + *result = NULL; + + if (context->inner_context->index >= + context->inner_context->entries) { + return FLB_FILE_GLOB_ERROR_NO_MORE_RESULTS; + } + + if (context->inner_context->current_entry == NULL) { + context->inner_context->current_entry = + cfl_list_entry_first(&context->inner_context->results, + struct flb_file_glob_inner_entry, + _head); + } + else { + context->inner_context->current_entry = + cfl_list_entry_next(&context->inner_context->current_entry->_head, + struct flb_file_glob_inner_entry, + _head, + &context->inner_context->results); + } + + *result = context->inner_context->current_entry->path; + + context->inner_context->index++; + + return FLB_FILE_GLOB_ERROR_SUCCESS; +} diff --git a/tests/internal/file.c b/tests/internal/file.c index 974c266873e..730e2c5ceea 100644 --- a/tests/internal/file.c +++ b/tests/internal/file.c @@ -22,21 +22,21 @@ static void check_equals(flb_sds_t result, const char *expected) static void test_file_read_text_file() { - flb_sds_t result = flb_file_read(TEXT_FILE); + flb_sds_t result = flb_file_read_contents(TEXT_FILE); check_equals(result, "Some text file\n\nline 3\n\nline 5\n"); flb_sds_destroy(result); } static void test_file_read_empty_file() { - flb_sds_t result = flb_file_read(EMPTY_FILE); + flb_sds_t result = flb_file_read_contents(EMPTY_FILE); check_equals(result, ""); flb_sds_destroy(result); } static void test_file_read_missing() { - flb_sds_t result = flb_file_read(TEXT_FILE ".missing"); + flb_sds_t result = flb_file_read_contents(TEXT_FILE ".missing"); TEST_CHECK(result == NULL); }