diff --git a/include/fluent-bit/flb_file.h b/include/fluent-bit/flb_file.h index 0fbb357bcfb..61eb6acbe30 100644 --- a/include/fluent-bit/flb_file.h +++ b/include/fluent-bit/flb_file.h @@ -22,8 +22,81 @@ #define FLB_FILE_H #include +#include +#include -flb_sds_t flb_file_read(const char *path); -// TODO int flb_file_write(const char *path, flb_sds_t contents); +#ifdef FLB_SYSTEM_WINDOWS +#include +#else +#include +#endif + +#define FLB_FILE_GLOB_ABORT_ON_ERROR (((uint64_t) 1) << 0) +#define FLB_FILE_GLOB_MARK_DIRECTORIES (((uint64_t) 1) << 1) +#define FLB_FILE_GLOB_DO_NOT_SORT (((uint64_t) 1) << 2) +#define FLB_FILE_GLOB_EXPAND_TILDE (((uint64_t) 1) << 3) + +#define FLB_FILE_GLOB_ERROR_SUCCESS 0 +#define FLB_FILE_GLOB_ERROR_ABORTED 1 +#define FLB_FILE_GLOB_ERROR_NO_MEMORY 2 +#define FLB_FILE_GLOB_ERROR_NO_FILE 3 +#define FLB_FILE_GLOB_ERROR_NO_ACCESS 4 +#define FLB_FILE_GLOB_ERROR_NO_MATCHES 5 +#define FLB_FILE_GLOB_ERROR_NO_MORE_RESULTS 6 +#define FLB_FILE_GLOB_ERROR_OVERSIZED_PATH 7 +#define FLB_FILE_GLOB_ERROR_INVALID_ARGUMENT 8 + +struct flb_file_glob_inner_context; + +struct flb_file_glob_context { + struct flb_file_glob_inner_context *inner_context; + uint64_t flags; + char *path; +}; + +struct flb_file_stat { + uint64_t device; + uint64_t inode; + uint16_t mode; + int64_t modification_time; + int16_t hard_link_count; + int64_t size; +}; + +int flb_file_glob_start(struct flb_file_glob_context *context, + const char *path, + uint64_t flags); + +void flb_file_glob_clean(struct flb_file_glob_context *context); + +int flb_file_glob_fetch(struct flb_file_glob_context *context, + char **result); + +flb_file_handle flb_file_open(const char *path, + unsigned int flags); + +void flb_file_close(flb_file_handle handle); + +ssize_t flb_file_read(flb_file_handle handle, + void *output_buffer, + size_t byte_count); + +int64_t flb_file_lseek(flb_file_handle handle, + int64_t offset, + int reference_point); + +int flb_file_stat(const char *path, + struct flb_file_stat *output_buffer); + +int flb_file_lstat(const char *path, + struct flb_file_stat *output_buffer); + +int flb_file_fstat(flb_file_handle handle, + struct flb_file_stat *output_buffer); + +char *flb_file_get_path(flb_file_handle handle); + +char *flb_file_basename(const char *path); +flb_sds_t flb_file_read_contents(const char *path); #endif diff --git a/include/fluent-bit/flb_file_unix.h b/include/fluent-bit/flb_file_unix.h new file mode 100644 index 00000000000..d01ed48192a --- /dev/null +++ b/include/fluent-bit/flb_file_unix.h @@ -0,0 +1,43 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FILE_UNIX_H +#define FLB_FILE_UNIX_H + +#define FLB_FILE_IFMT S_IFMT +#define FLB_FILE_IFIFO S_IFIFO +#define FLB_FILE_IFCHR S_IFCHR +#define FLB_FILE_IFDIR S_IFDIR +#define FLB_FILE_IFBLK S_IFBLK +#define FLB_FILE_IFREG S_IFREG +#define FLB_FILE_IFLNK S_IFLNK + +#define FLB_FILE_ISDIR(m) S_ISDIR(m) +#define FLB_FILE_ISCHR(m) S_ISCHR(m) +#define FLB_FILE_ISFIFO(m) S_ISFIFO(m) +#define FLB_FILE_ISREG(m) S_ISREG(m) +#define FLB_FILE_ISLNK(m) S_ISLNK(m) + +#define FLB_FILE_INVALID_HANDLE (-1) +#define FLB_FILE_MAX_PATH_LENGTH PATH_MAX + +typedef int flb_file_handle; + +#endif diff --git a/include/fluent-bit/flb_file_win32.h b/include/fluent-bit/flb_file_win32.h new file mode 100644 index 00000000000..88fd5745a57 --- /dev/null +++ b/include/fluent-bit/flb_file_win32.h @@ -0,0 +1,44 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FILE_WIN32_H +#define FLB_FILE_WIN32_H + +#define FLB_FILE_IFMT 0170000 +#define FLB_FILE_IFIFO 0010000 +#define FLB_FILE_IFCHR 0020000 +#define FLB_FILE_IFDIR 0040000 +#define FLB_FILE_IFBLK 0060000 +#define FLB_FILE_IFREG 0100000 +#define FLB_FILE_IFLNK 0120000 + +#define FLB_FILE_ISTYPE(m, t) (((m) & FLB_FILE_IFMT) == t) +#define FLB_FILE_ISDIR(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFDIR)) +#define FLB_FILE_ISCHR(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFCHR)) +#define FLB_FILE_ISFIFO(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFIFO)) +#define FLB_FILE_ISREG(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFREG)) +#define FLB_FILE_ISLNK(m) (FLB_FILE_ISTYPE(m, FLB_FILE_IFLNK)) + +#define FLB_FILE_INVALID_HANDLE (INVALID_HANDLE_VALUE) +#define FLB_FILE_MAX_PATH_LENGTH MAX_PATH + +typedef HANDLE flb_file_handle; + +#endif diff --git a/plugins/in_node_exporter_metrics/ne_textfile_linux.c b/plugins/in_node_exporter_metrics/ne_textfile_linux.c index 33b8951cc59..66ddd51435a 100644 --- a/plugins/in_node_exporter_metrics/ne_textfile_linux.c +++ b/plugins/in_node_exporter_metrics/ne_textfile_linux.c @@ -143,7 +143,7 @@ static int textfile_update(struct flb_ne *ctx) mk_list_foreach(head, &list) { entry = mk_list_entry(head, struct flb_slist_entry, _head); /* Update metrics from text file */ - contents = flb_file_read(entry->str); + contents = flb_file_read_contents(entry->str); if (contents == NULL) { flb_plg_debug(ctx->ins, "skip invalid file of prometheus: %s", entry->str); diff --git a/plugins/out_oracle_log_analytics/oci_logan_conf.c b/plugins/out_oracle_log_analytics/oci_logan_conf.c index 1e77271431c..ed875209c3d 100644 --- a/plugins/out_oracle_log_analytics/oci_logan_conf.c +++ b/plugins/out_oracle_log_analytics/oci_logan_conf.c @@ -103,7 +103,7 @@ static int load_oci_credentials(struct flb_oci_logan *ctx) char* key = NULL; char* val; - content = flb_file_read(ctx->config_file_location); + content = flb_file_read_contents(ctx->config_file_location); if (content == NULL || flb_sds_len(content) == 0) { return -1; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 084f2c57d52..ef9fafbefc6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -82,6 +82,36 @@ set(src flb_msgpack_append_message.c ) +# File access abstraction layer +if(FLB_SYSTEM_WINDOWS) + set(src + ${src} + flb_file_win32.c + ) +else() + if(FLB_SYSTEM_LINUX) + set(src + ${src} + flb_file_linux.c + ) + elseif(FLB_SYSTEM_MACOS) + set(src + ${src} + flb_file_macos.c + ) + elseif(FLB_SYSTEM_FREEBSD) + set(src + ${src} + flb_file_freebsd.c + ) + endif() + + set(src + ${src} + flb_file_unix.c + ) +endif() + # Config format set(src ${src} diff --git a/src/flb_file.c b/src/flb_file.c index 2225bc3c927..b4e11bb9bd0 100644 --- a/src/flb_file.c +++ b/src/flb_file.c @@ -18,14 +18,16 @@ * limitations under the License. */ +#include #include #include #include #include +#include #include -flb_sds_t flb_file_read(const char *path) +flb_sds_t flb_file_read_contents(const char *path) { long flen; FILE *f = NULL; diff --git a/src/flb_file_freebsd.c b/src/flb_file_freebsd.c new file mode 100644 index 00000000000..266f3f63b59 --- /dev/null +++ b/src/flb_file_freebsd.c @@ -0,0 +1,63 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +char *flb_file_get_path(flb_file_handle handle) +{ + char *buf; + struct kinfo_file *file_entries; + int file_count; + int file_index; + + buf = flb_calloc(sizeof(char), PATH_MAX); + + if (buf == NULL) { + flb_errno(); + return NULL; + } + + if ((file_entries = kinfo_getfile(getpid(), &file_count)) == NULL) { + flb_free(buf); + return NULL; + } + + for (file_index=0; file_index < file_count; file_index++) { + if (file_entries[file_index].kf_fd == handle) { + strncpy(buf, file_entries[file_index].kf_path, PATH_MAX - 1); + buf[PATH_MAX - 1] = 0; + break; + } + } + + free(file_entries); + + return buf; +} diff --git a/src/flb_file_linux.c b/src/flb_file_linux.c new file mode 100644 index 00000000000..55cfa310a57 --- /dev/null +++ b/src/flb_file_linux.c @@ -0,0 +1,62 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include + +char *flb_file_get_path(flb_file_handle handle) +{ + int ret; + char *buf; + ssize_t s; + char tmp[128]; + + buf = flb_calloc(sizeof(char), PATH_MAX); + + if (buf == NULL) { + flb_errno(); + return NULL; + } + + ret = snprintf(tmp, sizeof(tmp) - 1, "/proc/%i/fd/%i", getpid(), handle); + if (ret == -1) { + flb_errno(); + flb_free(buf); + return NULL; + } + + s = readlink(tmp, buf, PATH_MAX); + + if (s == -1) { + flb_free(buf); + flb_errno(); + return NULL; + } + + buf[s] = '\0'; + + return buf; +} diff --git a/src/flb_file_macos.c b/src/flb_file_macos.c new file mode 100644 index 00000000000..6aaa0f12735 --- /dev/null +++ b/src/flb_file_macos.c @@ -0,0 +1,59 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include + +char *flb_file_get_path(flb_file_handle handle) +{ + int ret; + char *buf; + char path[PATH_MAX]; + int len; + + buf = flb_calloc(sizeof(char), PATH_MAX); + + if (buf == NULL) { + flb_errno(); + return NULL; + } + + ret = fcntl(handle, F_GETPATH, path); + + if (ret == -1) { + flb_errno(); + flb_free(buf); + return NULL; + } + + len = strlen(path); + + memcpy(buf, path, len); + + buf[len] = '\0'; + + return buf; +} diff --git a/src/flb_file_unix.c b/src/flb_file_unix.c new file mode 100644 index 00000000000..37e6cedaac7 --- /dev/null +++ b/src/flb_file_unix.c @@ -0,0 +1,386 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifndef GLOB_TILDE +static char *expand_tilde(const char *path, + int *expansion_attempted) +{ + int len; + char user[256]; + char *p = NULL; + char *dir = NULL; + char *tmp = NULL; + struct passwd *uinfo = NULL; + + if (expansion_attempted != NULL) { + *expansion_attempted = FLB_TRUE; + } + + if (path[0] == '~') { + p = strchr(path, '/'); + + if (p) { + /* check case '~/' */ + if ((p - path) == 1) { + dir = getenv("HOME"); + if (!dir) { + return flb_strdup(path); + } + } + else { + /* + * it refers to a different user: ~user/abc, first step grab + * the user name. + */ + len = (p - path) - 1; + memcpy(user, path + 1, len); + user[len] = '\0'; + + /* use getpwnam() to resolve user information */ + uinfo = getpwnam(user); + if (!uinfo) { + return flb_strdup(path); + } + + dir = uinfo->pw_dir; + } + } + else { + dir = getenv("HOME"); + if (!dir) { + return path; + } + } + + if (p) { + tmp = flb_malloc(PATH_MAX); + if (!tmp) { + flb_errno(); + return NULL; + } + snprintf(tmp, PATH_MAX - 1, "%s%s", dir, p); + } + else { + dir = getenv("HOME"); + if (!dir) { + return flb_strdup(path); + } + + tmp = flb_strdup(dir); + if (!tmp) { + return flb_strdup(path); + } + } + + return tmp; + } + + return flb_strdup(path); +} +#else +static char *expand_tilde(const char *path, + int *expansion_attempted) +{ + if (expansion_attempted != NULL) { + *expansion_attempted = FLB_FALSE; + } + + return flb_strdup(path); +} +#endif + +static void convert_stat_buffer(struct flb_file_stat *output_buffer, + struct stat *input_buffer) +{ + output_buffer->device = (uint64_t) input_buffer->st_dev; + output_buffer->inode = (uint64_t) input_buffer->st_ino; + output_buffer->mode = (uint16_t) input_buffer->st_mode; + output_buffer->hard_link_count = (uint16_t) input_buffer->st_nlink; + output_buffer->size = (uint64_t) input_buffer->st_size; + +#if (defined(FLB_SYSTEM_MACOS) && !defined(_POSIX_C_SOURCE)) + output_buffer->modification_time = + (int64_t) input_buffer->st_mtimespec.tv_sec; + +#elif (defined(FLB_SYSTEM_LINUX) || \ + defined(FLB_SYSTEM_FREEBSD) || \ + defined(FLB_SYSTEM_ANDROID) || \ + defined(FLB_SYSTEM_SOLARIS) || \ + _POSIX_C_SOURCE >= 200809L || \ + defined(_BSD_SOURCE) || \ + defined(_SVID_SOURCE)) + + output_buffer->modification_time = + (int64_t) input_buffer->st_mtim.tv_sec; +#else + output_buffer->modification_time = + (int64_t) input_buffer->st_mtime; +#endif +} + +flb_file_handle flb_file_open(const char *path, unsigned int flags) +{ + return open(path, flags); +} + +void flb_file_close(flb_file_handle handle) +{ + if (handle != FLB_FILE_INVALID_HANDLE) { + close(handle); + } +} + +ssize_t flb_file_read(flb_file_handle handle, + void *output_buffer, + size_t byte_count) +{ + return read(handle, output_buffer, byte_count); +} + +int64_t flb_file_lseek(flb_file_handle handle, + int64_t offset, + int reference_point) +{ + return (int64_t) lseek(handle, (off_t) offset, reference_point); +} + +int flb_file_stat(const char *path, + struct flb_file_stat *output_buffer) +{ + struct stat stat_buffer; + int result; + + result = stat(path, &stat_buffer); + + if (result != -1) { + convert_stat_buffer(output_buffer, &stat_buffer); + } + + return result; +} + +int flb_file_lstat(const char *path, + struct flb_file_stat *output_buffer) +{ + struct stat stat_buffer; + int result; + + result = lstat(path, &stat_buffer); + + if (result != -1) { + convert_stat_buffer(output_buffer, &stat_buffer); + } + + return result; +} + +int flb_file_fstat(flb_file_handle handle, + struct flb_file_stat *output_buffer) +{ + struct stat stat_buffer; + int result; + + result = fstat(handle, &stat_buffer); + + if (result != -1) { + convert_stat_buffer(output_buffer, &stat_buffer); + } + + return result; +} + +char *flb_file_basename(const char *path) +{ + char *mutable_path; + char *result; + char *name; + + mutable_path = NULL; + result = NULL; + name = NULL; + + mutable_path = flb_strdup(path); + + if (mutable_path != NULL) { + name = basename(mutable_path); + + if (name != NULL) { + result = flb_strdup(name); + + if (result == NULL) { + flb_errno(); + } + } + else { + flb_errno(); + } + + flb_free(mutable_path); + } + else { + flb_errno(); + } + + return result; +} + +struct flb_file_glob_inner_context { + glob_t results; + size_t index; + uint64_t flags; +}; + +int flb_file_glob_start(struct flb_file_glob_context *context, + const char *path, + uint64_t flags) +{ + int tilde_expansion_attempted; + struct flb_file_stat path_stat; + int result; + + if (context == NULL) { + return -1; + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + + context->inner_context = + flb_calloc(1, sizeof(struct flb_file_glob_inner_context)); + + if (context->inner_context == NULL) { + return -2; + } + + context->inner_context->flags = 0; + context->flags = flags; + + if (flags & FLB_FILE_GLOB_ABORT_ON_ERROR) { + context->inner_context->flags |= GLOB_ERR; + } + + if (flags & FLB_FILE_GLOB_EXPAND_TILDE) { + tilde_expansion_attempted = FLB_FALSE; + + context->path = expand_tilde(path, &tilde_expansion_attempted); + + if (tilde_expansion_attempted == FLB_FALSE) { + context->inner_context->flags |= GLOB_TILDE; + } + } + else { + context->path = flb_strdup(path); + } + + if (context->path == NULL) { + flb_file_glob_clean(context); + + return -3; + } + + result = glob(context->path, + context->inner_context->flags, + NULL, + &context->inner_context->results); + + if (result == GLOB_ABORTED) { + result = FLB_FILE_GLOB_ERROR_ABORTED; + } + else if (result == GLOB_NOSPACE) { + result = FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + else if (result == GLOB_NOMATCH) { + result = flb_file_stat(context->path, &path_stat); + + if (result == -1) { + result = FLB_FILE_GLOB_ERROR_NO_FILE; + } + else { + result = access(context->path, R_OK); + + if (result == -1 && errno == EACCES) { + result = FLB_FILE_GLOB_ERROR_NO_ACCESS; + } + else { + result = FLB_FILE_GLOB_ERROR_NO_MATCHES; + } + } + } + else { + result = FLB_FILE_GLOB_ERROR_SUCCESS; + } + + return result; +} + +void flb_file_glob_clean(struct flb_file_glob_context *context) +{ + if (context != NULL) { + if (context->path != NULL) { + flb_free(context->path); + } + + if (context->inner_context != NULL) { + globfree(&context->inner_context->results); + + flb_free(context->inner_context); + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + } +} + +int flb_file_glob_fetch(struct flb_file_glob_context *context, + char **result) +{ + if (context == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + if (result == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + *result = NULL; + + if (context->inner_context->index >= + context->inner_context->results.gl_pathc) { + return FLB_FILE_GLOB_ERROR_NO_MORE_RESULTS; + } + + *result = context->inner_context->results.gl_pathv[ + context->inner_context->index]; + + context->inner_context->index++; + + return FLB_FILE_GLOB_ERROR_SUCCESS; +} diff --git a/src/flb_file_win32.c b/src/flb_file_win32.c new file mode 100644 index 00000000000..d54e28c10a9 --- /dev/null +++ b/src/flb_file_win32.c @@ -0,0 +1,911 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * NTFS stat(2) emulation tailored for in_tail's usage. + * + * (1) Support st_ino (inode) for Windows NTFS. + * (2) Support NTFS symlinks. + * (3) Support large files >= 2GB. + * + * To use it, include "win32.h" and it will transparently + * replace stat(), lstat() and fstat(). + */ + +#define WINDOWS_TICKS_TO_SECONDS_RATIO 10000000 +#define WINDOWS_EPOCH_TO_UNIX_EPOCH_DELTA 11644473600 + +/* + * FILETIME timestamps are represented in 100-nanosecond intervals, + * because of this, that's why we need to divide the number by 10000000 + * in order to convert it to seconds. + * + * While UNIX timestamps use January 1, 1970 as epoch Windows FILETIME + * timestamps use January 1, 1601. Because of this we need to subtract + * 11644473600 seconds to account for it. + * + * Note: Even though this does not account for leap seconds it should be + * accurate enough. + */ + +static uint64_t filetime_to_epoch(FILETIME *ft) +{ + ULARGE_INTEGER timestamp; + + if (ft == NULL) { + return 0; + } + + timestamp.HighPart = ft->dwHighDateTime; + timestamp.LowPart = ft->dwLowDateTime; + + timestamp.QuadPart /= WINDOWS_TICKS_TO_SECONDS_RATIO; + timestamp.QuadPart -= WINDOWS_EPOCH_TO_UNIX_EPOCH_DELTA; + + return timestamp.QuadPart; +} + +static void reset_errno() +{ + errno = 0; +} + +static void propagate_last_error_to_errno() +{ + DWORD error_code; + + error_code = GetLastError(); + + switch (error_code) { + case ERROR_INVALID_TARGET_HANDLE: + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + + case ERROR_TOO_MANY_OPEN_FILES: + errno = EMFILE; + break; + + case ERROR_INVALID_FLAG_NUMBER: + case ERROR_INVALID_PARAMETER: + errno = EINVAL; + break; + + case ERROR_NOT_ENOUGH_MEMORY: + case ERROR_OUTOFMEMORY: + errno = ENOMEM; + break; + + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + case ERROR_PATH_BUSY: + case ERROR_BUSY: + errno = EBUSY; + break; + + case ERROR_HANDLE_DISK_FULL: + case ERROR_DISK_FULL: + errno = ENOSPC; + break; + + case ERROR_INVALID_ADDRESS: + errno = EFAULT; + break; + + case ERROR_FILE_TOO_LARGE: + errno = EFBIG; + break; + + case ERROR_ALREADY_EXISTS: + case ERROR_FILE_EXISTS: + errno = EEXIST; + break; + + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_INVALID_DRIVE: + case ERROR_BAD_PATHNAME: + case ERROR_INVALID_NAME: + case ERROR_BAD_UNIT: + errno = ENOENT; + break; + + case ERROR_SEEK_ON_DEVICE: + errno = ESPIPE; + break; + + case ERROR_NEGATIVE_SEEK: + errno = EINVAL; + break; + + case ERROR_ACCESS_DENIED: + errno = EACCES; + break; + + case ERROR_DIR_NOT_EMPTY: + errno = ENOTEMPTY; + break; + + case ERROR_BROKEN_PIPE: + errno = EPIPE; + break; + + case ERROR_GEN_FAILURE: + errno = EIO; + break; + + case ERROR_OPEN_FAILED: + errno = EIO; + break; + + case ERROR_SUCCESS: + errno = 0; + break; + + default: + /* This is just a canary, if you find this + * error then it means we need to expand the + * translation list. + */ + + errno = EOWNERDEAD; + break; + } +} + +static int is_symlink(const char *path) +{ + WIN32_FIND_DATA data; + HANDLE h; + + SetLastError(0); + reset_errno(); + + h = FindFirstFileA(path, &data); + + if (h == INVALID_HANDLE_VALUE) { + propagate_last_error_to_errno(); + + return 0; + } + + FindClose(h); + + /* + * A NTFS symlink is a file with a bit of metadata ("reparse point"), + * So (1) check if the file has metadata and then (2) confirm that + * it is indeed a symlink. + */ + if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + if (data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) { + return 1; + } + } + + return 0; +} + +flb_file_handle flb_file_open(const char *path, unsigned int flags) +{ + DWORD creation_disposition; + DWORD sharing_disposition; + DWORD desired_access; + HANDLE handle; + + creation_disposition = OPEN_EXISTING; + sharing_disposition = FILE_SHARE_READ | + FILE_SHARE_WRITE | + FILE_SHARE_DELETE; + desired_access = 0; + + if (flags == O_RDONLY) { + desired_access |= FILE_READ_DATA; + } + else if (flags == O_WRONLY) { + desired_access |= FILE_WRITE_DATA; + } + else if (flags == O_RDWR) { + desired_access |= FILE_READ_DATA; + desired_access |= FILE_WRITE_DATA; + } + + if ((flags & O_APPEND) != 0) { + desired_access |= FILE_APPEND_DATA; + } + + if ((flags & O_CREAT) != 0) { + if ((flags & O_EXCL) != 0) { + creation_disposition = CREATE_NEW; + } + else { + if ((flags & O_TRUNC) != 0) { + creation_disposition = CREATE_ALWAYS; + } + else { + creation_disposition = OPEN_ALWAYS; + } + } + } + else if ((flags & O_TRUNC) != 0) { + creation_disposition = TRUNCATE_EXISTING; + } + + handle = CreateFileA(path, + desired_access, + sharing_disposition, + NULL, + creation_disposition, + 0, + NULL); + + if (handle == INVALID_HANDLE_VALUE) { + return FLB_FILE_INVALID_HANDLE; + } + + return handle; +} + +void flb_file_close(flb_file_handle handle) +{ + if (handle != FLB_FILE_INVALID_HANDLE) { + CloseHandle(handle); + } +} + +ssize_t flb_file_read(flb_file_handle handle, + void *output_buffer, + size_t byte_count) +{ + DWORD bytes_read; + DWORD result; + + bytes_read = 0; + + result = ReadFile(handle, + output_buffer, + byte_count, + &bytes_read, + NULL); + + if (result == 0) { + propagate_last_error_to_errno(); + + return -1; + } + + return (ssize_t) bytes_read; +} + +int64_t flb_file_lseek(flb_file_handle handle, + int64_t offset, + int reference_point) +{ + LONG distance_high; + LONG distance_low; + DWORD result; + + distance_high = (LONG) ((offset & 0xFFFFFFFF00000000) >> 32); + distance_low = (LONG) ((offset & 0x00000000FFFFFFFF)); + + if (reference_point == SEEK_SET) { + reference_point = FILE_BEGIN; + } + else if (reference_point == SEEK_CUR) { + reference_point = FILE_CURRENT; + } + else if (reference_point == SEEK_END) { + reference_point = FILE_END; + } + else { + return -1; + } + + result = SetFilePointer(handle, + distance_low, + &distance_high, + reference_point); + + if (result == INVALID_SET_FILE_POINTER) { + propagate_last_error_to_errno(); + + return -1; + } + + offset = (int64_t) (((uint64_t) distance_high) << 32); + offset |= (int64_t) (((uint64_t) result)); + + return offset; +} + +static int flb_file_hstat(HANDLE handle, + struct flb_file_stat *output_buffer) +{ + FILE_STANDARD_INFO standard_info; + BY_HANDLE_FILE_INFORMATION handle_info; + DWORD result; + + SetLastError(0); + reset_errno(); + + result = GetFileInformationByHandle(handle, &handle_info); + + if (result == 0) { + propagate_last_error_to_errno(); + + return -1; + } + + result = GetFileInformationByHandleEx(handle, + FileStandardInfo, + &standard_info, + sizeof(standard_info)); + + if (result == 0) { + propagate_last_error_to_errno(); + + return -1; + } + + memset(output_buffer, 0, sizeof(struct flb_file_stat)); + + if (standard_info.DeletePending == 0) { + output_buffer->hard_link_count = standard_info.NumberOfLinks; + } + else { + output_buffer->hard_link_count = 0; + } + + output_buffer->mode = 0; + + if ((handle_info.dwFileAttributes & + FILE_ATTRIBUTE_DIRECTORY) != 0) { + output_buffer->mode = FLB_FILE_IFDIR; + } + else if ((handle_info.dwFileAttributes & + FILE_ATTRIBUTE_REPARSE_POINT) != 0) { + output_buffer->mode = FLB_FILE_IFLNK; + } + else { + output_buffer->mode = FLB_FILE_IFREG; + } + + output_buffer->size = (uint64_t) handle_info.nFileSizeHigh; + output_buffer->size <<= 32; + output_buffer->size |= (uint64_t) handle_info.nFileSizeLow; + + output_buffer->inode = (uint64_t) handle_info.nFileIndexHigh; + output_buffer->inode <<= 32; + output_buffer->inode |= (uint64_t) handle_info.nFileIndexLow; + + output_buffer->modification_time = + filetime_to_epoch(&handle_info.ftLastWriteTime); + + return 0; +} + +int flb_file_stat(const char *path, + struct flb_file_stat *output_buffer) +{ + HANDLE handle; + int result; + + SetLastError(0); + reset_errno(); + + handle = CreateFileA(path, + GENERIC_READ, + FILE_SHARE_READ | + FILE_SHARE_WRITE | + FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + 0, + NULL); + + if (handle == INVALID_HANDLE_VALUE) { + propagate_last_error_to_errno(); + + return -1; + } + + result = flb_file_hstat(handle, output_buffer); + + CloseHandle(handle); + + return result; +} + +int flb_file_lstat(const char *path, + struct flb_file_stat *output_buffer) +{ + HANDLE handle; + int result; + + SetLastError(0); + reset_errno(); + + handle = CreateFileA(path, + GENERIC_READ, + FILE_SHARE_READ | + FILE_SHARE_WRITE | + FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_OPEN_REPARSE_POINT, + NULL); + + if (handle == INVALID_HANDLE_VALUE) { + propagate_last_error_to_errno(); + + return -1; + } + + result = flb_file_hstat(handle, output_buffer); + + CloseHandle(handle); + + if (result != 0) { + return -1; + } + + if (is_symlink(path)) { + output_buffer->mode = FLB_FILE_IFLNK; + } + + return 0; +} + +int flb_file_fstat(flb_file_handle handle, + struct flb_file_stat *output_buffer) +{ + return flb_file_hstat(handle, output_buffer); +} + +char *flb_file_get_path(flb_file_handle handle) +{ + char *buf; + int len; + + buf = flb_calloc(sizeof(char), PATH_MAX); + + if (buf == NULL) { + flb_errno(); + return NULL; + } + + /* This function returns the length of the string excluding "\0" + * and the resulting path has a "\\?\" prefix. + */ + len = GetFinalPathNameByHandleA(handle, buf, PATH_MAX, FILE_NAME_NORMALIZED); + + if (len == 0 || len >= PATH_MAX) { + flb_free(buf); + return NULL; + } + + if (strstr(buf, "\\\\?\\")) { + memmove(buf, buf + 4, len + 1); + } + + return buf; +} + +char *flb_file_basename(const char *path) +{ + char *mutable_path; + char *result; + char *name; + + mutable_path = NULL; + result = NULL; + name = NULL; + + mutable_path = flb_strdup(path); + + if (mutable_path != NULL) { + name = basename(mutable_path); + + if (name != NULL) { + result = flb_strdup(name); + + if (result == NULL) { + flb_errno(); + } + } + else { + flb_errno(); + } + + flb_free(mutable_path); + } + else { + flb_errno(); + } + + return result; +} + + +struct flb_file_glob_inner_entry { + char *path; + struct cfl_list _head; +}; + +struct flb_file_glob_inner_context { + struct flb_file_glob_inner_entry *current_entry; + struct cfl_list results; + size_t entries; + size_t index; + uint64_t flags; +}; + +static int limited_win32_glob_append_entry( + struct flb_file_glob_inner_context *context, + char *path, + uint16_t mode_filter) +{ + char entry_path_buffer[FLB_FILE_MAX_PATH_LENGTH]; + char *entry_path; + struct flb_file_stat entry_info; + int result; + struct flb_file_glob_inner_entry *entry; + + result = flb_file_stat(path, &entry_info); + + if (result != 0) { + result = FLB_FILE_GLOB_ERROR_NO_FILE; + } + else { + result = FLB_FILE_GLOB_ERROR_SUCCESS; + + if (mode_filter != 0) { + if (!FLB_FILE_ISTYPE(entry_info.mode, mode_filter)) { + result = FLB_FILE_GLOB_ERROR_NO_MATCHES; + } + } + } + + if (result == FLB_FILE_GLOB_ERROR_SUCCESS) { + entry_path = _fullpath(entry_path_buffer, + path, + FLB_FILE_MAX_PATH_LENGTH); + + if (entry_path == NULL) { + result = FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; + } + } + + if (result == FLB_FILE_GLOB_ERROR_SUCCESS) { + entry = flb_calloc(1, sizeof(struct flb_file_glob_inner_entry)); + + if (entry == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + entry->path = flb_strdup(entry_path); + + if (entry->path == NULL) { + flb_free(entry); + + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + cfl_list_append(&entry->_head, &context->results); + + context->entries++; + } + + return result; +} + +/* + * Perform patern match on the given path string. This function + * supports patterns with "nested" wildcards like below. + * + * tail_scan_pattern("C:\fluent-bit\*\*.txt", ctx); + * + * On success, the number of files found is returned (zero indicates + * "no file found"). On error, -1 is returned. + */ +static int limited_win32_glob(struct flb_file_glob_inner_context *context, + char *path) +{ + char *star, *p0, *p1; + char pattern[FLB_FILE_MAX_PATH_LENGTH]; + char buf[FLB_FILE_MAX_PATH_LENGTH]; + int ret; + int n_added = 0; + time_t now; + int64_t mtime; + HANDLE h; + WIN32_FIND_DATA data; + struct flb_file_glob_inner_entry *entry; + int transverse_directory; + struct flb_file_stat entry_info; + + if (strlen(path) >= FLB_FILE_MAX_PATH_LENGTH) { + return FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; + } + + star = strchr(path, '*'); + + if (star == NULL) { + return limited_win32_glob_append_entry(context, path, 0); + } + + /* + * C:\data\tmp\input_*.conf + * 0<-----| + */ + p0 = star; + while (path <= p0 && *p0 != '\\') { + p0--; + } + + /* + * C:\data\tmp\input_*.conf + * |---->1 + */ + p1 = star; + while (*p1 && *p1 != '\\') { + p1++; + } + + memcpy(pattern, path, (p1 - path)); + pattern[p1 - path] = '\0'; + + h = FindFirstFileA(pattern, &data); + + if (h == INVALID_HANDLE_VALUE) { + return FLB_FILE_GLOB_ERROR_NO_MATCHES; + } + + ret = FLB_FILE_GLOB_ERROR_SUCCESS; + + do { + /* Ignore the current and parent dirs */ + if (!strcmp(".", data.cFileName) || + !strcmp("..", data.cFileName)) { + continue; + } + + /* Avoid an infinite loop */ + if (strchr(data.cFileName, '*')) { + continue; + } + + /* Create a path (prefix + filename + suffix) */ + memcpy(buf, path, p0 - path + 1); + buf[p0 - path + 1] = '\0'; + + if ((strlen(buf) + + strlen(data.cFileName) + + strlen(p1)) >= FLB_FILE_MAX_PATH_LENGTH) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + ret = FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; + + break; + } + else { + continue; + } + } + + strcat(buf, data.cFileName); + + if (strchr(p1, '*')) { + transverse_directory = FLB_FALSE; + + if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + transverse_directory = FLB_TRUE; + } + else if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + ret = flb_file_stat(data.cFileName, + &entry_info); + + if (ret != 0) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + ret = FLB_FILE_GLOB_ERROR_NO_FILE; + + break; + } + } + + if (FLB_FILE_ISDIR(entry_info.mode)) { + transverse_directory = FLB_TRUE; + } + } + + if (transverse_directory) { + strcat(buf, p1); + + ret = limited_win32_glob(context, buf); + + if (ret != FLB_FILE_GLOB_ERROR_SUCCESS && + ret != FLB_FILE_GLOB_ERROR_NO_FILE && + ret != FLB_FILE_GLOB_ERROR_NO_MATCHES) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + break; + } + } + + continue; + } + } + + strcat(buf, p1); + + ret = limited_win32_glob_append_entry(context, buf, 0); + + if (ret != FLB_FILE_GLOB_ERROR_SUCCESS && + ret != FLB_FILE_GLOB_ERROR_NO_FILE) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + break; + } + } + + ret = FLB_FILE_GLOB_ERROR_SUCCESS; + } while (FindNextFileA(h, &data) != 0); + + FindClose(h); + + if (!(context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR)) { + ret = FLB_FILE_GLOB_ERROR_SUCCESS; + } + + return ret; +} + +int flb_file_glob_start(struct flb_file_glob_context *context, + const char *path, + uint64_t flags) +{ + + int tilde_expansion_attempted; + struct flb_file_stat path_stat; + int result; + + if (context == NULL) { + return -1; + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + + context->inner_context = + flb_calloc(1, sizeof(struct flb_file_glob_inner_context)); + + if (context->inner_context == NULL) { + return -2; + } + + cfl_list_init(&context->inner_context->results); + + context->inner_context->flags = 0; + context->flags = flags; + + if (flags & FLB_FILE_GLOB_ABORT_ON_ERROR) { + context->inner_context->flags |= FLB_FILE_GLOB_ABORT_ON_ERROR; + } + + context->path = flb_strdup(path); + + if (context->path == NULL) { + flb_file_glob_clean(context); + + return -3; + } + + return limited_win32_glob(context->inner_context, + context->path); +} + +void flb_file_glob_clean(struct flb_file_glob_context *context) +{ + struct cfl_list *iterator_backup; + struct cfl_list *iterator; + struct flb_file_glob_inner_entry *entry; + + if (context != NULL) { + if (context->path != NULL) { + flb_free(context->path); + } + + if (context->inner_context != NULL) { + cfl_list_foreach_safe(iterator, + iterator_backup, + &context->inner_context->results) { + entry = cfl_list_entry(iterator, + struct flb_file_glob_inner_entry, + _head); + + if (entry->path != NULL) { + flb_free(entry->path); + } + + cfl_list_del(&entry->_head); + + flb_free(entry); + } + + flb_free(context->inner_context); + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + } + +} + +int flb_file_glob_fetch(struct flb_file_glob_context *context, + char **result) +{ + + if (context == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + if (result == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + *result = NULL; + + if (context->inner_context->index >= + context->inner_context->entries) { + return FLB_FILE_GLOB_ERROR_NO_MORE_RESULTS; + } + + if (context->inner_context->current_entry == NULL) { + context->inner_context->current_entry = + cfl_list_entry_first(&context->inner_context->results, + struct flb_file_glob_inner_entry, + _head); + } + else { + context->inner_context->current_entry = + cfl_list_entry_next(&context->inner_context->current_entry->_head, + struct flb_file_glob_inner_entry, + _head, + &context->inner_context->results); + } + + *result = context->inner_context->current_entry->path; + + context->inner_context->index++; + + return FLB_FILE_GLOB_ERROR_SUCCESS; +} diff --git a/tests/internal/file.c b/tests/internal/file.c index 3f01487bd12..05c888bcdfb 100644 --- a/tests/internal/file.c +++ b/tests/internal/file.c @@ -22,7 +22,7 @@ static void check_equals(flb_sds_t result, const char *expected) static void test_file_read_text_file() { - flb_sds_t result = flb_file_read(TEXT_FILE); + flb_sds_t result = flb_file_read_contents(TEXT_FILE); /* In Windows, \n is replaced with \r\n by git settings. */ if (strstr(result, "\r\n") != NULL) { check_equals(result, "Some text file\r\n\r\nline 3\r\n\r\nline 5\r\n"); @@ -35,14 +35,14 @@ static void test_file_read_text_file() static void test_file_read_empty_file() { - flb_sds_t result = flb_file_read(EMPTY_FILE); + flb_sds_t result = flb_file_read_contents(EMPTY_FILE); check_equals(result, ""); flb_sds_destroy(result); } static void test_file_read_missing() { - flb_sds_t result = flb_file_read(TEXT_FILE ".missing"); + flb_sds_t result = flb_file_read_contents(TEXT_FILE ".missing"); TEST_CHECK(result == NULL); }