From ef08eec6a682ffcc0afd22d096bf10c9aac426b6 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 4 May 2026 17:24:19 -0600 Subject: [PATCH 01/18] downstream_worker: add runtime helper Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_downstream_worker.h | 82 ++++++++ src/CMakeLists.txt | 1 + src/flb_downstream_worker.c | 223 +++++++++++++++++++++ 3 files changed, 306 insertions(+) create mode 100644 include/fluent-bit/flb_downstream_worker.h create mode 100644 src/flb_downstream_worker.c diff --git a/include/fluent-bit/flb_downstream_worker.h b/include/fluent-bit/flb_downstream_worker.h new file mode 100644 index 00000000000..cae95e4c845 --- /dev/null +++ b/include/fluent-bit/flb_downstream_worker.h @@ -0,0 +1,82 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_DOWNSTREAM_WORKER_H +#define FLB_DOWNSTREAM_WORKER_H + +#include + +#include + +#include + +struct flb_downstream_worker; +struct flb_downstream_worker_runtime; + +typedef int (*flb_downstream_worker_init_cb)(struct flb_downstream_worker *worker, + void *parent, + void **worker_context); + +typedef void (*flb_downstream_worker_exit_cb)(struct flb_downstream_worker *worker, + void *worker_context); + +typedef void (*flb_downstream_worker_maintenance_cb)( + struct flb_downstream_worker *worker, + void *worker_context); + +typedef void (*flb_downstream_worker_foreach_cb)(struct flb_downstream_worker *worker, + void *worker_context, + void *data); + +struct flb_downstream_worker { + struct flb_downstream_worker_runtime *runtime; + struct mk_event_loop *event_loop; + void *context; + void *parent; + int worker_id; + int worker_count; + + pthread_t thread; + pthread_mutex_t mutex; + pthread_cond_t condition; + int should_exit; + int initialized; + int thread_created; + int startup_result; +}; + +struct flb_downstream_worker_options { + int workers; + struct flb_config *config; + void *parent; + flb_downstream_worker_init_cb cb_init; + flb_downstream_worker_exit_cb cb_exit; + flb_downstream_worker_maintenance_cb cb_maintenance; +}; + +int flb_downstream_worker_runtime_start(struct flb_downstream_worker_runtime **out_runtime, + struct flb_downstream_worker_options *options); + +void flb_downstream_worker_runtime_stop(struct flb_downstream_worker_runtime *runtime); + +void flb_downstream_worker_runtime_foreach(struct flb_downstream_worker_runtime *runtime, + flb_downstream_worker_foreach_cb callback, + void *data); + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a47a5d9d658..a984ecc6b70 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -52,6 +52,7 @@ set(src flb_storage.c flb_connection.c flb_downstream.c + flb_downstream_worker.c flb_upstream.c flb_upstream_ha.c flb_upstream_node.c diff --git a/src/flb_downstream_worker.c b/src/flb_downstream_worker.c new file mode 100644 index 00000000000..0cf2869c427 --- /dev/null +++ b/src/flb_downstream_worker.c @@ -0,0 +1,223 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include + +struct flb_downstream_worker_runtime { + struct flb_downstream_worker *workers; + int worker_count; + struct flb_config *config; + void *parent; + flb_downstream_worker_init_cb cb_init; + flb_downstream_worker_exit_cb cb_exit; + flb_downstream_worker_maintenance_cb cb_maintenance; +}; + +static void downstream_worker_context_reset(struct flb_downstream_worker *worker) +{ + memset(worker, 0, sizeof(struct flb_downstream_worker)); + pthread_mutex_init(&worker->mutex, NULL); + pthread_cond_init(&worker->condition, NULL); +} + +static void downstream_worker_context_cleanup(struct flb_downstream_worker *worker) +{ + pthread_mutex_destroy(&worker->mutex); + pthread_cond_destroy(&worker->condition); +} + +static void *downstream_worker_thread(void *data) +{ + int ret; + struct mk_event *event; + struct flb_net_dns dns_ctx = {0}; + struct flb_downstream_worker *worker; + struct flb_downstream_worker_runtime *runtime; + + worker = data; + runtime = worker->runtime; + + worker->event_loop = mk_event_loop_create(256); + if (worker->event_loop == NULL) { + ret = -1; + goto signal_and_exit; + } + + flb_engine_evl_set(worker->event_loop); + flb_net_ctx_init(&dns_ctx); + flb_net_dns_ctx_set(&dns_ctx); + + ret = runtime->cb_init(worker, runtime->parent, &worker->context); + +signal_and_exit: + pthread_mutex_lock(&worker->mutex); + worker->startup_result = ret; + worker->initialized = FLB_TRUE; + pthread_cond_signal(&worker->condition); + pthread_mutex_unlock(&worker->mutex); + + if (ret != 0) { + goto cleanup; + } + + while (worker->should_exit == FLB_FALSE) { + mk_event_wait_2(worker->event_loop, 250); + + mk_event_foreach(event, worker->event_loop) { + if (event->type == FLB_ENGINE_EV_CUSTOM) { + event->handler(event); + } + } + + if (runtime->cb_maintenance != NULL) { + runtime->cb_maintenance(worker, worker->context); + } + } + +cleanup: + if (runtime->cb_exit != NULL && worker->context != NULL) { + runtime->cb_exit(worker, worker->context); + worker->context = NULL; + } + + if (worker->event_loop != NULL) { + mk_event_loop_destroy(worker->event_loop); + worker->event_loop = NULL; + } + + return NULL; +} + +int flb_downstream_worker_runtime_start(struct flb_downstream_worker_runtime **out_runtime, + struct flb_downstream_worker_options *options) +{ + int i; + int ret; + struct flb_downstream_worker_runtime *runtime; + + if (out_runtime == NULL || options == NULL || options->workers <= 0 || + options->cb_init == NULL) { + return -1; + } + + runtime = flb_calloc(1, sizeof(struct flb_downstream_worker_runtime)); + if (runtime == NULL) { + flb_errno(); + return -1; + } + + runtime->workers = flb_calloc(options->workers, + sizeof(struct flb_downstream_worker)); + if (runtime->workers == NULL) { + flb_errno(); + flb_free(runtime); + return -1; + } + + runtime->worker_count = options->workers; + runtime->config = options->config; + runtime->parent = options->parent; + runtime->cb_init = options->cb_init; + runtime->cb_exit = options->cb_exit; + runtime->cb_maintenance = options->cb_maintenance; + + *out_runtime = runtime; + + for (i = 0; i < runtime->worker_count; i++) { + downstream_worker_context_reset(&runtime->workers[i]); + runtime->workers[i].runtime = runtime; + runtime->workers[i].parent = runtime->parent; + runtime->workers[i].worker_id = i; + runtime->workers[i].worker_count = runtime->worker_count; + + ret = pthread_create(&runtime->workers[i].thread, + NULL, + downstream_worker_thread, + &runtime->workers[i]); + if (ret != 0) { + runtime->workers[i].startup_result = -1; + break; + } + + runtime->workers[i].thread_created = FLB_TRUE; + pthread_mutex_lock(&runtime->workers[i].mutex); + while (runtime->workers[i].initialized == FLB_FALSE) { + pthread_cond_wait(&runtime->workers[i].condition, + &runtime->workers[i].mutex); + } + ret = runtime->workers[i].startup_result; + pthread_mutex_unlock(&runtime->workers[i].mutex); + + if (ret != 0) { + break; + } + } + + if (i != runtime->worker_count) { + flb_downstream_worker_runtime_stop(runtime); + *out_runtime = NULL; + return -1; + } + + return 0; +} + +void flb_downstream_worker_runtime_stop(struct flb_downstream_worker_runtime *runtime) +{ + int i; + + if (runtime == NULL) { + return; + } + + for (i = 0; i < runtime->worker_count; i++) { + runtime->workers[i].should_exit = FLB_TRUE; + if (runtime->workers[i].thread_created == FLB_TRUE) { + pthread_join(runtime->workers[i].thread, NULL); + } + downstream_worker_context_cleanup(&runtime->workers[i]); + } + + flb_free(runtime->workers); + flb_free(runtime); +} + +void flb_downstream_worker_runtime_foreach(struct flb_downstream_worker_runtime *runtime, + flb_downstream_worker_foreach_cb callback, + void *data) +{ + int i; + + if (runtime == NULL || callback == NULL) { + return; + } + + for (i = 0; i < runtime->worker_count; i++) { + if (runtime->workers[i].context != NULL) { + callback(&runtime->workers[i], + runtime->workers[i].context, + data); + } + } +} From 1177cd9a55d1172655e58f1db92d357ca79c5ad8 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 4 May 2026 17:24:24 -0600 Subject: [PATCH 02/18] http_server: use downstream worker runtime Signed-off-by: Eduardo Silva --- .../fluent-bit/http_server/flb_http_server.h | 4 +- src/http_server/flb_http_server.c | 257 +++++------------- 2 files changed, 75 insertions(+), 186 deletions(-) diff --git a/include/fluent-bit/http_server/flb_http_server.h b/include/fluent-bit/http_server/flb_http_server.h index 83bb10fb90b..a32061b1e14 100755 --- a/include/fluent-bit/http_server/flb_http_server.h +++ b/include/fluent-bit/http_server/flb_http_server.h @@ -58,7 +58,7 @@ typedef int (*flb_http_server_request_processor_callback)( struct flb_http_response *response); struct flb_http_server; -struct flb_http_server_runtime; +struct flb_downstream_worker_runtime; typedef int (*flb_http_server_worker_callback)(struct flb_http_server *server, void *data); @@ -145,7 +145,7 @@ struct flb_http_server { int tls_alpn_configured; flb_http_server_worker_callback cb_worker_init; flb_http_server_worker_callback cb_worker_exit; - struct flb_http_server_runtime *runtime; + struct flb_downstream_worker_runtime *runtime; }; struct flb_http_server_session { diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index 96bf24d149f..f77c64e4dcb 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -33,42 +34,12 @@ /* PRIVATE */ struct flb_http_server_worker_context { - struct flb_http_server parent; struct flb_http_server server; struct flb_net_setup net_setup; - struct mk_event_loop *event_loop; - pthread_t thread; - pthread_mutex_t mutex; - pthread_cond_t condition; - int worker_id; - int should_exit; - int initialized; - int thread_created; - int startup_result; -}; - -struct flb_http_server_runtime { - struct flb_http_server_worker_context *workers; - int worker_count; }; static void flb_http_server_runtime_stop(struct flb_http_server *session); -static void flb_http_server_worker_context_reset( - struct flb_http_server_worker_context *worker) -{ - memset(worker, 0, sizeof(struct flb_http_server_worker_context)); - pthread_mutex_init(&worker->mutex, NULL); - pthread_cond_init(&worker->condition, NULL); -} - -static void flb_http_server_worker_context_cleanup( - struct flb_http_server_worker_context *worker) -{ - pthread_mutex_destroy(&worker->mutex); - pthread_cond_destroy(&worker->condition); -} - static int flb_http_server_running_on_caller_context( struct flb_http_server *session) { @@ -457,144 +428,116 @@ static int flb_http_server_client_connection_event_handler(void *data) return 0; } -static void flb_http_server_worker_maintenance(struct flb_config *config, - void *data) +static void flb_http_server_worker_maintenance(struct flb_downstream_worker *worker, + void *worker_context) { - struct flb_http_server_worker_context *worker; + struct flb_http_server_worker_context *context; - (void) config; + (void) worker; - worker = data; + context = worker_context; - if (worker->server.downstream != NULL) { - flb_downstream_conn_timeouts_stream(worker->server.downstream); + if (context->server.downstream != NULL) { + flb_downstream_conn_timeouts_stream(context->server.downstream); + flb_downstream_conn_pending_destroy(context->server.downstream); } } -static int flb_http_server_worker_initialize( - struct flb_http_server_worker_context *worker) +static int flb_http_server_worker_initialize(struct flb_downstream_worker *worker, + struct flb_http_server *parent, + struct flb_http_server_worker_context *context) { int result; struct flb_http_server_options options; + memcpy(&context->net_setup, + parent->networking_setup, + sizeof(struct flb_net_setup)); + context->net_setup.share_port = FLB_TRUE; + flb_http_server_options_init(&options); - options.protocol_version = worker->parent.protocol_version; - options.flags = worker->parent.flags; - options.request_callback = worker->parent.request_callback; - options.user_data = worker->parent.user_data; - options.address = worker->parent.address; - options.port = worker->parent.port; - options.tls_provider = worker->parent.tls_provider; - options.networking_flags = worker->parent.networking_flags; - options.networking_setup = &worker->net_setup; + options.protocol_version = parent->protocol_version; + options.flags = parent->flags; + options.request_callback = parent->request_callback; + options.user_data = parent->user_data; + options.address = parent->address; + options.port = parent->port; + options.tls_provider = parent->tls_provider; + options.networking_flags = parent->networking_flags; + options.networking_setup = &context->net_setup; options.event_loop = worker->event_loop; - options.system_context = worker->parent.system_context; - options.buffer_max_size = worker->parent.buffer_max_size; + options.system_context = parent->system_context; + options.buffer_max_size = parent->buffer_max_size; options.workers = 1; options.use_caller_event_loop = FLB_TRUE; - options.reuse_port = worker->parent.reuse_port; - options.cb_worker_init = worker->parent.cb_worker_init; - options.cb_worker_exit = worker->parent.cb_worker_exit; + options.reuse_port = FLB_TRUE; + options.cb_worker_init = parent->cb_worker_init; + options.cb_worker_exit = parent->cb_worker_exit; - result = flb_http_server_init_with_options(&worker->server, &options); + result = flb_http_server_init_with_options(&context->server, &options); if (result != 0) { return result; } - result = flb_http_server_start(&worker->server); + result = flb_http_server_start(&context->server); if (result != 0) { return result; } - flb_downstream_thread_safe(worker->server.downstream); + flb_downstream_thread_safe(context->server.downstream); - worker->server.worker_id = worker->worker_id; - worker->server.workers = worker->parent.workers; + context->server.worker_id = worker->worker_id; + context->server.workers = parent->workers; return 0; } -static void *flb_http_server_worker_thread(void *data) +static int flb_http_server_worker_init(struct flb_downstream_worker *worker, + void *parent, + void **worker_context) { - int result; - struct mk_event *event; - struct flb_net_dns dns_ctx = {0}; - struct flb_http_server_worker_context *worker; + int ret; + struct flb_http_server *parent_server; + struct flb_http_server_worker_context *context; - worker = data; + parent_server = parent; - worker->event_loop = mk_event_loop_create(256); - if (worker->event_loop == NULL) { - result = -1; - goto signal_and_exit; + context = flb_calloc(1, sizeof(struct flb_http_server_worker_context)); + if (context == NULL) { + flb_errno(); + return -1; } - flb_engine_evl_set(worker->event_loop); - flb_net_ctx_init(&dns_ctx); - flb_net_dns_ctx_set(&dns_ctx); - - result = flb_http_server_worker_initialize(worker); - -signal_and_exit: - pthread_mutex_lock(&worker->mutex); - worker->startup_result = result; - worker->initialized = FLB_TRUE; - pthread_cond_signal(&worker->condition); - pthread_mutex_unlock(&worker->mutex); - - if (result != 0) { - goto cleanup; + ret = flb_http_server_worker_initialize(worker, parent_server, context); + if (ret != 0) { + flb_free(context); + return ret; } - while (worker->should_exit == FLB_FALSE) { - mk_event_wait_2(worker->event_loop, 250); + *worker_context = context; - mk_event_foreach(event, worker->event_loop) { - if (event->type == FLB_ENGINE_EV_CUSTOM) { - event->handler(event); - } - } + return 0; +} - flb_http_server_worker_maintenance(worker->parent.system_context, - worker); - flb_downstream_conn_pending_destroy(worker->server.downstream); - } +static void flb_http_server_worker_exit(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_http_server_worker_context *context; -cleanup: - flb_http_server_destroy(&worker->server); + (void) worker; - if (worker->event_loop != NULL) { - mk_event_loop_destroy(worker->event_loop); - worker->event_loop = NULL; - } + context = worker_context; - return NULL; + flb_http_server_destroy(&context->server); + flb_free(context); } static int flb_http_server_runtime_start(struct flb_http_server *session) { const char *alpn; - int index; int result; - struct flb_http_server_runtime *runtime; - - runtime = flb_calloc(1, sizeof(struct flb_http_server_runtime)); - if (runtime == NULL) { - flb_errno(); - return -1; - } - - runtime->workers = flb_calloc(session->workers, - sizeof(struct flb_http_server_worker_context)); - if (runtime->workers == NULL) { - flb_errno(); - flb_free(runtime); - return -1; - } - - runtime->worker_count = session->workers; - session->runtime = runtime; + struct flb_downstream_worker_options options; if (session->tls_provider != NULL && session->tls_alpn_configured == FLB_FALSE) { @@ -602,56 +545,22 @@ static int flb_http_server_runtime_start(struct flb_http_server *session) result = flb_tls_set_alpn(session->tls_provider, alpn); if (result != 0) { - flb_free(runtime->workers); - flb_free(runtime); - session->runtime = NULL; - return -1; } session->tls_alpn_configured = FLB_TRUE; } - for (index = 0; index < runtime->worker_count; index++) { - flb_http_server_worker_context_reset(&runtime->workers[index]); - memcpy(&runtime->workers[index].parent, - session, - sizeof(struct flb_http_server)); - memcpy(&runtime->workers[index].net_setup, - session->networking_setup, - sizeof(struct flb_net_setup)); - - runtime->workers[index].net_setup.share_port = FLB_TRUE; - runtime->workers[index].worker_id = index; - runtime->workers[index].parent.reuse_port = FLB_TRUE; - runtime->workers[index].parent.runtime = NULL; - runtime->workers[index].parent.workers = session->workers; - - result = pthread_create(&runtime->workers[index].thread, - NULL, - flb_http_server_worker_thread, - &runtime->workers[index]); - if (result != 0) { - runtime->workers[index].startup_result = -1; - break; - } - runtime->workers[index].thread_created = FLB_TRUE; - - pthread_mutex_lock(&runtime->workers[index].mutex); - while (runtime->workers[index].initialized == FLB_FALSE) { - pthread_cond_wait(&runtime->workers[index].condition, - &runtime->workers[index].mutex); - } - result = runtime->workers[index].startup_result; - pthread_mutex_unlock(&runtime->workers[index].mutex); - - if (result != 0) { - break; - } - } + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = session->workers; + options.config = session->system_context; + options.parent = session; + options.cb_init = flb_http_server_worker_init; + options.cb_exit = flb_http_server_worker_exit; + options.cb_maintenance = flb_http_server_worker_maintenance; - if (index != runtime->worker_count) { - flb_http_server_runtime_stop(session); + result = flb_downstream_worker_runtime_start(&session->runtime, &options); + if (result != 0) { return -1; } @@ -662,27 +571,7 @@ static int flb_http_server_runtime_start(struct flb_http_server *session) static void flb_http_server_runtime_stop(struct flb_http_server *session) { - int index; - struct flb_http_server_runtime *runtime; - - runtime = session->runtime; - if (runtime == NULL) { - return; - } - - for (index = 0; index < runtime->worker_count; index++) { - runtime->workers[index].should_exit = FLB_TRUE; - - if (runtime->workers[index].thread_created == FLB_TRUE) { - pthread_join(runtime->workers[index].thread, NULL); - } - - flb_http_server_worker_context_cleanup(&runtime->workers[index]); - } - - flb_free(runtime->workers); - flb_free(runtime); - + flb_downstream_worker_runtime_stop(session->runtime); session->runtime = NULL; } From 23e0ad236be573fd2d7dca8756d00c7961e78295 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 4 May 2026 17:24:28 -0600 Subject: [PATCH 03/18] in_tcp: add worker support Signed-off-by: Eduardo Silva --- plugins/in_tcp/tcp.c | 308 +++++++++++++++++++++++++++++++----- plugins/in_tcp/tcp.h | 21 +++ plugins/in_tcp/tcp_config.c | 6 + plugins/in_tcp/tcp_conn.c | 12 +- 4 files changed, 299 insertions(+), 48 deletions(-) diff --git a/plugins/in_tcp/tcp.c b/plugins/in_tcp/tcp.c index 6095ba766eb..755039f28a6 100644 --- a/plugins/in_tcp/tcp.c +++ b/plugins/in_tcp/tcp.c @@ -18,13 +18,223 @@ */ #include +#include #include +#include +#include #include #include "tcp.h" #include "tcp_conn.h" #include "tcp_config.h" +static int in_tcp_collect_ctx(struct flb_in_tcp_config *ctx); +static int in_tcp_collect(struct flb_input_instance *in, + struct flb_config *config, void *in_context); + +static void in_tcp_connections_destroy(struct flb_in_tcp_config *ctx) +{ + struct mk_list *tmp; + struct mk_list *head; + struct tcp_conn *conn; + + mk_list_foreach_safe(head, tmp, &ctx->connections) { + conn = mk_list_entry(head, struct tcp_conn, _head); + tcp_conn_del(conn); + } +} + +static int in_tcp_worker_listener_event(void *data) +{ + struct mk_event *event; + + event = data; + + return in_tcp_collect_ctx(event->data); +} + +static int in_tcp_start_listener(struct flb_in_tcp_config *ctx, + struct flb_config *config, + struct flb_net_setup *net_setup, + struct mk_event_loop *event_loop, + int use_collector) +{ + int ret; + unsigned short int port; + + port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); + + ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, + ctx->ins->flags, + ctx->listen, + port, + ctx->ins->tls, + config, + net_setup); + + if (ctx->downstream == NULL) { + flb_plg_error(ctx->ins, + "could not initialize downstream on %s:%s. Aborting", + ctx->listen, ctx->tcp_port); + return -1; + } + + flb_input_downstream_set(ctx->downstream, ctx->ins); + + if (use_collector == FLB_TRUE) { + ret = flb_input_set_collector_socket(ctx->ins, + in_tcp_collect, + ctx->downstream->server_fd, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); + return -1; + } + + ctx->collector_id = ret; + } + else { + ctx->event_loop = event_loop; + MK_EVENT_NEW(&ctx->listener_event); + ctx->listener_event.type = FLB_ENGINE_EV_CUSTOM; + ctx->listener_event.data = ctx; + ctx->listener_event.handler = in_tcp_worker_listener_event; + + ret = mk_event_add(event_loop, + ctx->downstream->server_fd, + FLB_ENGINE_EV_CUSTOM, + MK_EVENT_READ, + &ctx->listener_event); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not register TCP worker listener"); + return -1; + } + + ctx->listener_registered = FLB_TRUE; + } + + return 0; +} + +static int in_tcp_worker_init(struct flb_downstream_worker *worker, + void *parent, + void **worker_context) +{ + int ret; + struct flb_in_tcp_config *ctx; + struct flb_in_tcp_config *parent_ctx; + + parent_ctx = parent; + + ctx = tcp_config_init(parent_ctx->ins); + if (ctx == NULL) { + return -1; + } + + *worker_context = ctx; + + ctx->collector_id = -1; + ctx->ins = parent_ctx->ins; + ctx->workers = parent_ctx->workers; + ctx->worker_id = worker->worker_id; + ctx->use_ingress_queue = FLB_TRUE; + ctx->net_setup = parent_ctx->ins->net_setup; + ctx->net_setup.share_port = FLB_TRUE; + mk_list_init(&ctx->connections); + + ret = in_tcp_start_listener(ctx, + parent_ctx->ins->config, + &ctx->net_setup, + worker->event_loop, + FLB_FALSE); + if (ret == 0) { + flb_downstream_thread_safe(ctx->downstream); + } + + return ret; +} + +static void in_tcp_worker_exit(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_tcp_config *ctx; + + (void) worker; + + ctx = worker_context; + + in_tcp_connections_destroy(ctx); + tcp_config_destroy(ctx); +} + +static void in_tcp_worker_maintenance(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_tcp_config *ctx; + + (void) worker; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_conn_timeouts_stream(ctx->downstream); + flb_downstream_conn_pending_destroy(ctx->downstream); + } +} + +static int in_tcp_workers_start(struct flb_in_tcp_config *ctx) +{ + struct flb_downstream_worker_options options; + + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = ctx->workers; + options.config = ctx->ins->config; + options.parent = ctx; + options.cb_init = in_tcp_worker_init; + options.cb_exit = in_tcp_worker_exit; + options.cb_maintenance = in_tcp_worker_maintenance; + + return flb_downstream_worker_runtime_start(&ctx->runtime, &options); +} + +static void in_tcp_workers_stop(struct flb_in_tcp_config *ctx) +{ + flb_downstream_worker_runtime_stop(ctx->runtime); + ctx->runtime = NULL; +} + +static void in_tcp_worker_pause(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct flb_in_tcp_config *ctx; + + (void) worker; + (void) data; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_pause(ctx->downstream); + } +} + +static void in_tcp_worker_resume(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct flb_in_tcp_config *ctx; + + (void) worker; + (void) data; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_resume(ctx->downstream); + } +} + /* * For a server event, the collection event means a new client have arrived, we * accept the connection and create a new TCP instance which will wait for @@ -32,12 +242,17 @@ */ static int in_tcp_collect(struct flb_input_instance *in, struct flb_config *config, void *in_context) +{ + (void) in; + (void) config; + + return in_tcp_collect_ctx(in_context); +} + +static int in_tcp_collect_ctx(struct flb_in_tcp_config *ctx) { struct flb_connection *connection; struct tcp_conn *conn; - struct flb_in_tcp_config *ctx; - - ctx = in_context; connection = flb_downstream_conn_get(ctx->downstream); @@ -65,7 +280,6 @@ static int in_tcp_collect(struct flb_input_instance *in, static int in_tcp_init(struct flb_input_instance *in, struct flb_config *config, void *data) { - unsigned short int port; int ret; struct flb_in_tcp_config *ctx; @@ -83,61 +297,52 @@ static int in_tcp_init(struct flb_input_instance *in, /* Set the context */ flb_input_set_context(in, ctx); - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - in->flags, - ctx->listen, - port, - in->tls, - config, - &in->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - tcp_config_destroy(ctx); - - return -1; + if (ctx->workers <= 0) { + ctx->workers = 1; } - flb_input_downstream_set(ctx->downstream, ctx->ins); - - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(in, - in_tcp_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_TCP input plugin"); - tcp_config_destroy(ctx); + if (ctx->workers > 1) { + ret = flb_input_ingress_enable(in); + if (ret != 0) { + tcp_config_destroy(ctx); + return -1; + } - return -1; + ret = in_tcp_workers_start(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not start TCP listener workers on %s:%s. Aborting", + ctx->listen, ctx->tcp_port); + tcp_config_destroy(ctx); + return -1; + } + } + else { + ret = in_tcp_start_listener(ctx, config, &in->net_setup, NULL, FLB_TRUE); + if (ret != 0) { + tcp_config_destroy(ctx); + return -1; + } } - ctx->collector_id = ret; + flb_plg_info(ctx->ins, + "listening on %s:%s with %i worker%s", + ctx->listen, ctx->tcp_port, ctx->workers, + ctx->workers == 1 ? "" : "s"); return 0; } static int in_tcp_exit(void *data, struct flb_config *config) { - struct mk_list *tmp; - struct mk_list *head; struct flb_in_tcp_config *ctx; - struct tcp_conn *conn; (void) *config; ctx = data; - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct tcp_conn, _head); - - tcp_conn_del(conn); - } + in_tcp_workers_stop(ctx); + in_tcp_connections_destroy(ctx); tcp_config_destroy(ctx); @@ -153,6 +358,13 @@ static void in_tcp_pause(void *data, struct flb_config *config) (void) config; + if (ctx->runtime != NULL) { + flb_downstream_worker_runtime_foreach(ctx->runtime, + in_tcp_worker_pause, + NULL); + return; + } + flb_downstream_pause(ctx->downstream); mk_list_foreach_safe(head, tmp, &ctx->connections) { @@ -172,6 +384,13 @@ static void in_tcp_resume(void *data, struct flb_config *config) (void) config; + if (ctx->runtime != NULL) { + flb_downstream_worker_runtime_foreach(ctx->runtime, + in_tcp_worker_resume, + NULL); + return; + } + flb_downstream_resume(ctx->downstream); } @@ -206,6 +425,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_in_tcp_config, source_address_key), "Key where the source address will be injected" }, + { + FLB_CONFIG_MAP_INT, "workers", "1", + 0, FLB_TRUE, offsetof(struct flb_in_tcp_config, workers), + "Set the number of TCP listener workers" + }, /* EOF */ {0} }; diff --git a/plugins/in_tcp/tcp.h b/plugins/in_tcp/tcp.h index 8700455c24b..5da1affc94c 100644 --- a/plugins/in_tcp/tcp.h +++ b/plugins/in_tcp/tcp.h @@ -27,11 +27,14 @@ #include #include #include +#include #ifdef FLB_HAVE_PARSER #include #endif #include +struct flb_downstream_worker_runtime; + struct flb_in_tcp_config { flb_sds_t format_name; /* Data format name */ int format; /* Data format */ @@ -51,10 +54,28 @@ struct flb_in_tcp_config { void *parser; #endif int collector_id; /* Listener collector id */ + int workers; /* Listener worker count */ + int worker_id; /* Worker id */ + int use_ingress_queue; /* Queue records to main loop */ + int listener_registered; /* Listener event registered */ + struct mk_event listener_event; /* Worker listener event */ + struct mk_event_loop *event_loop; /* Worker event loop */ + struct flb_net_setup net_setup; /* Worker network setup */ struct flb_downstream *downstream; /* Client manager */ struct mk_list connections; /* List of active connections */ struct flb_input_instance *ins; /* Input plugin instace */ struct flb_log_event_encoder *log_encoder; + struct flb_downstream_worker_runtime *runtime; }; +static inline int tcp_ingest_logs(struct flb_in_tcp_config *ctx, + const void *buf, size_t buf_size) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_log(ctx->ins, NULL, 0, buf, buf_size); + } + + return flb_input_log_append(ctx->ins, NULL, 0, buf, buf_size); +} + #endif diff --git a/plugins/in_tcp/tcp_config.c b/plugins/in_tcp/tcp_config.c index 78c9915833a..0204d9821e8 100644 --- a/plugins/in_tcp/tcp_config.c +++ b/plugins/in_tcp/tcp_config.c @@ -43,6 +43,7 @@ struct flb_in_tcp_config *tcp_config_init(struct flb_input_instance *ins) } ctx->ins = ins; ctx->format = FLB_TCP_FMT_JSON; + ctx->workers = 1; /* Load the config map */ ret = flb_input_config_map_set(ins, (void *)ctx); @@ -168,6 +169,11 @@ int tcp_config_destroy(struct flb_in_tcp_config *ctx) ctx->collector_id = -1; } + if (ctx->listener_registered == FLB_TRUE && ctx->event_loop != NULL) { + mk_event_del(ctx->event_loop, &ctx->listener_event); + ctx->listener_registered = FLB_FALSE; + } + if (ctx->downstream != NULL) { flb_downstream_destroy(ctx->downstream); } diff --git a/plugins/in_tcp/tcp_conn.c b/plugins/in_tcp/tcp_conn.c index ddd15e57ab1..df95df1cf03 100644 --- a/plugins/in_tcp/tcp_conn.c +++ b/plugins/in_tcp/tcp_conn.c @@ -140,9 +140,9 @@ static inline int process_pack(struct tcp_conn *conn, if (ret == FLB_EVENT_ENCODER_SUCCESS) { if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); + tcp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); } ret = 0; } @@ -384,9 +384,9 @@ static ssize_t parse_payload_none(struct tcp_conn *conn) if (ret == FLB_EVENT_ENCODER_SUCCESS) { if (ctx->log_encoder->output_length > 0) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); + tcp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); } } else { From 97d6f2f027199b8621c600e0540100c3d639b576 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 4 May 2026 17:24:33 -0600 Subject: [PATCH 04/18] in_udp: add worker support Signed-off-by: Eduardo Silva --- plugins/in_udp/udp.c | 290 ++++++++++++++++++++++++++++-------- plugins/in_udp/udp.h | 20 +++ plugins/in_udp/udp_config.c | 6 + plugins/in_udp/udp_conn.c | 12 +- 4 files changed, 261 insertions(+), 67 deletions(-) diff --git a/plugins/in_udp/udp.c b/plugins/in_udp/udp.c index c3e9717e066..cdaca20c157 100644 --- a/plugins/in_udp/udp.c +++ b/plugins/in_udp/udp.c @@ -18,121 +18,285 @@ */ #include +#include #include +#include +#include #include #include "udp.h" #include "udp_conn.h" #include "udp_config.h" +static int in_udp_collect_ctx(struct flb_in_udp_config *ctx); static int in_udp_collect(struct flb_input_instance *in, struct flb_config *config, - void *in_context) -{ - struct flb_connection *connection; - struct flb_in_udp_config *ctx; - - ctx = in_context; + void *in_context); - connection = flb_downstream_conn_get(ctx->downstream); - - if (connection == NULL) { - flb_plg_error(ctx->ins, "could get UDP server dummy connection"); +static int in_udp_worker_listener_event(void *data) +{ + struct mk_event *event; - return -1; - } + event = data; - return udp_conn_event(connection); + return in_udp_collect_ctx(event->data); } -/* Initialize plugin */ -static int in_udp_init(struct flb_input_instance *in, - struct flb_config *config, void *data) +static void in_udp_dummy_conn_destroy(struct flb_in_udp_config *ctx) { - struct flb_connection *connection; - unsigned short int port; - int ret; - struct flb_in_udp_config *ctx; - - (void) data; - - /* Allocate space for the configuration */ - ctx = udp_config_init(in); - - if (ctx == NULL) { - return -1; + if (ctx->dummy_conn != NULL) { + udp_conn_del(ctx->dummy_conn); + ctx->dummy_conn = NULL; } +} - ctx->collector_id = -1; - ctx->ins = in; - - /* Set the context */ - flb_input_set_context(in, ctx); +static int in_udp_start_listener(struct flb_in_udp_config *ctx, + struct flb_config *config, + struct flb_net_setup *net_setup, + struct mk_event_loop *event_loop, + int use_collector) +{ + int ret; + unsigned short int port; + struct flb_connection *connection; port = (unsigned short int) strtoul(ctx->port, NULL, 10); ctx->downstream = flb_downstream_create(FLB_TRANSPORT_UDP, - in->flags, + ctx->ins->flags, ctx->listen, port, - in->tls, + ctx->ins->tls, config, - &in->net_setup); + net_setup); if (ctx->downstream == NULL) { flb_plg_error(ctx->ins, "could not initialize downstream on %s:%s. Aborting", ctx->listen, ctx->port); - - udp_config_destroy(ctx); - return -1; } flb_input_downstream_set(ctx->downstream, ctx->ins); connection = flb_downstream_conn_get(ctx->downstream); - if (connection == NULL) { flb_plg_error(ctx->ins, "could not get UDP server dummy connection"); - - udp_config_destroy(ctx); - return -1; } ctx->dummy_conn = udp_conn_add(connection, ctx); - if (ctx->dummy_conn == NULL) { flb_plg_error(ctx->ins, "could not track UDP server dummy connection"); + return -1; + } + + if (use_collector == FLB_TRUE) { + ret = flb_input_set_collector_socket(ctx->ins, + in_udp_collect, + ctx->downstream->server_fd, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "Could not set collector for IN_UDP input plugin"); + return -1; + } + + ctx->collector_id = ret; + ctx->collector_event = flb_input_collector_get_event(ret, ctx->ins); + + if (ctx->collector_event == NULL) { + flb_plg_error(ctx->ins, "Could not get collector event"); + return -1; + } + } + else { + ctx->event_loop = event_loop; + MK_EVENT_NEW(&ctx->listener_event); + ctx->listener_event.type = FLB_ENGINE_EV_CUSTOM; + ctx->listener_event.data = ctx; + ctx->listener_event.handler = in_udp_worker_listener_event; + + ret = mk_event_add(event_loop, + ctx->downstream->server_fd, + FLB_ENGINE_EV_CUSTOM, + MK_EVENT_READ, + &ctx->listener_event); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not register UDP worker listener"); + return -1; + } + + ctx->listener_registered = FLB_TRUE; + } + + return 0; +} + +static int in_udp_worker_init(struct flb_downstream_worker *worker, + void *parent, + void **worker_context) +{ + int ret; + struct flb_in_udp_config *ctx; + struct flb_in_udp_config *parent_ctx; - udp_config_destroy(ctx); + parent_ctx = parent; + ctx = udp_config_init(parent_ctx->ins); + if (ctx == NULL) { return -1; } - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(in, - in_udp_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not set collector for IN_UDP input plugin"); - udp_config_destroy(ctx); + *worker_context = ctx; + + ctx->collector_id = -1; + ctx->ins = parent_ctx->ins; + ctx->workers = parent_ctx->workers; + ctx->worker_id = worker->worker_id; + ctx->use_ingress_queue = FLB_TRUE; + ctx->net_setup = parent_ctx->ins->net_setup; + ctx->net_setup.share_port = FLB_TRUE; + + ret = in_udp_start_listener(ctx, + parent_ctx->ins->config, + &ctx->net_setup, + worker->event_loop, + FLB_FALSE); + if (ret == 0) { + flb_downstream_thread_safe(ctx->downstream); + } + + return ret; +} + +static void in_udp_worker_exit(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_udp_config *ctx; + + (void) worker; + + ctx = worker_context; + + in_udp_dummy_conn_destroy(ctx); + udp_config_destroy(ctx); +} + +static void in_udp_worker_maintenance(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_udp_config *ctx; + + (void) worker; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_conn_pending_destroy(ctx->downstream); + } +} + +static int in_udp_workers_start(struct flb_in_udp_config *ctx) +{ + struct flb_downstream_worker_options options; + + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = ctx->workers; + options.config = ctx->ins->config; + options.parent = ctx; + options.cb_init = in_udp_worker_init; + options.cb_exit = in_udp_worker_exit; + options.cb_maintenance = in_udp_worker_maintenance; + + return flb_downstream_worker_runtime_start(&ctx->runtime, &options); +} + +static void in_udp_workers_stop(struct flb_in_udp_config *ctx) +{ + flb_downstream_worker_runtime_stop(ctx->runtime); + ctx->runtime = NULL; +} + +static int in_udp_collect(struct flb_input_instance *in, + struct flb_config *config, + void *in_context) +{ + (void) in; + (void) config; + + return in_udp_collect_ctx(in_context); +} + +static int in_udp_collect_ctx(struct flb_in_udp_config *ctx) +{ + struct flb_connection *connection; + + connection = flb_downstream_conn_get(ctx->downstream); + + if (connection == NULL) { + flb_plg_error(ctx->ins, "could get UDP server dummy connection"); return -1; } - ctx->collector_id = ret; - ctx->collector_event = flb_input_collector_get_event(ret, in); + return udp_conn_event(connection); +} + +/* Initialize plugin */ +static int in_udp_init(struct flb_input_instance *in, + struct flb_config *config, void *data) +{ + int ret; + struct flb_in_udp_config *ctx; + + (void) data; - if (ret == -1) { - flb_plg_error(ctx->ins, "Could not get collector event"); - udp_config_destroy(ctx); + /* Allocate space for the configuration */ + ctx = udp_config_init(in); + if (ctx == NULL) { return -1; } + ctx->collector_id = -1; + ctx->ins = in; + + /* Set the context */ + flb_input_set_context(in, ctx); + + if (ctx->workers <= 0) { + ctx->workers = 1; + } + + if (ctx->workers > 1) { + ret = flb_input_ingress_enable(in); + if (ret != 0) { + udp_config_destroy(ctx); + return -1; + } + + ret = in_udp_workers_start(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not start UDP listener workers on %s:%s. Aborting", + ctx->listen, ctx->port); + udp_config_destroy(ctx); + return -1; + } + } + else { + ret = in_udp_start_listener(ctx, config, &in->net_setup, NULL, FLB_TRUE); + if (ret != 0) { + udp_config_destroy(ctx); + return -1; + } + } + + flb_plg_info(ctx->ins, + "listening on %s:%s with %i worker%s", + ctx->listen, ctx->port, ctx->workers, + ctx->workers == 1 ? "" : "s"); + return 0; } @@ -144,9 +308,8 @@ static int in_udp_exit(void *data, struct flb_config *config) ctx = data; - if (ctx->dummy_conn != NULL) { - udp_conn_del(ctx->dummy_conn); - } + in_udp_workers_stop(ctx); + in_udp_dummy_conn_destroy(ctx); udp_config_destroy(ctx); @@ -184,6 +347,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_in_udp_config, source_address_key), "Key where the source address will be injected" }, + { + FLB_CONFIG_MAP_INT, "workers", "1", + 0, FLB_TRUE, offsetof(struct flb_in_udp_config, workers), + "Set the number of UDP listener workers" + }, /* EOF */ {0} }; diff --git a/plugins/in_udp/udp.h b/plugins/in_udp/udp.h index 16943a92bcf..da04cdb0ffa 100644 --- a/plugins/in_udp/udp.h +++ b/plugins/in_udp/udp.h @@ -27,12 +27,14 @@ #include #include #include +#include #ifdef FLB_HAVE_PARSER #include #endif #include struct udp_conn; +struct flb_downstream_worker_runtime; struct flb_in_udp_config { struct mk_event *collector_event; @@ -54,10 +56,28 @@ struct flb_in_udp_config { void *parser; #endif int collector_id; /* Listener collector id */ + int workers; /* Listener worker count */ + int worker_id; /* Worker id */ + int use_ingress_queue; /* Queue records to main loop */ + int listener_registered; /* Listener event registered */ + struct mk_event listener_event; /* Worker listener event */ + struct mk_event_loop *event_loop; /* Worker event loop */ + struct flb_net_setup net_setup; /* Worker network setup */ struct flb_downstream *downstream; /* Client manager */ struct udp_conn *dummy_conn; /* Datagram dummy connection */ struct flb_input_instance *ins; /* Input plugin instace */ struct flb_log_event_encoder *log_encoder; + struct flb_downstream_worker_runtime *runtime; }; +static inline int udp_ingest_logs(struct flb_in_udp_config *ctx, + const void *buf, size_t buf_size) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_log(ctx->ins, NULL, 0, buf, buf_size); + } + + return flb_input_log_append(ctx->ins, NULL, 0, buf, buf_size); +} + #endif diff --git a/plugins/in_udp/udp_config.c b/plugins/in_udp/udp_config.c index c4468aedf6c..d2cb69e5a32 100644 --- a/plugins/in_udp/udp_config.c +++ b/plugins/in_udp/udp_config.c @@ -43,6 +43,7 @@ struct flb_in_udp_config *udp_config_init(struct flb_input_instance *ins) } ctx->ins = ins; ctx->format = FLB_UDP_FMT_JSON; + ctx->workers = 1; /* Load the config map */ ret = flb_input_config_map_set(ins, (void *)ctx); @@ -168,6 +169,11 @@ int udp_config_destroy(struct flb_in_udp_config *ctx) ctx->collector_id = -1; } + if (ctx->listener_registered == FLB_TRUE && ctx->event_loop != NULL) { + mk_event_del(ctx->event_loop, &ctx->listener_event); + ctx->listener_registered = FLB_FALSE; + } + if (ctx->downstream != NULL) { flb_downstream_destroy(ctx->downstream); } diff --git a/plugins/in_udp/udp_conn.c b/plugins/in_udp/udp_conn.c index 216ff9fa875..19966824698 100644 --- a/plugins/in_udp/udp_conn.c +++ b/plugins/in_udp/udp_conn.c @@ -210,9 +210,9 @@ static inline int process_pack(struct udp_conn *conn, msgpack_unpacked_destroy(&result); if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); + udp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); ret = 0; } else { @@ -441,9 +441,9 @@ static ssize_t parse_payload_none(struct udp_conn *conn) } if (ret == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ins, NULL, 0, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); + udp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); } else { flb_plg_error(ctx->ins, "log event encoding error : %d", ret); From cffcc859b3740111e0590fe78e793d1c081a2d5a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 4 May 2026 17:24:37 -0600 Subject: [PATCH 05/18] in_forward: add worker support Signed-off-by: Eduardo Silva --- plugins/in_forward/fw.c | 359 ++++++++++++++++++++++++++++----- plugins/in_forward/fw.h | 47 +++++ plugins/in_forward/fw_config.c | 6 + plugins/in_forward/fw_prot.c | 35 ++-- 4 files changed, 377 insertions(+), 70 deletions(-) diff --git a/plugins/in_forward/fw.c b/plugins/in_forward/fw.c index 3c2ba41b0f7..2a454208014 100644 --- a/plugins/in_forward/fw.c +++ b/plugins/in_forward/fw.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,13 @@ #include "fw_conn.h" #include "fw_config.h" +static int in_fw_collect(struct flb_input_instance *ins, + struct flb_config *config, void *in_context); +static int in_fw_collect_ctx(struct flb_in_fw_config *ctx); +static void delete_users(struct flb_in_fw_config *ctx); +static int setup_users(struct flb_in_fw_config *ctx, + struct flb_input_instance *ins); + #ifdef FLB_HAVE_UNIX_SOCKET static int remove_existing_socket_file(char *socket_path) { @@ -123,13 +131,18 @@ static int fw_unix_create(struct flb_in_fw_config *ctx) */ static int in_fw_collect(struct flb_input_instance *ins, struct flb_config *config, void *in_context) +{ + (void) ins; + (void) config; + + return in_fw_collect_ctx(in_context); +} + +static int in_fw_collect_ctx(struct flb_in_fw_config *ctx) { int state_backup; struct flb_connection *connection; struct fw_conn *conn; - struct flb_in_fw_config *ctx; - - ctx = in_context; state_backup = ctx->state; ctx->state = FW_INSTANCE_STATE_ACCEPTING_CLIENT; @@ -143,7 +156,7 @@ static int in_fw_collect(struct flb_input_instance *ins, return -1; } - if (!config->is_ingestion_active) { + if (!ctx->ins->config->is_ingestion_active) { flb_downstream_conn_release(connection); ctx->state = state_backup; @@ -152,13 +165,13 @@ static int in_fw_collect(struct flb_input_instance *ins, if(ctx->is_paused) { flb_downstream_conn_release(connection); - flb_plg_trace(ins, "TCP connection will be closed FD=%i", connection->fd); + flb_plg_trace(ctx->ins, "TCP connection will be closed FD=%i", connection->fd); ctx->state = state_backup; return -1; } - flb_plg_trace(ins, "new TCP connection arrived FD=%i", connection->fd); + flb_plg_trace(ctx->ins, "new TCP connection arrived FD=%i", connection->fd); conn = fw_conn_add(connection, ctx); if (!conn) { @@ -272,11 +285,216 @@ static int setup_users(struct flb_in_fw_config *ctx, return 0; } +static int in_fw_worker_listener_event(void *data) +{ + struct mk_event *event; + + event = data; + + return in_fw_collect_ctx(event->data); +} + +static int in_fw_start_tcp_listener(struct flb_in_fw_config *ctx, + struct flb_config *config, + struct flb_net_setup *net_setup, + struct mk_event_loop *event_loop, + int use_collector) +{ + int ret; + unsigned short int port; + + port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); + + ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, + ctx->ins->flags, + ctx->listen, + port, + ctx->ins->tls, + config, + net_setup); + + if (ctx->downstream == NULL) { + flb_plg_error(ctx->ins, + "could not initialize downstream on %s:%s. Aborting", + ctx->listen, ctx->tcp_port); + return -1; + } + + flb_input_downstream_set(ctx->downstream, ctx->ins); + flb_net_socket_nonblocking(ctx->downstream->server_fd); + + if (use_collector == FLB_TRUE) { + ret = flb_input_set_collector_socket(ctx->ins, + in_fw_collect, + ctx->downstream->server_fd, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not set server socket collector"); + return -1; + } + + ctx->coll_fd = ret; + } + else { + ctx->event_loop = event_loop; + MK_EVENT_NEW(&ctx->listener_event); + ctx->listener_event.type = FLB_ENGINE_EV_CUSTOM; + ctx->listener_event.data = ctx; + ctx->listener_event.handler = in_fw_worker_listener_event; + + ret = mk_event_add(event_loop, + ctx->downstream->server_fd, + FLB_ENGINE_EV_CUSTOM, + MK_EVENT_READ, + &ctx->listener_event); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not register forward worker listener"); + return -1; + } + + ctx->listener_registered = FLB_TRUE; + } + + return 0; +} + +static int in_fw_worker_init(struct flb_downstream_worker *worker, + void *parent, + void **worker_context) +{ + int ret; + struct flb_in_fw_config *ctx; + struct flb_in_fw_config *parent_ctx; + + parent_ctx = parent; + + ctx = fw_config_init(parent_ctx->ins); + if (ctx == NULL) { + return -1; + } + + *worker_context = ctx; + + ctx->state = FW_INSTANCE_STATE_RUNNING; + ctx->coll_fd = -1; + ctx->ins = parent_ctx->ins; + ctx->workers = parent_ctx->workers; + ctx->worker_id = worker->worker_id; + ctx->use_ingress_queue = FLB_TRUE; + ctx->is_paused = FLB_FALSE; + ctx->net_setup = parent_ctx->ins->net_setup; + ctx->net_setup.share_port = FLB_TRUE; + mk_list_init(&ctx->connections); + mk_list_init(&ctx->users); + pthread_mutex_init(&ctx->conn_mutex, NULL); + + ret = setup_users(ctx, parent_ctx->ins); + if (ret == -1) { + return -1; + } + + ret = in_fw_start_tcp_listener(ctx, + parent_ctx->ins->config, + &ctx->net_setup, + worker->event_loop, + FLB_FALSE); + if (ret == 0) { + flb_downstream_thread_safe(ctx->downstream); + } + + return ret; +} + +static void in_fw_worker_exit(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_fw_config *ctx; + + (void) worker; + + ctx = worker_context; + + delete_users(ctx); + fw_conn_del_all(ctx); + fw_config_destroy(ctx); +} + +static void in_fw_worker_maintenance(struct flb_downstream_worker *worker, + void *worker_context) +{ + struct flb_in_fw_config *ctx; + + (void) worker; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_conn_timeouts_stream(ctx->downstream); + flb_downstream_conn_pending_destroy(ctx->downstream); + } +} + +static int in_fw_workers_start(struct flb_in_fw_config *ctx) +{ + struct flb_downstream_worker_options options; + + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); + options.workers = ctx->workers; + options.config = ctx->ins->config; + options.parent = ctx; + options.cb_init = in_fw_worker_init; + options.cb_exit = in_fw_worker_exit; + options.cb_maintenance = in_fw_worker_maintenance; + + return flb_downstream_worker_runtime_start(&ctx->runtime, &options); +} + +static void in_fw_workers_stop(struct flb_in_fw_config *ctx) +{ + flb_downstream_worker_runtime_stop(ctx->runtime); + ctx->runtime = NULL; +} + +static void in_fw_worker_pause(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct flb_in_fw_config *ctx; + + (void) worker; + (void) data; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_pause(ctx->downstream); + ctx->is_paused = FLB_TRUE; + ctx->state = FW_INSTANCE_STATE_PAUSED; + } +} + +static void in_fw_worker_resume(struct flb_downstream_worker *worker, + void *worker_context, + void *data) +{ + struct flb_in_fw_config *ctx; + + (void) worker; + (void) data; + + ctx = worker_context; + + if (ctx->downstream != NULL) { + flb_downstream_resume(ctx->downstream); + ctx->is_paused = FLB_FALSE; + ctx->state = FW_INSTANCE_STATE_RUNNING; + } +} + /* Initialize plugin */ static int in_fw_init(struct flb_input_instance *ins, struct flb_config *config, void *data) { - unsigned short int port; int ret; struct flb_in_fw_config *ctx; @@ -300,8 +518,18 @@ static int in_fw_init(struct flb_input_instance *ins, /* Set plugin ingestion to active */ ctx->is_paused = FLB_FALSE; + if (ctx->workers <= 0) { + ctx->workers = 1; + } + /* Unix Socket mode */ if (ctx->unix_path) { + if (ctx->workers > 1) { + flb_plg_error(ctx->ins, "workers is not supported with unix_path"); + fw_config_destroy(ctx); + return -1; + } + #ifndef FLB_HAVE_UNIX_SOCKET flb_plg_error(ctx->ins, "unix address is not supported %s:%s. Aborting", ctx->listen, ctx->tcp_port); @@ -319,38 +547,7 @@ static int in_fw_init(struct flb_input_instance *ins, #endif } else { - port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); - - ctx->downstream = flb_downstream_create(FLB_TRANSPORT_TCP, - ctx->ins->flags, - ctx->listen, - port, - ctx->ins->tls, - config, - &ctx->ins->net_setup); - - if (ctx->downstream == NULL) { - flb_plg_error(ctx->ins, - "could not initialize downstream on unix://%s. Aborting", - ctx->listen); - - fw_config_destroy(ctx); - - return -1; - } - - if (ctx->downstream != NULL) { - flb_plg_info(ctx->ins, "listening on %s:%s", - ctx->listen, ctx->tcp_port); - } - else { - flb_plg_error(ctx->ins, "could not bind address %s:%s. Aborting", - ctx->listen, ctx->tcp_port); - - fw_config_destroy(ctx); - - return -1; - } + /* TCP listener startup happens after security.users validation. */ } /* Load users */ @@ -370,22 +567,56 @@ static int in_fw_init(struct flb_input_instance *ins, return -1; } - flb_input_downstream_set(ctx->downstream, ctx->ins); - - flb_net_socket_nonblocking(ctx->downstream->server_fd); + if (ctx->unix_path) { + flb_input_downstream_set(ctx->downstream, ctx->ins); + flb_net_socket_nonblocking(ctx->downstream->server_fd); + + ret = flb_input_set_collector_socket(ins, + in_fw_collect, + ctx->downstream->server_fd, + config); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not set server socket collector"); + delete_users(ctx); + fw_config_destroy(ctx); + return -1; + } - /* Collect upon data available on the standard input */ - ret = flb_input_set_collector_socket(ins, - in_fw_collect, - ctx->downstream->server_fd, - config); - if (ret == -1) { - flb_plg_error(ctx->ins, "could not set server socket collector"); - fw_config_destroy(ctx); - return -1; + ctx->coll_fd = ret; } + else { + if (ctx->workers > 1) { + ret = flb_input_ingress_enable(ins); + if (ret != 0) { + delete_users(ctx); + fw_config_destroy(ctx); + return -1; + } + + ret = in_fw_workers_start(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not start forward listener workers on %s:%s. Aborting", + ctx->listen, ctx->tcp_port); + delete_users(ctx); + fw_config_destroy(ctx); + return -1; + } + } + else { + ret = in_fw_start_tcp_listener(ctx, config, &ins->net_setup, NULL, FLB_TRUE); + if (ret != 0) { + delete_users(ctx); + fw_config_destroy(ctx); + return -1; + } + } - ctx->coll_fd = ret; + flb_plg_info(ctx->ins, + "listening on %s:%s with %i worker%s", + ctx->listen, ctx->tcp_port, ctx->workers, + ctx->workers == 1 ? "" : "s"); + } pthread_mutex_init(&ctx->conn_mutex, NULL); @@ -404,7 +635,14 @@ static void in_fw_pause(void *data, struct flb_config *config) * pause the ingestion. The plugin should close all the connections * and wait for the ingestion to resume. */ - flb_input_collector_pause(ctx->coll_fd, ctx->ins); + if (ctx->runtime != NULL) { + flb_downstream_worker_runtime_foreach(ctx->runtime, + in_fw_worker_pause, + NULL); + } + else { + flb_input_collector_pause(ctx->coll_fd, ctx->ins); + } ret = pthread_mutex_lock(&ctx->conn_mutex); if (ret != 0) { @@ -444,7 +682,14 @@ static void in_fw_resume(void *data, struct flb_config *config) { struct flb_in_fw_config *ctx = data; if (config->is_running == FLB_TRUE) { - flb_input_collector_resume(ctx->coll_fd, ctx->ins); + if (ctx->runtime != NULL) { + flb_downstream_worker_runtime_foreach(ctx->runtime, + in_fw_worker_resume, + NULL); + } + else { + flb_input_collector_resume(ctx->coll_fd, ctx->ins); + } ret = pthread_mutex_lock(&ctx->conn_mutex); if (ret != 0) { @@ -474,6 +719,7 @@ static int in_fw_exit(void *data, struct flb_config *config) } delete_users(ctx); + in_fw_workers_stop(ctx); fw_conn_del_all(ctx); fw_config_destroy(ctx); return 0; @@ -526,6 +772,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_in_fw_config, empty_shared_key), "Enable an empty string as the shared key for authentication." }, + { + FLB_CONFIG_MAP_INT, "workers", "1", + 0, FLB_TRUE, offsetof(struct flb_in_fw_config, workers), + "Set the number of Forward listener workers" + }, {0} }; diff --git a/plugins/in_forward/fw.h b/plugins/in_forward/fw.h index 7389b5a4bf3..16b808b557f 100644 --- a/plugins/in_forward/fw.h +++ b/plugins/in_forward/fw.h @@ -22,6 +22,9 @@ #include #include +#include +#include +#include #include #include @@ -50,6 +53,8 @@ struct flb_in_fw_user { struct mk_list _head; }; +struct flb_downstream_worker_runtime; + struct flb_in_fw_config { size_t buffer_max_size; /* Max Buffer size */ size_t buffer_chunk_size; /* Chunk allocation size */ @@ -73,6 +78,13 @@ struct flb_in_fw_config { int empty_shared_key; /* use an empty string as shared key */ int coll_fd; + int workers; /* Listener worker count */ + int worker_id; /* Worker id */ + int use_ingress_queue; /* Queue records to main loop */ + int listener_registered; /* Listener event registered */ + struct mk_event listener_event; /* Worker listener event */ + struct mk_event_loop *event_loop; /* Worker event loop */ + struct flb_net_setup net_setup; /* Worker network setup */ struct flb_downstream *downstream; /* Client manager */ struct mk_list connections; /* List of active connections */ struct flb_input_instance *ins; /* Input plugin instace */ @@ -86,6 +98,41 @@ struct flb_in_fw_config { /* Plugin is paused */ int is_paused; + + struct flb_downstream_worker_runtime *runtime; }; +static inline int fw_ingest_logs(struct flb_in_fw_config *ctx, + const char *tag, size_t tag_len, + const void *buf, size_t buf_size) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_log(ctx->ins, tag, tag_len, buf, buf_size); + } + + return flb_input_log_append(ctx->ins, tag, tag_len, buf, buf_size); +} + +static inline int fw_ingest_metrics(struct flb_in_fw_config *ctx, + const char *tag, size_t tag_len, + struct cmt *cmt) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_metrics(ctx->ins, tag, tag_len, cmt); + } + + return flb_input_metrics_append(ctx->ins, tag, tag_len, cmt); +} + +static inline int fw_ingest_traces(struct flb_in_fw_config *ctx, + const char *tag, size_t tag_len, + struct ctrace *ctr) +{ + if (ctx->use_ingress_queue == FLB_TRUE) { + return flb_input_ingress_queue_traces(ctx->ins, tag, tag_len, ctr); + } + + return flb_input_trace_append(ctx->ins, tag, tag_len, ctr); +} + #endif diff --git a/plugins/in_forward/fw_config.c b/plugins/in_forward/fw_config.c index 0baff3a47dd..08187e52305 100644 --- a/plugins/in_forward/fw_config.c +++ b/plugins/in_forward/fw_config.c @@ -68,6 +68,7 @@ struct flb_in_fw_config *fw_config_init(struct flb_input_instance *i_ins) return NULL; } config->coll_fd = -1; + config->workers = 1; config->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); @@ -149,6 +150,11 @@ int fw_config_destroy(struct flb_in_fw_config *config) config->coll_fd = -1; } + if (config->listener_registered == FLB_TRUE && config->event_loop != NULL) { + mk_event_del(config->event_loop, &config->listener_event); + config->listener_registered = FLB_FALSE; + } + if (config->downstream != NULL) { flb_downstream_destroy(config->downstream); } diff --git a/plugins/in_forward/fw_prot.c b/plugins/in_forward/fw_prot.c index 844c8494e66..fc76c60a6d5 100644 --- a/plugins/in_forward/fw_prot.c +++ b/plugins/in_forward/fw_prot.c @@ -979,9 +979,9 @@ static int fw_process_forward_mode_entry( } if (result == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(conn->ctx->ins, tag, tag_len, - conn->ctx->log_encoder->output_buffer, - conn->ctx->log_encoder->output_length); + fw_ingest_logs(conn->ctx, tag, tag_len, + conn->ctx->log_encoder->output_buffer, + conn->ctx->log_encoder->output_length); } flb_log_event_encoder_reset(conn->ctx->log_encoder); @@ -1050,9 +1050,9 @@ static int fw_process_message_mode_entry( } if (result == FLB_EVENT_ENCODER_SUCCESS) { - flb_input_log_append(in, tag, tag_len, - conn->ctx->log_encoder->output_buffer, - conn->ctx->log_encoder->output_length); + fw_ingest_logs(conn->ctx, tag, tag_len, + conn->ctx->log_encoder->output_buffer, + conn->ctx->log_encoder->output_length); } flb_log_event_encoder_reset(conn->ctx->log_encoder); @@ -1120,9 +1120,9 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, struct ctrace *ctr; if (event_type == FLB_EVENT_TYPE_LOGS) { - ret = flb_input_log_append(conn->in, - out_tag, flb_sds_len(out_tag), - data, len); + ret = fw_ingest_logs(conn->ctx, + out_tag, flb_sds_len(out_tag), + data, len); if (ret != 0) { flb_plg_error(ins, "could not append logs. ret=%d", ret); return -1; @@ -1137,15 +1137,18 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, return -1; } - ret = flb_input_metrics_append(conn->in, - out_tag, flb_sds_len(out_tag), - cmt); + ret = fw_ingest_metrics(conn->ctx, + out_tag, flb_sds_len(out_tag), + cmt); if (ret != 0) { flb_plg_error(ins, "could not append metrics. ret=%d", ret); cmt_decode_msgpack_destroy(cmt); return -1; } - cmt_decode_msgpack_destroy(cmt); + + if (conn->ctx->use_ingress_queue == FLB_FALSE) { + cmt_decode_msgpack_destroy(cmt); + } } else if (event_type == FLB_EVENT_TYPE_TRACES) { off = 0; @@ -1155,9 +1158,9 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, return -1; } - ret = flb_input_trace_append(ins, - out_tag, flb_sds_len(out_tag), - ctr); + ret = fw_ingest_traces(conn->ctx, + out_tag, flb_sds_len(out_tag), + ctr); if (ret != 0) { flb_plg_error(ins, "could not append traces. ret=%d", ret); ctr_decode_msgpack_destroy(ctr); From 9e576e909b1960ad3320c56060c6467252b4e74a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 4 May 2026 17:24:44 -0600 Subject: [PATCH 06/18] tests: integration: cover downstream workers Signed-off-by: Eduardo Silva --- .../config/in_forward_tls_workers.yaml | 24 +++ .../in_forward/config/in_forward_workers.yaml | 21 +++ .../in_forward/tests/test_in_forward_001.py | 136 +++++++++++++++ .../in_tcp_parser_json_tls_workers.yaml | 29 ++++ .../config/in_tcp_parser_json_workers.yaml | 26 +++ .../scenarios/in_tcp/tests/test_in_tcp_001.py | 156 ++++++++++++++++++ .../in_udp/config/in_udp_json_workers.yaml | 23 +++ .../config/in_udp_parser_json_workers.yaml | 26 +++ .../scenarios/in_udp/tests/test_in_udp_001.py | 80 +++++++++ 9 files changed, 521 insertions(+) create mode 100644 tests/integration/scenarios/in_forward/config/in_forward_tls_workers.yaml create mode 100644 tests/integration/scenarios/in_forward/config/in_forward_workers.yaml create mode 100644 tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_tls_workers.yaml create mode 100644 tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_workers.yaml create mode 100644 tests/integration/scenarios/in_udp/config/in_udp_json_workers.yaml create mode 100644 tests/integration/scenarios/in_udp/config/in_udp_parser_json_workers.yaml diff --git a/tests/integration/scenarios/in_forward/config/in_forward_tls_workers.yaml b/tests/integration/scenarios/in_forward/config/in_forward_tls_workers.yaml new file mode 100644 index 00000000000..39c0b510051 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_tls_workers.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + workers: 4 + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_workers.yaml b/tests/integration/scenarios/in_forward/config/in_forward_workers.yaml new file mode 100644 index 00000000000..f199f862c74 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_workers.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + workers: 4 + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py index 167f86eb5a5..2ffb2077156 100644 --- a/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py +++ b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py @@ -7,7 +7,9 @@ import ssl import subprocess import tempfile +import time import uuid +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import opentelemetry @@ -102,6 +104,7 @@ def _stop_receiver(self, service): def start(self): self.service.start() + self.flb = self.service.flb self.flb_listener_port = self.service.flb_listener_port def stop(self): @@ -124,6 +127,16 @@ def wait_for_record_count(self, minimum_count, timeout=10): description=f"{minimum_count} forwarded forward records", ) + def wait_for_log_message(self, pattern, timeout=10, interval=0.25): + deadline = time.time() + timeout + while time.time() < deadline: + if self.flb and self.flb.log_file and os.path.exists(self.flb.log_file): + with open(self.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + if pattern in log_file.read(): + return True + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for log pattern: {pattern}") + class StorageLimitService(Service): def __init__(self, config_file): @@ -413,6 +426,11 @@ def _send_tcp_payload(port, payload): sock.sendall(payload) +def _drop_partial_tcp_payload(port, payload): + with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: + sock.sendall(payload) + + def _send_unix_payload(path, payload): with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: sock.settimeout(5) @@ -427,6 +445,11 @@ def _send_tls_payload(port, payload, cafile): tls_sock.sendall(payload) +def _drop_raw_tls_connection(port, payload): + with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: + sock.sendall(payload) + + def _recv_msgpack_value(sock): sock.settimeout(5) data = sock.recv(4096) @@ -599,6 +622,61 @@ def test_in_forward_forward_mode_multiple_entries(): assert [record["message"] for record in records[:2]] == ["entry-1", "entry-2"] +def test_in_forward_workers_concurrent_message_mode_records(): + total_records = 16 + service = Service("in_forward_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + payloads = [ + _message_mode_payload(TEST_TAG, {"message": f"worker-{i}", "value": i}) + for i in range(total_records) + ] + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda payload: _send_tcp_payload(service.flb_listener_port, payload), + payloads, + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_forward_workers_drop_partial_connections_and_continue(): + dropped_connections = 8 + valid_records = 8 + service = Service("in_forward_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + with ThreadPoolExecutor(max_workers=dropped_connections) as executor: + list(executor.map( + lambda _: _drop_partial_tcp_payload(service.flb_listener_port, b"\x93\xa4test"), + range(dropped_connections), + )) + + payloads = [ + _message_mode_payload(TEST_TAG, {"message": f"valid-{i}", "value": i}) + for i in range(valid_records) + ] + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda payload: _send_tcp_payload(service.flb_listener_port, payload), + payloads, + )) + records = service.wait_for_record_count(valid_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) + + def test_in_forward_packed_forward_gzip(): service = Service("in_forward.yaml") service.start() @@ -784,6 +862,64 @@ def test_in_forward_tls_message_mode(): assert records[0]["message"] == "tls-message" +def test_in_forward_tls_workers_concurrent_message_mode_records(): + total_records = 16 + service = Service("in_forward_tls_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda i: _send_tls_payload(service.flb_listener_port, + _message_mode_payload( + TEST_TAG, + {"message": f"tls-worker-{i}", + "value": i}), + service.tls_crt_file), + range(total_records), + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_forward_tls_workers_drop_bad_handshakes_and_continue(): + dropped_connections = 8 + valid_records = 8 + service = Service("in_forward_tls_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + with ThreadPoolExecutor(max_workers=dropped_connections) as executor: + list(executor.map( + lambda i: _drop_raw_tls_connection(service.flb_listener_port, + f"not-tls-{i}".encode("utf-8")), + range(dropped_connections), + )) + + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda i: _send_tls_payload(service.flb_listener_port, + _message_mode_payload( + TEST_TAG, + {"message": f"valid-tls-{i}", + "value": i}), + service.tls_crt_file), + range(valid_records), + )) + records = service.wait_for_record_count(valid_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) + + def test_in_forward_secure_forward_auth_success(): service = Service("in_forward_secure.yaml") service.start() diff --git a/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_tls_workers.yaml b/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_tls_workers.yaml new file mode 100644 index 00000000000..719d1d9b12c --- /dev/null +++ b/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_tls_workers.yaml @@ -0,0 +1,29 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: tcp + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + format: none + separator: "\\n" + parser: json + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + workers: 4 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_workers.yaml b/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_workers.yaml new file mode 100644 index 00000000000..7455f087fe7 --- /dev/null +++ b/tests/integration/scenarios/in_tcp/config/in_tcp_parser_json_workers.yaml @@ -0,0 +1,26 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: tcp + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + format: none + separator: "\\n" + parser: json + workers: 4 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_tcp/tests/test_in_tcp_001.py b/tests/integration/scenarios/in_tcp/tests/test_in_tcp_001.py index 314bb144464..65a8f1001a2 100644 --- a/tests/integration/scenarios/in_tcp/tests/test_in_tcp_001.py +++ b/tests/integration/scenarios/in_tcp/tests/test_in_tcp_001.py @@ -1,5 +1,8 @@ import os import socket +import ssl +import time +from concurrent.futures import ThreadPoolExecutor import requests @@ -14,12 +17,17 @@ def __init__(self, config_file): self.parsers_file = os.environ.get("FLUENT_BIT_PARSERS_FILE") or os.path.abspath( os.path.join(test_path, "../../../../../conf/parsers.conf") ) + cert_dir = os.path.abspath(os.path.join(test_path, "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") self.service = FluentBitTestService( self.config_file, data_storage=data_storage, data_keys=["payloads"], extra_env={ "PARSERS_FILE_TEST": self.parsers_file, + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, }, pre_start=self._start_receiver, post_stop=self._stop_receiver, @@ -41,6 +49,7 @@ def _stop_receiver(self, service): def start(self): self.service.start() + self.flb = self.service.flb self.flb_listener_port = self.service.flb_listener_port def stop(self): @@ -60,6 +69,33 @@ def wait_for_single_record(self, timeout=10): return payloads[0][0] + def flattened_records(self): + records = [] + for payload in data_storage["payloads"]: + if isinstance(payload, list): + records.extend(payload) + elif payload is not None: + records.append(payload) + return records + + def wait_for_record_count(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: self.flattened_records() if len(self.flattened_records()) >= minimum_count else None, + timeout=timeout, + interval=0.2, + description=f"{minimum_count} forwarded TCP payloads", + ) + + def wait_for_log_message(self, pattern, timeout=10, interval=0.25): + deadline = time.time() + timeout + while time.time() < deadline: + if self.flb and self.flb.log_file and os.path.exists(self.flb.log_file): + with open(self.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + if pattern in log_file.read(): + return True + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for log pattern: {pattern}") + def _send_line(port, line): with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: @@ -67,6 +103,24 @@ def _send_line(port, line): sock.shutdown(socket.SHUT_WR) +def _drop_partial_connection(port, payload): + with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: + sock.sendall(payload.encode("utf-8")) + + +def _send_tls_line(port, line, cafile): + context = ssl.create_default_context(cafile=cafile) + with socket.create_connection(("127.0.0.1", port), timeout=5) as raw_sock: + with context.wrap_socket(raw_sock, server_hostname="localhost") as tls_sock: + tls_sock.sendall(line.encode("utf-8")) + tls_sock.shutdown(socket.SHUT_WR) + + +def _drop_raw_tls_connection(port, payload): + with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: + sock.sendall(payload) + + def test_in_tcp_parser_json_success(): service = Service("in_tcp_parser_json.yaml") service.start() @@ -93,3 +147,105 @@ def test_in_tcp_parser_json_fallback_to_log(): assert "log" in record assert record["log"].strip() == "not-json" + + +def test_in_tcp_parser_json_workers_concurrent_records(): + total_records = 16 + service = Service("in_tcp_parser_json_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda i: _send_line(service.flb_listener_port, + f'{{"message":"worker-{i}","value":{i}}}\n'), + range(total_records), + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_tcp_workers_drop_partial_connections_and_continue(): + dropped_connections = 8 + valid_records = 8 + service = Service("in_tcp_parser_json_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + with ThreadPoolExecutor(max_workers=dropped_connections) as executor: + list(executor.map( + lambda i: _drop_partial_connection(service.flb_listener_port, + f'{{"message":"partial-{i}"'), + range(dropped_connections), + )) + + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda i: _send_line(service.flb_listener_port, + f'{{"message":"valid-{i}","value":{i}}}\n'), + range(valid_records), + )) + records = service.wait_for_record_count(valid_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) + + +def test_in_tcp_tls_workers_concurrent_records(): + total_records = 16 + service = Service("in_tcp_parser_json_tls_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda i: _send_tls_line(service.flb_listener_port, + f'{{"message":"tls-worker-{i}","value":{i}}}\n', + service.tls_crt_file), + range(total_records), + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_tcp_tls_workers_drop_bad_handshakes_and_continue(): + dropped_connections = 8 + valid_records = 8 + service = Service("in_tcp_parser_json_tls_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + with ThreadPoolExecutor(max_workers=dropped_connections) as executor: + list(executor.map( + lambda i: _drop_raw_tls_connection(service.flb_listener_port, + f"not-tls-{i}".encode("utf-8")), + range(dropped_connections), + )) + + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda i: _send_tls_line(service.flb_listener_port, + f'{{"message":"valid-tls-{i}","value":{i}}}\n', + service.tls_crt_file), + range(valid_records), + )) + records = service.wait_for_record_count(valid_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) diff --git a/tests/integration/scenarios/in_udp/config/in_udp_json_workers.yaml b/tests/integration/scenarios/in_udp/config/in_udp_json_workers.yaml new file mode 100644 index 00000000000..7f37132a3df --- /dev/null +++ b/tests/integration/scenarios/in_udp/config/in_udp_json_workers.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: udp + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + format: json + workers: 4 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_udp/config/in_udp_parser_json_workers.yaml b/tests/integration/scenarios/in_udp/config/in_udp_parser_json_workers.yaml new file mode 100644 index 00000000000..872a5e855e1 --- /dev/null +++ b/tests/integration/scenarios/in_udp/config/in_udp_parser_json_workers.yaml @@ -0,0 +1,26 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: udp + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + format: none + separator: "\\n" + parser: json + workers: 4 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py b/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py index 83315f332ac..8553f5e4fa1 100644 --- a/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py +++ b/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py @@ -1,5 +1,7 @@ import os import socket +import time +from concurrent.futures import ThreadPoolExecutor import requests @@ -41,6 +43,7 @@ def _stop_receiver(self, service): def start(self): self.service.start() + self.flb = self.service.flb self.flb_listener_port = self.service.flb_listener_port def stop(self): @@ -60,6 +63,33 @@ def wait_for_single_record(self, timeout=10): return payloads[0][0] + def flattened_records(self): + records = [] + for payload in data_storage["payloads"]: + if isinstance(payload, list): + records.extend(payload) + elif payload is not None: + records.append(payload) + return records + + def wait_for_record_count(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: self.flattened_records() if len(self.flattened_records()) >= minimum_count else None, + timeout=timeout, + interval=0.2, + description=f"{minimum_count} forwarded UDP payloads", + ) + + def wait_for_log_message(self, pattern, timeout=10, interval=0.25): + deadline = time.time() + timeout + while time.time() < deadline: + if self.flb and self.flb.log_file and os.path.exists(self.flb.log_file): + with open(self.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + if pattern in log_file.read(): + return True + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for log pattern: {pattern}") + def _send_datagram(port, payload): with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: @@ -92,3 +122,53 @@ def test_in_udp_parser_json_fallback_to_log(): assert "log" in record assert record["log"].strip() == "not-json" + + +def test_in_udp_parser_json_workers_concurrent_records(): + total_records = 16 + service = Service("in_udp_parser_json_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + with ThreadPoolExecutor(max_workers=total_records) as executor: + list(executor.map( + lambda i: _send_datagram(service.flb_listener_port, + f'{{"message":"worker-{i}","value":{i}}}\n'), + range(total_records), + )) + records = service.wait_for_record_count(total_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(total_records)) + + +def test_in_udp_workers_drop_malformed_datagrams_and_continue(): + malformed_datagrams = 8 + valid_records = 8 + service = Service("in_udp_json_workers.yaml") + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + try: + with ThreadPoolExecutor(max_workers=malformed_datagrams) as executor: + list(executor.map( + lambda i: _send_datagram(service.flb_listener_port, + f'{{"message":"malformed-{i}"'), + range(malformed_datagrams), + )) + + with ThreadPoolExecutor(max_workers=valid_records) as executor: + list(executor.map( + lambda i: _send_datagram(service.flb_listener_port, + f'{{"message":"valid-{i}","value":{i}}}'), + range(valid_records), + )) + records = service.wait_for_record_count(valid_records, timeout=20) + finally: + service.stop() + + values = sorted(record["value"] for record in records) + assert values == list(range(valid_records)) From 64b59124cd39e1bb6549cc410f3d6a7c4b3b5da7 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 5 May 2026 09:34:03 -0600 Subject: [PATCH 07/18] downstream_worker: fix shutdown cleanup Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_downstream_worker.h | 3 ++- src/flb_downstream_worker.c | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/include/fluent-bit/flb_downstream_worker.h b/include/fluent-bit/flb_downstream_worker.h index cae95e4c845..9bf511ce2e8 100644 --- a/include/fluent-bit/flb_downstream_worker.h +++ b/include/fluent-bit/flb_downstream_worker.h @@ -20,6 +20,7 @@ #ifndef FLB_DOWNSTREAM_WORKER_H #define FLB_DOWNSTREAM_WORKER_H +#include #include #include @@ -55,7 +56,7 @@ struct flb_downstream_worker { pthread_t thread; pthread_mutex_t mutex; pthread_cond_t condition; - int should_exit; + atomic_int should_exit; int initialized; int thread_created; int startup_result; diff --git a/src/flb_downstream_worker.c b/src/flb_downstream_worker.c index 0cf2869c427..19f93736804 100644 --- a/src/flb_downstream_worker.c +++ b/src/flb_downstream_worker.c @@ -27,6 +27,7 @@ struct flb_downstream_worker_runtime { struct flb_downstream_worker *workers; int worker_count; + int active_workers; struct flb_config *config; void *parent; flb_downstream_worker_init_cb cb_init; @@ -81,7 +82,7 @@ static void *downstream_worker_thread(void *data) goto cleanup; } - while (worker->should_exit == FLB_FALSE) { + while (atomic_load(&worker->should_exit) == FLB_FALSE) { mk_event_wait_2(worker->event_loop, 250); mk_event_foreach(event, worker->event_loop) { @@ -146,6 +147,7 @@ int flb_downstream_worker_runtime_start(struct flb_downstream_worker_runtime **o for (i = 0; i < runtime->worker_count; i++) { downstream_worker_context_reset(&runtime->workers[i]); + runtime->active_workers++; runtime->workers[i].runtime = runtime; runtime->workers[i].parent = runtime->parent; runtime->workers[i].worker_id = i; @@ -191,8 +193,8 @@ void flb_downstream_worker_runtime_stop(struct flb_downstream_worker_runtime *ru return; } - for (i = 0; i < runtime->worker_count; i++) { - runtime->workers[i].should_exit = FLB_TRUE; + for (i = 0; i < runtime->active_workers; i++) { + atomic_store(&runtime->workers[i].should_exit, FLB_TRUE); if (runtime->workers[i].thread_created == FLB_TRUE) { pthread_join(runtime->workers[i].thread, NULL); } From 92b97c3393717ec486c6266282e3fda63a896d53 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 5 May 2026 09:34:08 -0600 Subject: [PATCH 08/18] http_server: clean up failed worker start Signed-off-by: Eduardo Silva --- src/http_server/flb_http_server.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index f77c64e4dcb..8580e545b9d 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -482,6 +482,7 @@ static int flb_http_server_worker_initialize(struct flb_downstream_worker *worke result = flb_http_server_start(&context->server); if (result != 0) { + flb_http_server_destroy(&context->server); return result; } From 1a58578bf51fdc53e4c8e8e3c63f2b046c9cd6ee Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 5 May 2026 09:34:11 -0600 Subject: [PATCH 09/18] in_forward: fix worker cleanup paths Signed-off-by: Eduardo Silva --- plugins/in_forward/fw.c | 4 ++-- plugins/in_forward/fw_prot.c | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/in_forward/fw.c b/plugins/in_forward/fw.c index 2a454208014..8a0928446b0 100644 --- a/plugins/in_forward/fw.c +++ b/plugins/in_forward/fw.c @@ -163,9 +163,9 @@ static int in_fw_collect_ctx(struct flb_in_fw_config *ctx) return -1; } - if(ctx->is_paused) { - flb_downstream_conn_release(connection); + if (ctx->is_paused) { flb_plg_trace(ctx->ins, "TCP connection will be closed FD=%i", connection->fd); + flb_downstream_conn_release(connection); ctx->state = state_backup; return -1; diff --git a/plugins/in_forward/fw_prot.c b/plugins/in_forward/fw_prot.c index fc76c60a6d5..938891e6a97 100644 --- a/plugins/in_forward/fw_prot.c +++ b/plugins/in_forward/fw_prot.c @@ -1142,7 +1142,9 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, cmt); if (ret != 0) { flb_plg_error(ins, "could not append metrics. ret=%d", ret); - cmt_decode_msgpack_destroy(cmt); + if (conn->ctx->use_ingress_queue == FLB_FALSE) { + cmt_decode_msgpack_destroy(cmt); + } return -1; } @@ -1163,7 +1165,9 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, ctr); if (ret != 0) { flb_plg_error(ins, "could not append traces. ret=%d", ret); - ctr_decode_msgpack_destroy(ctr); + if (conn->ctx->use_ingress_queue == FLB_FALSE) { + ctr_decode_msgpack_destroy(ctr); + } return -1; } /* Note: flb_input_trace_append takes ownership of ctr and destroys it on success */ From 39299facb64d195574033ae7e7c0a5081a49cb01 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 5 May 2026 09:34:14 -0600 Subject: [PATCH 10/18] in_tcp: handle worker pause and ingest errors Signed-off-by: Eduardo Silva --- plugins/in_tcp/tcp.c | 31 +++++++++++++++++++------------ plugins/in_tcp/tcp_conn.c | 24 ++++++++++++++++++------ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/plugins/in_tcp/tcp.c b/plugins/in_tcp/tcp.c index 755039f28a6..c0b902172d5 100644 --- a/plugins/in_tcp/tcp.c +++ b/plugins/in_tcp/tcp.c @@ -44,6 +44,23 @@ static void in_tcp_connections_destroy(struct flb_in_tcp_config *ctx) } } +static void in_tcp_connections_pause(struct flb_in_tcp_config *ctx) +{ + struct mk_list *tmp; + struct mk_list *head; + struct tcp_conn *conn; + + mk_list_foreach_safe(head, tmp, &ctx->connections) { + conn = mk_list_entry(head, struct tcp_conn, _head); + if (conn->busy) { + conn->pending_close = FLB_TRUE; + continue; + } + + tcp_conn_del(conn); + } +} + static int in_tcp_worker_listener_event(void *data) { struct mk_event *event; @@ -217,6 +234,7 @@ static void in_tcp_worker_pause(struct flb_downstream_worker *worker, if (ctx->downstream != NULL) { flb_downstream_pause(ctx->downstream); } + in_tcp_connections_pause(ctx); } static void in_tcp_worker_resume(struct flb_downstream_worker *worker, @@ -352,9 +370,6 @@ static int in_tcp_exit(void *data, struct flb_config *config) static void in_tcp_pause(void *data, struct flb_config *config) { struct flb_in_tcp_config *ctx = data; - struct mk_list *head; - struct mk_list *tmp; - struct tcp_conn *conn; (void) config; @@ -367,15 +382,7 @@ static void in_tcp_pause(void *data, struct flb_config *config) flb_downstream_pause(ctx->downstream); - mk_list_foreach_safe(head, tmp, &ctx->connections) { - conn = mk_list_entry(head, struct tcp_conn, _head); - if (conn->busy) { - conn->pending_close = FLB_TRUE; - continue; - } - - tcp_conn_del(conn); - } + in_tcp_connections_pause(ctx); } static void in_tcp_resume(void *data, struct flb_config *config) diff --git a/plugins/in_tcp/tcp_conn.c b/plugins/in_tcp/tcp_conn.c index df95df1cf03..fdf87c3b7d3 100644 --- a/plugins/in_tcp/tcp_conn.c +++ b/plugins/in_tcp/tcp_conn.c @@ -140,9 +140,15 @@ static inline int process_pack(struct tcp_conn *conn, if (ret == FLB_EVENT_ENCODER_SUCCESS) { if (ctx->log_encoder->output_length > 0) { - tcp_ingest_logs(ctx, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); + ret = tcp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not append TCP logs for %s:%s. ret=%d", + ctx->listen, ctx->tcp_port, ret); + return -1; + } } ret = 0; } @@ -384,9 +390,15 @@ static ssize_t parse_payload_none(struct tcp_conn *conn) if (ret == FLB_EVENT_ENCODER_SUCCESS) { if (ctx->log_encoder->output_length > 0) { - tcp_ingest_logs(ctx, - ctx->log_encoder->output_buffer, - ctx->log_encoder->output_length); + ret = tcp_ingest_logs(ctx, + ctx->log_encoder->output_buffer, + ctx->log_encoder->output_length); + if (ret != 0) { + flb_plg_error(ctx->ins, + "could not append TCP logs for %s:%s. ret=%d", + ctx->listen, ctx->tcp_port, ret); + return -1; + } } } else { From f72da43445f9cedba04b3a16997dae39acfea780 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 5 May 2026 09:34:17 -0600 Subject: [PATCH 11/18] in_udp: clean up listener dummy connection Signed-off-by: Eduardo Silva --- plugins/in_udp/udp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/in_udp/udp.c b/plugins/in_udp/udp.c index cdaca20c157..6e233755659 100644 --- a/plugins/in_udp/udp.c +++ b/plugins/in_udp/udp.c @@ -98,6 +98,7 @@ static int in_udp_start_listener(struct flb_in_udp_config *ctx, config); if (ret == -1) { flb_plg_error(ctx->ins, "Could not set collector for IN_UDP input plugin"); + in_udp_dummy_conn_destroy(ctx); return -1; } @@ -106,6 +107,7 @@ static int in_udp_start_listener(struct flb_in_udp_config *ctx, if (ctx->collector_event == NULL) { flb_plg_error(ctx->ins, "Could not get collector event"); + in_udp_dummy_conn_destroy(ctx); return -1; } } @@ -123,6 +125,7 @@ static int in_udp_start_listener(struct flb_in_udp_config *ctx, &ctx->listener_event); if (ret == -1) { flb_plg_error(ctx->ins, "could not register UDP worker listener"); + in_udp_dummy_conn_destroy(ctx); return -1; } From 22a2cebe3b0d39fbe3e66bb9af5ac708a3e257e6 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 5 May 2026 09:34:21 -0600 Subject: [PATCH 12/18] tests: integration: clean worker helpers Signed-off-by: Eduardo Silva --- .../in_forward/tests/test_in_forward_001.py | 18 +++++------------- .../scenarios/in_udp/tests/test_in_udp_001.py | 9 ++++++++- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py index 2ffb2077156..0215c7f6be7 100644 --- a/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py +++ b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py @@ -426,11 +426,6 @@ def _send_tcp_payload(port, payload): sock.sendall(payload) -def _drop_partial_tcp_payload(port, payload): - with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: - sock.sendall(payload) - - def _send_unix_payload(path, payload): with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: sock.settimeout(5) @@ -445,11 +440,6 @@ def _send_tls_payload(port, payload, cafile): tls_sock.sendall(payload) -def _drop_raw_tls_connection(port, payload): - with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: - sock.sendall(payload) - - def _recv_msgpack_value(sock): sock.settimeout(5) data = sock.recv(4096) @@ -654,9 +644,10 @@ def test_in_forward_workers_drop_partial_connections_and_continue(): service.wait_for_log_message("with 4 workers", timeout=10) try: + # Send partial MessagePack bytes and close to exercise drop cleanup. with ThreadPoolExecutor(max_workers=dropped_connections) as executor: list(executor.map( - lambda _: _drop_partial_tcp_payload(service.flb_listener_port, b"\x93\xa4test"), + lambda _: _send_tcp_payload(service.flb_listener_port, b"\x93\xa4test"), range(dropped_connections), )) @@ -895,10 +886,11 @@ def test_in_forward_tls_workers_drop_bad_handshakes_and_continue(): service.wait_for_log_message("with 4 workers", timeout=10) try: + # Send raw non-TLS bytes and close to exercise handshake cleanup. with ThreadPoolExecutor(max_workers=dropped_connections) as executor: list(executor.map( - lambda i: _drop_raw_tls_connection(service.flb_listener_port, - f"not-tls-{i}".encode("utf-8")), + lambda i: _send_tcp_payload(service.flb_listener_port, + f"not-tls-{i}".encode("utf-8")), range(dropped_connections), )) diff --git a/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py b/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py index 8553f5e4fa1..5cf05e5f4c8 100644 --- a/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py +++ b/tests/integration/scenarios/in_udp/tests/test_in_udp_001.py @@ -73,8 +73,14 @@ def flattened_records(self): return records def wait_for_record_count(self, minimum_count, timeout=10): + def records_ready(): + records = self.flattened_records() + if len(records) >= minimum_count: + return records + return None + return self.service.wait_for_condition( - lambda: self.flattened_records() if len(self.flattened_records()) >= minimum_count else None, + records_ready, timeout=timeout, interval=0.2, description=f"{minimum_count} forwarded UDP payloads", @@ -167,6 +173,7 @@ def test_in_udp_workers_drop_malformed_datagrams_and_continue(): range(valid_records), )) records = service.wait_for_record_count(valid_records, timeout=20) + assert len(records) == valid_records finally: service.stop() From 4182c4b7bb1b30b5bfcf5c56864656ba201b6b7c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 6 May 2026 11:31:47 -0600 Subject: [PATCH 13/18] tests: integration: stop forward workers on startup wait Signed-off-by: Eduardo Silva --- .../scenarios/in_forward/tests/test_in_forward_001.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py index 0215c7f6be7..4eaaabe9f5c 100644 --- a/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py +++ b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py @@ -616,9 +616,9 @@ def test_in_forward_workers_concurrent_message_mode_records(): total_records = 16 service = Service("in_forward_workers.yaml") service.start() - service.wait_for_log_message("with 4 workers", timeout=10) try: + service.wait_for_log_message("with 4 workers", timeout=10) payloads = [ _message_mode_payload(TEST_TAG, {"message": f"worker-{i}", "value": i}) for i in range(total_records) @@ -641,9 +641,9 @@ def test_in_forward_workers_drop_partial_connections_and_continue(): valid_records = 8 service = Service("in_forward_workers.yaml") service.start() - service.wait_for_log_message("with 4 workers", timeout=10) try: + service.wait_for_log_message("with 4 workers", timeout=10) # Send partial MessagePack bytes and close to exercise drop cleanup. with ThreadPoolExecutor(max_workers=dropped_connections) as executor: list(executor.map( @@ -857,9 +857,9 @@ def test_in_forward_tls_workers_concurrent_message_mode_records(): total_records = 16 service = Service("in_forward_tls_workers.yaml") service.start() - service.wait_for_log_message("with 4 workers", timeout=10) try: + service.wait_for_log_message("with 4 workers", timeout=10) with ThreadPoolExecutor(max_workers=total_records) as executor: list(executor.map( lambda i: _send_tls_payload(service.flb_listener_port, @@ -883,9 +883,9 @@ def test_in_forward_tls_workers_drop_bad_handshakes_and_continue(): valid_records = 8 service = Service("in_forward_tls_workers.yaml") service.start() - service.wait_for_log_message("with 4 workers", timeout=10) try: + service.wait_for_log_message("with 4 workers", timeout=10) # Send raw non-TLS bytes and close to exercise handshake cleanup. with ThreadPoolExecutor(max_workers=dropped_connections) as executor: list(executor.map( From e4823f8bcebc951d2b66f29ee1c91efcfb7fe5ba Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 11 May 2026 19:16:11 -0600 Subject: [PATCH 14/18] lib: cfl: upgrade to v0.7.0 Signed-off-by: Eduardo Silva --- lib/cfl/.github/workflows/build.yaml | 55 ++-- lib/cfl/.github/workflows/lint.yaml | 4 +- lib/cfl/.github/workflows/packages.yaml | 10 +- lib/cfl/AGENTS.md | 76 ++++++ lib/cfl/CMakeLists.txt | 4 +- lib/cfl/README.md | 71 +++++- lib/cfl/include/cfl/cfl.h | 2 + lib/cfl/include/cfl/cfl_array.h | 18 ++ lib/cfl/include/cfl/cfl_atomic.h | 31 +++ lib/cfl/include/cfl/cfl_checksum.h | 1 + lib/cfl/include/cfl/cfl_container.h | 48 ++++ lib/cfl/include/cfl/cfl_kv.h | 2 + lib/cfl/include/cfl/cfl_kvlist.h | 8 + lib/cfl/include/cfl/cfl_list.h | 52 ++++ lib/cfl/include/cfl/cfl_object.h | 5 + lib/cfl/include/cfl/cfl_sds.h | 15 +- lib/cfl/include/cfl/cfl_time.h | 2 +- lib/cfl/include/cfl/cfl_utils.h | 2 + lib/cfl/include/cfl/cfl_variant.h | 4 + lib/cfl/src/CMakeLists.txt | 28 +- lib/cfl/src/cfl.c | 3 +- lib/cfl/src/cfl_array.c | 126 +++++++-- lib/cfl/src/cfl_atomic_clang.c | 42 +++ lib/cfl/src/cfl_atomic_gcc.c | 42 +++ lib/cfl/src/cfl_atomic_generic.c | 121 +++++++++ lib/cfl/src/cfl_atomic_msvc.c | 160 ++++++++++++ lib/cfl/src/cfl_checksum.c | 4 + lib/cfl/src/cfl_container.c | 325 ++++++++++++++++++++++++ lib/cfl/src/cfl_kv.c | 39 ++- lib/cfl/src/cfl_kvlist.c | 284 +++++++++++++++++++-- lib/cfl/src/cfl_object.c | 198 ++++++++++++++- lib/cfl/src/cfl_sds.c | 176 ++++++++++--- lib/cfl/src/cfl_utils.c | 76 +++++- lib/cfl/src/cfl_variant.c | 154 ++++++++++- lib/cfl/tests/CMakeLists.txt | 53 ++++ lib/cfl/tests/array.c | 158 ++++++++++++ lib/cfl/tests/atomic_operations.c | 177 +++++++++++++ lib/cfl/tests/checksum.c | 38 +++ lib/cfl/tests/headers.c | 49 ++++ lib/cfl/tests/kv.c | 23 ++ lib/cfl/tests/kvlist.c | 263 +++++++++++++++++++ lib/cfl/tests/object.c | 259 +++++++++++++++++++ lib/cfl/tests/sds.c | 74 ++++++ lib/cfl/tests/utils.c | 15 ++ lib/cfl/tests/variant.c | 166 +++++++++++- 45 files changed, 3297 insertions(+), 166 deletions(-) create mode 100644 lib/cfl/AGENTS.md create mode 100644 lib/cfl/include/cfl/cfl_atomic.h create mode 100644 lib/cfl/include/cfl/cfl_container.h create mode 100644 lib/cfl/src/cfl_atomic_clang.c create mode 100644 lib/cfl/src/cfl_atomic_gcc.c create mode 100644 lib/cfl/src/cfl_atomic_generic.c create mode 100644 lib/cfl/src/cfl_atomic_msvc.c create mode 100644 lib/cfl/src/cfl_container.c create mode 100644 lib/cfl/tests/atomic_operations.c create mode 100644 lib/cfl/tests/checksum.c create mode 100644 lib/cfl/tests/headers.c diff --git a/lib/cfl/.github/workflows/build.yaml b/lib/cfl/.github/workflows/build.yaml index b33123dc1cf..22f3d80057d 100644 --- a/lib/cfl/.github/workflows/build.yaml +++ b/lib/cfl/.github/workflows/build.yaml @@ -15,10 +15,10 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, windows-2019] + os: [windows-latest, windows-2022] steps: - - uses: actions/checkout@v4 - - name: Build on ${{ matrix.os }} with vs-2019 + - uses: actions/checkout@v6 + - name: Build on ${{ matrix.os }} with MSVC run: | .\scripts\win_build.bat - name: Run unit tests. @@ -31,9 +31,9 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, windows-2019] + os: [windows-latest, windows-2022] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Get dependencies w/ chocolatey uses: crazy-max/ghaction-chocolatey@v3 with: @@ -106,7 +106,11 @@ jobs: steps: - name: Set up base image dependencies run: | - apt-get update + printf '%s\n' \ + 'deb http://archive.debian.org/debian buster main' \ + 'deb http://archive.debian.org/debian-security buster/updates main' \ + > /etc/apt/sources.list + apt-get -o Acquire::Check-Valid-Until=false update apt-get install -y build-essential wget make gcc g++ - name: Install CMake 3.20.0 @@ -122,7 +126,7 @@ jobs: # Confirm CMake installation /usr/local/bin/cmake --version - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Run compilation run: | @@ -138,7 +142,7 @@ jobs: os: [ubuntu-latest] compiler: [ gcc, clang ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build on ${{ matrix.os }} with ${{ matrix.compiler }} uses: uraimo/run-on-arch-action@v3.0.1 with: @@ -167,7 +171,7 @@ jobs: os: [ubuntu-latest, macos-latest] compiler: [ gcc, clang ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build on ${{ matrix.os }} with ${{ matrix.compiler }} run: | echo "CC = $CC, CXX = $CXX" @@ -193,36 +197,21 @@ jobs: contents: read runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: true - dependencies_debian: 'wget' - - name: Install CMake 3.20.0 - run: | - CMAKE_VERSION=3.20.0 - wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh - chmod +x cmake-${CMAKE_VERSION}-linux-x86_64.sh - - # Create a writable temporary directory - mkdir -p /tmp/cmake - - # Install CMake to /tmp/cmake - ./cmake-${CMAKE_VERSION}-linux-x86_64.sh --skip-license --prefix=/tmp/cmake - - # Add CMake to the PATH - echo 'export PATH=/tmp/cmake/bin:$PATH' >> ~/.bashrc - source ~/.bashrc - - # Verify installation - /tmp/cmake/bin/cmake --version - - - uses: actions/checkout@v4 + - uses: docker://lpenz/ghaction-cmake:0.19 with: + pre_command: | + CMAKE_VERSION=3.20.0 + curl -fsSLO https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh + chmod +x cmake-${CMAKE_VERSION}-linux-x86_64.sh + ./cmake-${CMAKE_VERSION}-linux-x86_64.sh --skip-license --prefix=/usr/local + cmake --version preset: ${{ matrix.preset }} - cmakeflags: '-DCFL_TESTS=On -DCFL_DEV=on .' - build_command: /tmp/cmake/bin/cmake && make all + build_command: make all # this job provides the single required status for PRs to be merged into main. # instead of updating the protected branch status in github, developers can update the needs section below diff --git a/lib/cfl/.github/workflows/lint.yaml b/lib/cfl/.github/workflows/lint.yaml index e18e66b44f2..12f97917cf4 100644 --- a/lib/cfl/.github/workflows/lint.yaml +++ b/lib/cfl/.github/workflows/lint.yaml @@ -10,7 +10,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: ludeeus/action-shellcheck@master with: ignore_paths: lib @@ -21,7 +21,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - run: | echo "::add-matcher::.github/actionlint-matcher.json" bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) diff --git a/lib/cfl/.github/workflows/packages.yaml b/lib/cfl/.github/workflows/packages.yaml index 1611b7a676b..a73e49b19ae 100644 --- a/lib/cfl/.github/workflows/packages.yaml +++ b/lib/cfl/.github/workflows/packages.yaml @@ -18,7 +18,7 @@ jobs: matrix: format: [ rpm, deb ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: uraimo/run-on-arch-action@v3.0.1 name: Build the ${{matrix.format}} packages with: @@ -36,7 +36,7 @@ jobs: echo ${{ matrix.format }} | awk '{print toupper($0)}' | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ matrix.format }}-arm64 path: | @@ -51,14 +51,14 @@ jobs: runs-on: [ ubuntu-latest ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build the ${{matrix.format}} packages run: | cmake . echo ${{ matrix.format }} | awk '{print toupper($0)}' | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ matrix.format }}-amd64 path: | @@ -74,7 +74,7 @@ jobs: contents: write steps: - name: Download all artefacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: path: artifacts/ diff --git a/lib/cfl/AGENTS.md b/lib/cfl/AGENTS.md new file mode 100644 index 00000000000..8fccc629736 --- /dev/null +++ b/lib/cfl/AGENTS.md @@ -0,0 +1,76 @@ +# Repository Guidelines + +## Preferred Commands +- Configure with tests: `cmake -S . -B build -DCFL_TESTS=On` +- Build: `cmake --build build -j8` +- Run all tests: `ctest --test-dir build --output-on-failure` +- Run a focused test: `ctest --test-dir build -R --output-on-failure` +- Check staged or local patches for whitespace before closing a change: + `git diff --check` + +## Project Structure & Module Organization +CFL is a small C library built with CMake. + +- `include/cfl/`: public CFL headers. +- `src/`: library implementation files. +- `tests/`: acutest-based unit tests. +- `lib/xxhash/`: bundled xxHash dependency. +- `cmake/`: project CMake helpers. + +Keep changes scoped to the affected module. Put public declarations in +`include/cfl/`, implementation in `src/`, and matching unit coverage in +`tests/` when behavior changes. + +## Build, Test, and Development Commands +- `cmake -S . -B build -DCFL_TESTS=On`: configure the project with tests. +- `cmake --build build -j8`: compile the static library and tests. +- `ctest --test-dir build --output-on-failure`: run the enabled test suite. +- `ctest --test-dir build -R cfl-test- --output-on-failure`: run a + focused unit test. + +Prefer targeted test runs while iterating, then run the full enabled suite +before closing changes that touch shared code or public APIs. + +## Coding Style & Naming Conventions +- Follow the existing Apache-style C conventions used in this repository. +- Use 4-space indentation and keep lines readable; avoid unnecessary wrapping. +- Always use braces for `if/else/while/do` blocks. +- Put function opening braces on the next line: + `int fn(void)\n{ ... }` +- Declare variables at the start of functions, not mid-block. +- Prefer descriptive `snake_case` names with the `cfl_` prefix for public APIs. +- Use `CFL_TRUE` and `CFL_FALSE` for CFL boolean-style return values. +- Use `/* ... */` comments, and add comments only where they clarify non-obvious + behavior. +- Keep public headers self-contained by including the standard headers they need. + +## Testing Guidelines +- Add or update acutest unit coverage for behavior changes. +- Keep tests close to the affected module and name test binaries through + `tests/CMakeLists.txt`. +- Validate both success and failure paths for parsers, containers, allocation + handling, and boundary conditions. +- Run broader coverage when changing shared headers, CMake wiring, memory + ownership, or common data structures. +- If a relevant test cannot be run, report the exact blocker in the final + response. + +## Commit & Pull Request Guidelines +- Follow observed local history style: + `component: short imperative description` + Examples: `sds: do not export internal sds_alloc function`, + `build: bump to v0.6.2`, `atomic: add atomic operations API`. +- Keep each commit scoped to one component or interface. +- Keep subject/body lines concise; use a body when the reason or scope is not + obvious from the subject. +- Do not mix unrelated code and documentation updates in one commit unless the + user explicitly asks for a combined commit. +- Do not rewrite history, amend commits, create remote branches, or open pull + requests unless explicitly requested. + +## Agent Action Limits +- Do not modify repositories or files outside this project unless the user + explicitly asks. +- Do not revert user changes outside the requested scope. +- Preserve unrelated untracked or modified files in the worktree. +- Prefer minimal patches that avoid unrelated formatting or refactoring churn. diff --git a/lib/cfl/CMakeLists.txt b/lib/cfl/CMakeLists.txt index 4895700264a..505ac9ab8ed 100644 --- a/lib/cfl/CMakeLists.txt +++ b/lib/cfl/CMakeLists.txt @@ -5,8 +5,8 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # C Floppy Version set(CFL_VERSION_MAJOR 0) -set(CFL_VERSION_MINOR 6) -set(CFL_VERSION_PATCH 1) +set(CFL_VERSION_MINOR 7) +set(CFL_VERSION_PATCH 0) set(CFL_VERSION_STR "${CFL_VERSION_MAJOR}.${CFL_VERSION_MINOR}.${CFL_VERSION_PATCH}") # Configuration options diff --git a/lib/cfl/README.md b/lib/cfl/README.md index 4050bef8b27..c81b4bafd22 100644 --- a/lib/cfl/README.md +++ b/lib/cfl/README.md @@ -1,18 +1,71 @@ # CFL -CFL is a tiny library that provides interfaces for data structures, originally created to satisfy the needs of Fluent Bit and other libraries used internally like CMetrics and CTraces projects. +CFL is a tiny C library that provides small data-structure and utility +interfaces. It was originally created to satisfy the needs of Fluent Bit and +related libraries such as CMetrics and CTraces. -note: The name doesn't mean anything specific, you can call it `c:\ floppy` if you want. +Note: The name does not mean anything specific; you can call it `c:\ floppy` +if you want. ## Interfaces -- cfl_sds: string manipulation -- cfl_list: linked list -- cfl_kv: key value pairs by using a linked list (cfl_list) -- cfl_array: array of elements -- cfl_variant: interface to manage contexts with vairant types -- cfl_time: time utilities -- cfl_hash: 64bit hashing functions +Applications can include `` to pull in the common CFL interfaces. +Specialized headers are also available under `include/cfl/` when a caller only +needs one module. + +### Core + +- `cfl_init()`: initializes library-level facilities. +- `cfl_version()`: returns the CFL version string. +- `CFL_TRUE` and `CFL_FALSE`: common boolean-style constants used by CFL APIs. + +### Data Structures + +- `cfl_sds`: dynamic string storage with explicit length and allocation + tracking. It supports creation from strings or buffers, growth, concatenation, + formatted writes, length updates, and destruction. +- `cfl_list`: intrusive doubly linked list helpers. It provides initialization, + add, append, prepend, delete, concatenate, size, entry lookup, and safe + iteration macros. +- `cfl_kv`: SDS-backed string key/value entries stored in a `cfl_list`. This is + useful for simple string maps where values are plain strings. +- `cfl_variant`: tagged value container for bool, signed integer, unsigned + integer, double, null, reference, string, bytes, array, and key/value list + values. +- `cfl_array`: ordered collection of `cfl_variant` entries. It supports fixed + or resizable arrays, append helpers for all variant types, fetch by index, + removal, size inspection, and printing. +- `cfl_kvlist`: string-keyed map whose values are `cfl_variant` instances. It + supports typed insert helpers, size-aware key APIs, fetch, contains, remove, + count, and printing. +- `cfl_object`: generic wrapper that can hold a `cfl_kvlist`, `cfl_variant`, or + `cfl_array` and print the associated value. + +### Utilities + +- `cfl_atomic`: 64-bit atomic initialization, compare-exchange, store, and load + operations with platform-specific backends. +- `cfl_time`: wall-clock timestamp helper that returns nanoseconds. +- `cfl_hash`: naming wrappers around xxHash 64-bit and 128-bit hashing APIs. +- `cfl_checksum`: CRC32C checksum helper. +- `cfl_utils`: string split helpers, including quote-aware splitting, with + results returned as `cfl_list` entries. +- `cfl_log`: runtime error reporting helpers. + +### Support Headers + +- `cfl_compat`: platform compatibility macros. +- `cfl_found`: lightweight include-probe helper for parent projects. +- `cfl_info`: generated build information and feature flags. +- `cfl_version`: version macros. + +## Build and Test + +```sh +cmake -S . -B build -DCFL_TESTS=On +cmake --build build -j8 +ctest --test-dir build --output-on-failure +``` ## License diff --git a/lib/cfl/include/cfl/cfl.h b/lib/cfl/include/cfl/cfl.h index 921767e8091..3491e0c8bcb 100644 --- a/lib/cfl/include/cfl/cfl.h +++ b/lib/cfl/include/cfl/cfl.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include #include #include +#include #include #include #include diff --git a/lib/cfl/include/cfl/cfl_array.h b/lib/cfl/include/cfl/cfl_array.h index 331a4cbc6b6..80923442c2a 100644 --- a/lib/cfl/include/cfl/cfl_array.h +++ b/lib/cfl/include/cfl/cfl_array.h @@ -21,8 +21,13 @@ #define CFL_ARRAY_H #include +#include +#include + #include +struct cfl_kvlist; + struct cfl_array { int resizable; struct cfl_variant **entries; @@ -36,6 +41,10 @@ void cfl_array_destroy(struct cfl_array *array); static inline struct cfl_variant *cfl_array_fetch_by_index(struct cfl_array *array, size_t position) { + if (array == NULL) { + return NULL; + } + if (position >= array->entry_count) { return NULL; } @@ -45,9 +54,18 @@ static inline struct cfl_variant *cfl_array_fetch_by_index(struct cfl_array *arr static inline size_t cfl_array_size(struct cfl_array *array) { + if (array == NULL) { + return 0; + } + return array->entry_count; } +/* + * Append APIs take ownership of the value on success. A variant, array, or + * kvlist must have a single owning parent; inserting the same pointer into + * multiple containers is unsupported and can result in double-free. + */ int cfl_array_append(struct cfl_array *array, struct cfl_variant *value); int cfl_array_append_string(struct cfl_array *array, char *value); int cfl_array_append_string_s(struct cfl_array *array, char *str, size_t str_len, int referenced); diff --git a/lib/cfl/include/cfl/cfl_atomic.h b/lib/cfl/include/cfl/cfl_atomic.h new file mode 100644 index 00000000000..e113a991afb --- /dev/null +++ b/lib/cfl/include/cfl/cfl_atomic.h @@ -0,0 +1,31 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2022 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CFL_ATOMIC_H +#define CFL_ATOMIC_H + +#include + +int cfl_atomic_initialize(); +int cfl_atomic_compare_exchange(uint64_t *storage, uint64_t old_value, + uint64_t new_value); +void cfl_atomic_store(uint64_t *storage, uint64_t new_value); +uint64_t cfl_atomic_load(uint64_t *storage); + +#endif diff --git a/lib/cfl/include/cfl/cfl_checksum.h b/lib/cfl/include/cfl/cfl_checksum.h index d9690aa8ca2..32e26fbf63f 100644 --- a/lib/cfl/include/cfl/cfl_checksum.h +++ b/lib/cfl/include/cfl/cfl_checksum.h @@ -20,6 +20,7 @@ #ifndef CFL_CHECKSUM_H #define CFL_CHECKSUM_H +#include #include uint32_t cfl_checksum_crc32c(unsigned char *buffer, size_t length); diff --git a/lib/cfl/include/cfl/cfl_container.h b/lib/cfl/include/cfl/cfl_container.h new file mode 100644 index 00000000000..6583e593482 --- /dev/null +++ b/lib/cfl/include/cfl/cfl_container.h @@ -0,0 +1,48 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2026 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CFL_CONTAINER_H +#define CFL_CONTAINER_H + +#include +#include +#include + +int cfl_container_array_contains_array(struct cfl_array *array, + struct cfl_array *target); +int cfl_container_array_contains_kvlist(struct cfl_array *array, + struct cfl_kvlist *target); +int cfl_container_array_contains_variant(struct cfl_array *array, + struct cfl_variant *target); + +int cfl_container_kvlist_contains_array(struct cfl_kvlist *kvlist, + struct cfl_array *target); +int cfl_container_kvlist_contains_kvlist(struct cfl_kvlist *kvlist, + struct cfl_kvlist *target); +int cfl_container_kvlist_contains_variant(struct cfl_kvlist *kvlist, + struct cfl_variant *target); + +int cfl_container_variant_contains_array(struct cfl_variant *variant, + struct cfl_array *target); +int cfl_container_variant_contains_kvlist(struct cfl_variant *variant, + struct cfl_kvlist *target); +int cfl_container_variant_contains_variant(struct cfl_variant *variant, + struct cfl_variant *target); + +#endif diff --git a/lib/cfl/include/cfl/cfl_kv.h b/lib/cfl/include/cfl/cfl_kv.h index f44c5be63d9..812820ecc39 100644 --- a/lib/cfl/include/cfl/cfl_kv.h +++ b/lib/cfl/include/cfl/cfl_kv.h @@ -20,6 +20,8 @@ #ifndef CFL_KV_H #define CFL_KV_H +#include + #include #include #include diff --git a/lib/cfl/include/cfl/cfl_kvlist.h b/lib/cfl/include/cfl/cfl_kvlist.h index ab25b162389..b2cb2a406e3 100644 --- a/lib/cfl/include/cfl/cfl_kvlist.h +++ b/lib/cfl/include/cfl/cfl_kvlist.h @@ -21,6 +21,9 @@ #define CFL_KVLIST_H #include +#include +#include + #include #include #include @@ -38,6 +41,11 @@ struct cfl_kvlist { struct cfl_kvlist *cfl_kvlist_create(); void cfl_kvlist_destroy(struct cfl_kvlist *list); +/* + * Insert APIs take ownership of array, kvlist, and variant values on success. + * A value must have a single owning parent; inserting the same pointer into + * multiple containers is unsupported and can result in double-free. + */ int cfl_kvlist_insert_string(struct cfl_kvlist *list, char *key, char *value); diff --git a/lib/cfl/include/cfl/cfl_list.h b/lib/cfl/include/cfl/cfl_list.h index f2709288017..47d6ea1cc64 100644 --- a/lib/cfl/include/cfl/cfl_list.h +++ b/lib/cfl/include/cfl/cfl_list.h @@ -31,6 +31,14 @@ #include #include +#ifndef CFL_FALSE +#define CFL_FALSE 0 +#endif + +#ifndef CFL_TRUE +#define CFL_TRUE !CFL_FALSE +#endif + #ifdef _WIN32 /* Windows */ #define cfl_container_of(address, type, field) ((type *)( \ @@ -53,6 +61,10 @@ struct cfl_list { static inline int cfl_list_is_empty(struct cfl_list *head) { + if (head == NULL) { + return 1; + } + if (head->next == head) { return 1; } @@ -62,6 +74,10 @@ static inline int cfl_list_is_empty(struct cfl_list *head) static inline void cfl_list_init(struct cfl_list *list) { + if (list == NULL) { + return; + } + list->next = list; list->prev = list; } @@ -69,12 +85,20 @@ static inline void cfl_list_init(struct cfl_list *list) static inline void __cfl_list_del(struct cfl_list *prev, struct cfl_list *next) { + if (prev == NULL || next == NULL) { + return; + } + prev->next = next; next->prev = prev; } static inline void cfl_list_del(struct cfl_list *entry) { + if (entry == NULL) { + return; + } + __cfl_list_del(entry->prev, entry->next); entry->prev = NULL; @@ -85,6 +109,10 @@ static inline void __cfl_list_add(struct cfl_list *_new, struct cfl_list *prev, struct cfl_list *next) { + if (_new == NULL || prev == NULL || next == NULL) { + return; + } + next->prev = _new; _new->next = next; _new->prev = prev; @@ -94,6 +122,10 @@ static inline void __cfl_list_add(struct cfl_list *_new, static inline void cfl_list_add(struct cfl_list *_new, struct cfl_list *head) { + if (_new == NULL || head == NULL) { + return; + } + __cfl_list_add(_new, head->prev, head); } @@ -134,6 +166,10 @@ static inline void cfl_list_add_before(struct cfl_list *_new, static inline void cfl_list_append(struct cfl_list *_new, struct cfl_list *head) { + if (_new == NULL || head == NULL) { + return; + } + if (cfl_list_is_empty(head)) { __cfl_list_add(_new, head->prev, head); } @@ -147,6 +183,10 @@ static inline void cfl_list_append(struct cfl_list *_new, static inline void cfl_list_prepend(struct cfl_list *_new, struct cfl_list *head) { + if (_new == NULL || head == NULL) { + return; + } + if (cfl_list_is_empty(head)) { __cfl_list_add(_new, head->prev, head); } @@ -162,6 +202,10 @@ static inline int cfl_list_size(struct cfl_list *head) int ret = 0; struct cfl_list *it; + if (head == NULL) { + return 0; + } + for (it = head->next; it != head; it = it->next, ret++); return ret; @@ -175,6 +219,10 @@ static inline void cfl_list_entry_init(struct cfl_list *entry) static inline int cfl_list_entry_is_orphan(struct cfl_list *entry) { + if (entry == NULL) { + return CFL_TRUE; + } + if (entry->next != NULL && entry->prev != NULL) { return CFL_FALSE; @@ -187,6 +235,10 @@ static inline void cfl_list_cat(struct cfl_list *list, struct cfl_list *head) { struct cfl_list *last; + if (list == NULL || head == NULL) { + return; + } + last = head->prev; last->next = list->next; list->next->prev = last; diff --git a/lib/cfl/include/cfl/cfl_object.h b/lib/cfl/include/cfl/cfl_object.h index f959ac4ef3b..97503c4cc02 100644 --- a/lib/cfl/include/cfl/cfl_object.h +++ b/lib/cfl/include/cfl/cfl_object.h @@ -20,6 +20,11 @@ #ifndef CFL_OBJECT_H #define CFL_OBJECT_H +#include + +#include +#include + enum { CFL_OBJECT_NONE = 0, CFL_OBJECT_KVLIST = 1, diff --git a/lib/cfl/include/cfl/cfl_sds.h b/lib/cfl/include/cfl/cfl_sds.h index 9ec5b347ed5..a923882a616 100644 --- a/lib/cfl/include/cfl/cfl_sds.h +++ b/lib/cfl/include/cfl/cfl_sds.h @@ -28,6 +28,7 @@ #include #include #include +#include #define CFL_SDS_HEADER_SIZE (sizeof(uint64_t) + sizeof(uint64_t)) @@ -45,7 +46,19 @@ struct cfl_sds { static inline void cfl_sds_len_set(cfl_sds_t s, size_t len) { - CFL_SDS_HEADER(s)->len = len; + struct cfl_sds *head; + + if (s == NULL) { + return; + } + + head = CFL_SDS_HEADER(s); + if (len > head->alloc) { + return; + } + + head->len = len; + s[len] = '\0'; } size_t cfl_sds_avail(cfl_sds_t s); diff --git a/lib/cfl/include/cfl/cfl_time.h b/lib/cfl/include/cfl/cfl_time.h index 9c141c1b880..592978f0814 100644 --- a/lib/cfl/include/cfl/cfl_time.h +++ b/lib/cfl/include/cfl/cfl_time.h @@ -20,7 +20,7 @@ #ifndef CFL_TIME_H #define CFL_TIME_H -#include +#include uint64_t cfl_time_now(); diff --git a/lib/cfl/include/cfl/cfl_utils.h b/lib/cfl/include/cfl/cfl_utils.h index db9d873ee1c..e4b7dc99c03 100644 --- a/lib/cfl/include/cfl/cfl_utils.h +++ b/lib/cfl/include/cfl/cfl_utils.h @@ -2,7 +2,9 @@ #define CFL_UTILS_H #include /* off_t */ + #include +#include #include struct cfl_split_entry { diff --git a/lib/cfl/include/cfl/cfl_variant.h b/lib/cfl/include/cfl/cfl_variant.h index 4f9953a6956..685f51a91bd 100644 --- a/lib/cfl/include/cfl/cfl_variant.h +++ b/lib/cfl/include/cfl/cfl_variant.h @@ -23,6 +23,10 @@ #include #include #include +#include +#include + +#include #define CFL_VARIANT_BOOL 1 #define CFL_VARIANT_INT 2 diff --git a/lib/cfl/src/CMakeLists.txt b/lib/cfl/src/CMakeLists.txt index f09a5c3d8b1..62c6cffb72a 100644 --- a/lib/cfl/src/CMakeLists.txt +++ b/lib/cfl/src/CMakeLists.txt @@ -8,13 +8,39 @@ set(src cfl_object.c cfl_array.c cfl_variant.c + cfl_container.c cfl_checksum.c cfl_utils.c ) +set(CFL_ATOMIC_NEEDS_THREADS Off) + +if(MSVC) + set(PLATFORM_SPECIFIC_ATOMIC_MODULE cfl_atomic_msvc.c) +elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") + set(PLATFORM_SPECIFIC_ATOMIC_MODULE cfl_atomic_clang.c) +elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "AppleClang") + set(PLATFORM_SPECIFIC_ATOMIC_MODULE cfl_atomic_clang.c) +elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") + set(PLATFORM_SPECIFIC_ATOMIC_MODULE cfl_atomic_gcc.c) +else() + set(PLATFORM_SPECIFIC_ATOMIC_MODULE cfl_atomic_generic.c) + set(CFL_ATOMIC_NEEDS_THREADS On) +endif() + +set(src + ${src} + ${PLATFORM_SPECIFIC_ATOMIC_MODULE} + ) + # Static Library add_library(cfl-static STATIC ${src}) -target_link_libraries(cfl-static xxhash) +target_link_libraries(cfl-static PRIVATE xxhash) + +if(CFL_ATOMIC_NEEDS_THREADS) + find_package(Threads REQUIRED) + target_link_libraries(cfl-static PUBLIC Threads::Threads) +endif() # Install Library if(MSVC) diff --git a/lib/cfl/src/cfl.c b/lib/cfl/src/cfl.c index 426319a2747..942d3c34065 100644 --- a/lib/cfl/src/cfl.c +++ b/lib/cfl/src/cfl.c @@ -21,11 +21,10 @@ int cfl_init() { - return 0; + return cfl_atomic_initialize(); } char *cfl_version() { return CFL_VERSION_STR; } - diff --git a/lib/cfl/src/cfl_array.c b/lib/cfl/src/cfl_array.c index 916b053e4af..10039dfab34 100644 --- a/lib/cfl/src/cfl_array.c +++ b/lib/cfl/src/cfl_array.c @@ -21,9 +21,14 @@ #include #include +#include + +#include + struct cfl_array *cfl_array_create(size_t slot_count) { struct cfl_array *array; + size_t alloc_count; array = malloc(sizeof(struct cfl_array)); if (array == NULL) { @@ -35,7 +40,16 @@ struct cfl_array *cfl_array_create(size_t slot_count) array->resizable = CFL_FALSE; /* allocate fixed number of entries */ - array->entries = calloc(slot_count, sizeof(void *)); + alloc_count = slot_count; + if (alloc_count == 0) { + alloc_count = 1; + } + if (alloc_count > SIZE_MAX / sizeof(void *)) { + free(array); + return NULL; + } + + array->entries = calloc(alloc_count, sizeof(void *)); if (array->entries == NULL) { cfl_errno(); free(array); @@ -70,6 +84,10 @@ void cfl_array_destroy(struct cfl_array *array) int cfl_array_resizable(struct cfl_array *array, int v) { + if (array == NULL) { + return -1; + } + if (v != CFL_TRUE && v != CFL_FALSE) { return -1; } @@ -81,6 +99,10 @@ int cfl_array_resizable(struct cfl_array *array, int v) int cfl_array_remove_by_index(struct cfl_array *array, size_t position) { + if (array == NULL) { + return -1; + } + if (position >= array->entry_count) { return -1; } @@ -105,6 +127,10 @@ int cfl_array_remove_by_reference(struct cfl_array *array, { size_t index; + if (array == NULL || value == NULL) { + return -1; + } + for (index = 0 ; index < array->entry_count ; index++) { if (array->entries[index] == value) { return cfl_array_remove_by_index(array, index); @@ -120,6 +146,32 @@ int cfl_array_append(struct cfl_array *array, void *tmp; size_t new_slot_count; size_t new_size; + size_t base_slot_count; + + if (array == NULL || value == NULL) { + return -1; + } + + if (cfl_container_array_contains_variant(array, value)) { + return -1; + } + + /* Only container-valued variants can participate in container cycles. */ + if (value->type == CFL_VARIANT_ARRAY || value->type == CFL_VARIANT_KVLIST) { + if (cfl_container_variant_contains_array(value, array)) { + return -1; + } + + if (value->type == CFL_VARIANT_ARRAY && + cfl_container_array_contains_array(array, value->data.as_array)) { + return -1; + } + + if (value->type == CFL_VARIANT_KVLIST && + cfl_container_array_contains_kvlist(array, value->data.as_kvlist)) { + return -1; + } + } if (array->entry_count >= array->slot_count) { /* @@ -129,17 +181,21 @@ int cfl_array_append(struct cfl_array *array, * it controls the input data. */ if (array->resizable) { - - /* - * if the array size is zero (created as an array of 0 slots), - * change the size to 1 so the resize can work properly - */ - if (array->slot_count == 0) { - array->slot_count = 1; + base_slot_count = array->slot_count; + if (base_slot_count == 0) { + base_slot_count = 1; } /* set new number of slots and total size */ - new_slot_count = (array->slot_count * 2); + if (base_slot_count > SIZE_MAX / 2) { + return -1; + } + + new_slot_count = (base_slot_count * 2); + if (new_slot_count > SIZE_MAX / sizeof(void *)) { + return -1; + } + new_size = (new_slot_count * sizeof(void *)); tmp = realloc(array->entries, new_size); @@ -360,6 +416,15 @@ int cfl_array_append_array(struct cfl_array *array, struct cfl_array *value) struct cfl_variant *value_instance; int result; + if (array == NULL || value == NULL) { + return -1; + } + + if (cfl_container_array_contains_array(array, value) || + cfl_container_array_contains_array(value, array)) { + return -1; + } + value_instance = cfl_variant_create_from_array(value); if (value_instance == NULL) { @@ -381,6 +446,10 @@ int cfl_array_append_new_array(struct cfl_array *array, size_t size) int result; struct cfl_array *value; + if (array == NULL) { + return -1; + } + value = cfl_array_create(size); if (value == NULL) { @@ -388,8 +457,7 @@ int cfl_array_append_new_array(struct cfl_array *array, size_t size) } result = cfl_array_append_array(array, value); - - if (result) { + if (result == -1) { cfl_array_destroy(value); } @@ -401,6 +469,15 @@ int cfl_array_append_kvlist(struct cfl_array *array, struct cfl_kvlist *value) struct cfl_variant *value_instance; int result; + if (array == NULL || value == NULL) { + return -1; + } + + if (cfl_container_array_contains_kvlist(array, value) || + cfl_container_kvlist_contains_array(value, array)) { + return -1; + } + value_instance = cfl_variant_create_from_kvlist(value); if (value_instance == NULL) { return -1; @@ -429,17 +506,36 @@ int cfl_array_print(FILE *fp, struct cfl_array *array) size = array->entry_count; if (size == 0) { - fputs("[]", fp); + if (fputs("[]", fp) == EOF) { + return -1; + } + return 0; } - fputs("[", fp); + if (fputc('[', fp) == EOF) { + return -1; + } + for (i=0; ientries[i]); - fputs(",", fp); + if (ret < 0) { + return -1; + } + + if (fputc(',', fp) == EOF) { + return -1; + } } + ret = cfl_variant_print(fp, array->entries[size-1]); - fputs("]", fp); + if (ret < 0) { + return -1; + } + + if (fputc(']', fp) == EOF) { + return -1; + } return ret; } diff --git a/lib/cfl/src/cfl_atomic_clang.c b/lib/cfl/src/cfl_atomic_clang.c new file mode 100644 index 00000000000..3c320ac2749 --- /dev/null +++ b/lib/cfl/src/cfl_atomic_clang.c @@ -0,0 +1,42 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2022 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +int cfl_atomic_initialize() +{ + return 0; +} + +int cfl_atomic_compare_exchange(uint64_t *storage, + uint64_t old_value, uint64_t new_value) +{ + return __atomic_compare_exchange(storage, &old_value, &new_value, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +} + +void cfl_atomic_store(uint64_t *storage, uint64_t new_value) +{ + __atomic_store_n(storage, new_value, __ATOMIC_SEQ_CST); +} + +uint64_t cfl_atomic_load(uint64_t *storage) +{ + return __atomic_load_n(storage, __ATOMIC_SEQ_CST); +} diff --git a/lib/cfl/src/cfl_atomic_gcc.c b/lib/cfl/src/cfl_atomic_gcc.c new file mode 100644 index 00000000000..3c320ac2749 --- /dev/null +++ b/lib/cfl/src/cfl_atomic_gcc.c @@ -0,0 +1,42 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2022 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +int cfl_atomic_initialize() +{ + return 0; +} + +int cfl_atomic_compare_exchange(uint64_t *storage, + uint64_t old_value, uint64_t new_value) +{ + return __atomic_compare_exchange(storage, &old_value, &new_value, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +} + +void cfl_atomic_store(uint64_t *storage, uint64_t new_value) +{ + __atomic_store_n(storage, new_value, __ATOMIC_SEQ_CST); +} + +uint64_t cfl_atomic_load(uint64_t *storage) +{ + return __atomic_load_n(storage, __ATOMIC_SEQ_CST); +} diff --git a/lib/cfl/src/cfl_atomic_generic.c b/lib/cfl/src/cfl_atomic_generic.c new file mode 100644 index 00000000000..e59bbfa1209 --- /dev/null +++ b/lib/cfl/src/cfl_atomic_generic.c @@ -0,0 +1,121 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2022 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +static pthread_mutex_t cfl_atomic_operation_lock; +static pthread_once_t cfl_atomic_operation_system_once = PTHREAD_ONCE_INIT; +static int cfl_atomic_operation_system_initialized = 0; +static int cfl_atomic_operation_system_status = 0; + +static void cfl_atomic_bootstrap() +{ + cfl_atomic_operation_system_status = + pthread_mutex_init(&cfl_atomic_operation_lock, NULL); + + if (cfl_atomic_operation_system_status == 0) { + cfl_atomic_operation_system_initialized = 1; + } +} + +int cfl_atomic_initialize() +{ + pthread_once(&cfl_atomic_operation_system_once, cfl_atomic_bootstrap); + + if (cfl_atomic_operation_system_status != 0) { + return 1; + } + + return 0; +} + +int cfl_atomic_compare_exchange(uint64_t *storage, + uint64_t old_value, uint64_t new_value) +{ + int result; + + if (cfl_atomic_initialize() != 0 || + cfl_atomic_operation_system_initialized == 0) { + return 0; + } + + result = pthread_mutex_lock(&cfl_atomic_operation_lock); + + if (result != 0) { + return 0; + } + + if (*storage == old_value) { + *storage = new_value; + + result = 1; + } + else { + result = 0; + } + + pthread_mutex_unlock(&cfl_atomic_operation_lock); + + return result; +} + +void cfl_atomic_store(uint64_t *storage, uint64_t new_value) +{ + int result; + + if (cfl_atomic_initialize() != 0 || + cfl_atomic_operation_system_initialized == 0) { + return; + } + + result = pthread_mutex_lock(&cfl_atomic_operation_lock); + + if (result != 0) { + return; + } + + *storage = new_value; + + pthread_mutex_unlock(&cfl_atomic_operation_lock); +} + +uint64_t cfl_atomic_load(uint64_t *storage) +{ + int result; + uint64_t retval; + + if (cfl_atomic_initialize() != 0 || + cfl_atomic_operation_system_initialized == 0) { + return 0; + } + + result = pthread_mutex_lock(&cfl_atomic_operation_lock); + + if (result != 0) { + return 0; + } + + retval = *storage; + + pthread_mutex_unlock(&cfl_atomic_operation_lock); + + return retval; +} diff --git a/lib/cfl/src/cfl_atomic_msvc.c b/lib/cfl/src/cfl_atomic_msvc.c new file mode 100644 index 00000000000..a900420ee95 --- /dev/null +++ b/lib/cfl/src/cfl_atomic_msvc.c @@ -0,0 +1,160 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2022 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#include +#undef WIN32_LEAN_AND_MEAN +#else +#include +#endif + +#ifdef _WIN64 +#include +#endif + +#ifndef _WIN64 +static CRITICAL_SECTION cfl_atomic_operation_lock; +static INIT_ONCE cfl_atomic_operation_system_once = INIT_ONCE_STATIC_INIT; +static int cfl_atomic_operation_system_initialized = 0; +static int cfl_atomic_operation_system_status = 0; + +static BOOL CALLBACK cfl_atomic_bootstrap(PINIT_ONCE once, PVOID parameter, + PVOID *context) +{ + (void) once; + (void) parameter; + (void) context; + + InitializeCriticalSection(&cfl_atomic_operation_lock); + cfl_atomic_operation_system_initialized = 1; + + return TRUE; +} + +int cfl_atomic_initialize() +{ + if (!InitOnceExecuteOnce(&cfl_atomic_operation_system_once, + cfl_atomic_bootstrap, NULL, NULL)) { + cfl_atomic_operation_system_status = 1; + return 1; + } + + cfl_atomic_operation_system_status = 0; + + return 0; +} + +int cfl_atomic_compare_exchange(uint64_t *storage, + uint64_t old_value, uint64_t new_value) +{ + int result; + + if (cfl_atomic_initialize() != 0 || + cfl_atomic_operation_system_initialized == 0 || + cfl_atomic_operation_system_status != 0) { + return 0; + } + + EnterCriticalSection(&cfl_atomic_operation_lock); + + if (*storage == old_value) { + *storage = new_value; + + result = 1; + } + else { + result = 0; + } + + LeaveCriticalSection(&cfl_atomic_operation_lock); + + return result; +} + +void cfl_atomic_store(uint64_t *storage, uint64_t new_value) +{ + if (cfl_atomic_initialize() != 0 || + cfl_atomic_operation_system_initialized == 0 || + cfl_atomic_operation_system_status != 0) { + return; + } + + EnterCriticalSection(&cfl_atomic_operation_lock); + + *storage = new_value; + + LeaveCriticalSection(&cfl_atomic_operation_lock); +} + +uint64_t cfl_atomic_load(uint64_t *storage) +{ + uint64_t result; + + if (cfl_atomic_initialize() != 0 || + cfl_atomic_operation_system_initialized == 0 || + cfl_atomic_operation_system_status != 0) { + return 0; + } + + EnterCriticalSection(&cfl_atomic_operation_lock); + + result = *storage; + + LeaveCriticalSection(&cfl_atomic_operation_lock); + + return result; +} + +#else /* _WIN64 */ + +int cfl_atomic_initialize() +{ + return 0; +} + +int cfl_atomic_compare_exchange(uint64_t *storage, + uint64_t old_value, uint64_t new_value) +{ + __int64 result; + + result = _InterlockedCompareExchange64((volatile __int64 *) storage, + (__int64) new_value, + (__int64) old_value); + + if ((uint64_t) result != old_value) { + return 0; + } + + return 1; +} + +void cfl_atomic_store(uint64_t *storage, uint64_t new_value) +{ + _InterlockedExchange64((volatile __int64 *) storage, (__int64) new_value); +} + +uint64_t cfl_atomic_load(uint64_t *storage) +{ + return (uint64_t) _InterlockedOr64((volatile __int64 *) storage, 0); +} + +#endif diff --git a/lib/cfl/src/cfl_checksum.c b/lib/cfl/src/cfl_checksum.c index 9b0297eaf02..d51acc65ffb 100644 --- a/lib/cfl/src/cfl_checksum.c +++ b/lib/cfl/src/cfl_checksum.c @@ -93,6 +93,10 @@ uint32_t cfl_checksum_crc32c(unsigned char *buffer, size_t length) uint32_t checksum; size_t index; + if (buffer == NULL && length > 0) { + return 0; + } + checksum = 0xFFFFFFFF; /* Keeping in mind that compilers are smart enough to optimize these diff --git a/lib/cfl/src/cfl_container.c b/lib/cfl/src/cfl_container.c new file mode 100644 index 00000000000..5bf7a231bce --- /dev/null +++ b/lib/cfl/src/cfl_container.c @@ -0,0 +1,325 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2026 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#define CFL_CONTAINER_MAX_DEPTH 512 + +static int variant_contains_array(struct cfl_variant *variant, + struct cfl_array *target, + size_t depth); +static int variant_contains_kvlist(struct cfl_variant *variant, + struct cfl_kvlist *target, + size_t depth); +static int variant_contains_variant(struct cfl_variant *variant, + struct cfl_variant *target, + size_t depth); + +static int depth_exceeded(size_t depth) +{ + if (depth > CFL_CONTAINER_MAX_DEPTH) { + return CFL_TRUE; + } + + return CFL_FALSE; +} + +static int array_contains_array(struct cfl_array *array, + struct cfl_array *target, + size_t depth) +{ + size_t i; + + if (array == NULL || target == NULL) { + return CFL_FALSE; + } + + if (depth_exceeded(depth)) { + return CFL_TRUE; + } + + if (array == target) { + return CFL_TRUE; + } + + for (i = 0; i < array->entry_count; i++) { + if (variant_contains_array(array->entries[i], target, depth + 1)) { + return CFL_TRUE; + } + } + + return CFL_FALSE; +} + +static int array_contains_kvlist(struct cfl_array *array, + struct cfl_kvlist *target, + size_t depth) +{ + size_t i; + + if (array == NULL || target == NULL) { + return CFL_FALSE; + } + + if (depth_exceeded(depth)) { + return CFL_TRUE; + } + + for (i = 0; i < array->entry_count; i++) { + if (variant_contains_kvlist(array->entries[i], target, depth + 1)) { + return CFL_TRUE; + } + } + + return CFL_FALSE; +} + +static int array_contains_variant(struct cfl_array *array, + struct cfl_variant *target, + size_t depth) +{ + size_t i; + + if (array == NULL || target == NULL) { + return CFL_FALSE; + } + + if (depth_exceeded(depth)) { + return CFL_TRUE; + } + + for (i = 0; i < array->entry_count; i++) { + if (variant_contains_variant(array->entries[i], target, depth + 1)) { + return CFL_TRUE; + } + } + + return CFL_FALSE; +} + +static int kvlist_contains_array(struct cfl_kvlist *kvlist, + struct cfl_array *target, + size_t depth) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + + if (kvlist == NULL || target == NULL) { + return CFL_FALSE; + } + + if (depth_exceeded(depth)) { + return CFL_TRUE; + } + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + + if (pair != NULL && variant_contains_array(pair->val, target, depth + 1)) { + return CFL_TRUE; + } + } + + return CFL_FALSE; +} + +static int kvlist_contains_kvlist(struct cfl_kvlist *kvlist, + struct cfl_kvlist *target, + size_t depth) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + + if (kvlist == NULL || target == NULL) { + return CFL_FALSE; + } + + if (depth_exceeded(depth)) { + return CFL_TRUE; + } + + if (kvlist == target) { + return CFL_TRUE; + } + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + + if (pair != NULL && variant_contains_kvlist(pair->val, target, depth + 1)) { + return CFL_TRUE; + } + } + + return CFL_FALSE; +} + +static int kvlist_contains_variant(struct cfl_kvlist *kvlist, + struct cfl_variant *target, + size_t depth) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + + if (kvlist == NULL || target == NULL) { + return CFL_FALSE; + } + + if (depth_exceeded(depth)) { + return CFL_TRUE; + } + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + + if (pair != NULL && variant_contains_variant(pair->val, target, depth + 1)) { + return CFL_TRUE; + } + } + + return CFL_FALSE; +} + +static int variant_contains_array(struct cfl_variant *variant, + struct cfl_array *target, + size_t depth) +{ + if (variant == NULL || target == NULL) { + return CFL_FALSE; + } + + if (depth_exceeded(depth)) { + return CFL_TRUE; + } + + if (variant->type == CFL_VARIANT_ARRAY) { + return array_contains_array(variant->data.as_array, target, depth + 1); + } + + if (variant->type == CFL_VARIANT_KVLIST) { + return kvlist_contains_array(variant->data.as_kvlist, target, depth + 1); + } + + return CFL_FALSE; +} + +static int variant_contains_kvlist(struct cfl_variant *variant, + struct cfl_kvlist *target, + size_t depth) +{ + if (variant == NULL || target == NULL) { + return CFL_FALSE; + } + + if (depth_exceeded(depth)) { + return CFL_TRUE; + } + + if (variant->type == CFL_VARIANT_ARRAY) { + return array_contains_kvlist(variant->data.as_array, target, depth + 1); + } + + if (variant->type == CFL_VARIANT_KVLIST) { + return kvlist_contains_kvlist(variant->data.as_kvlist, target, depth + 1); + } + + return CFL_FALSE; +} + +static int variant_contains_variant(struct cfl_variant *variant, + struct cfl_variant *target, + size_t depth) +{ + if (variant == NULL || target == NULL) { + return CFL_FALSE; + } + + if (depth_exceeded(depth)) { + return CFL_TRUE; + } + + if (variant == target) { + return CFL_TRUE; + } + + if (variant->type == CFL_VARIANT_ARRAY) { + return array_contains_variant(variant->data.as_array, target, depth + 1); + } + + if (variant->type == CFL_VARIANT_KVLIST) { + return kvlist_contains_variant(variant->data.as_kvlist, target, depth + 1); + } + + return CFL_FALSE; +} + +int cfl_container_array_contains_array(struct cfl_array *array, + struct cfl_array *target) +{ + return array_contains_array(array, target, 0); +} + +int cfl_container_array_contains_kvlist(struct cfl_array *array, + struct cfl_kvlist *target) +{ + return array_contains_kvlist(array, target, 0); +} + +int cfl_container_array_contains_variant(struct cfl_array *array, + struct cfl_variant *target) +{ + return array_contains_variant(array, target, 0); +} + +int cfl_container_kvlist_contains_array(struct cfl_kvlist *kvlist, + struct cfl_array *target) +{ + return kvlist_contains_array(kvlist, target, 0); +} + +int cfl_container_kvlist_contains_kvlist(struct cfl_kvlist *kvlist, + struct cfl_kvlist *target) +{ + return kvlist_contains_kvlist(kvlist, target, 0); +} + +int cfl_container_kvlist_contains_variant(struct cfl_kvlist *kvlist, + struct cfl_variant *target) +{ + return kvlist_contains_variant(kvlist, target, 0); +} + +int cfl_container_variant_contains_array(struct cfl_variant *variant, + struct cfl_array *target) +{ + return variant_contains_array(variant, target, 0); +} + +int cfl_container_variant_contains_kvlist(struct cfl_variant *variant, + struct cfl_kvlist *target) +{ + return variant_contains_kvlist(variant, target, 0); +} + +int cfl_container_variant_contains_variant(struct cfl_variant *variant, + struct cfl_variant *target) +{ + return variant_contains_variant(variant, target, 0); +} diff --git a/lib/cfl/src/cfl_kv.c b/lib/cfl/src/cfl_kv.c index 23d3c4d958c..7368581443b 100644 --- a/lib/cfl/src/cfl_kv.c +++ b/lib/cfl/src/cfl_kv.c @@ -21,9 +21,14 @@ #include #include +#include void cfl_kv_init(struct cfl_list *list) { + if (list == NULL) { + return; + } + cfl_list_init(list); } @@ -33,6 +38,14 @@ struct cfl_kv *cfl_kv_item_create_len(struct cfl_list *list, { struct cfl_kv *kv; + if (list == NULL || k_buf == NULL || k_len > INT_MAX || v_len > INT_MAX) { + return NULL; + } + + if (v_len > 0 && v_buf == NULL) { + return NULL; + } + kv = calloc(1, sizeof(struct cfl_kv)); if (kv == NULL) { @@ -41,7 +54,7 @@ struct cfl_kv *cfl_kv_item_create_len(struct cfl_list *list, return NULL; } - kv->key = cfl_sds_create_len(k_buf, k_len); + kv->key = cfl_sds_create_len(k_buf, (int) k_len); if (kv->key == NULL) { free(kv); @@ -50,7 +63,7 @@ struct cfl_kv *cfl_kv_item_create_len(struct cfl_list *list, } if (v_len > 0) { - kv->val = cfl_sds_create_len(v_buf, v_len); + kv->val = cfl_sds_create_len(v_buf, (int) v_len); if (kv->val == NULL) { cfl_sds_destroy(kv->key); @@ -68,10 +81,10 @@ struct cfl_kv *cfl_kv_item_create_len(struct cfl_list *list, struct cfl_kv *cfl_kv_item_create(struct cfl_list *list, char *k_buf, char *v_buf) { - int k_len; - int v_len; + size_t k_len; + size_t v_len; - if (k_buf == NULL) { + if (list == NULL || k_buf == NULL) { return NULL; } @@ -84,11 +97,19 @@ struct cfl_kv *cfl_kv_item_create(struct cfl_list *list, v_len = 0; } + if (k_len > INT_MAX || v_len > INT_MAX) { + return NULL; + } + return cfl_kv_item_create_len(list, k_buf, k_len, v_buf, v_len); } void cfl_kv_item_destroy(struct cfl_kv *kv) { + if (kv == NULL) { + return; + } + if (kv->key != NULL) { cfl_sds_destroy(kv->key); } @@ -108,6 +129,10 @@ void cfl_kv_release(struct cfl_list *list) struct cfl_list *tmp; struct cfl_kv *kv; + if (list == NULL) { + return; + } + cfl_list_foreach_safe(head, tmp, list) { kv = cfl_list_entry(head, struct cfl_kv, _head); @@ -118,10 +143,10 @@ void cfl_kv_release(struct cfl_list *list) const char *cfl_kv_get_key_value(const char *key, struct cfl_list *list) { struct cfl_list *head; - int len; + size_t len; struct cfl_kv *kv; - if (key == NULL) { + if (key == NULL || list == NULL) { return NULL; } diff --git a/lib/cfl/src/cfl_kvlist.c b/lib/cfl/src/cfl_kvlist.c index 4ffc5ea83a2..a13d56c4643 100644 --- a/lib/cfl/src/cfl_kvlist.c +++ b/lib/cfl/src/cfl_kvlist.c @@ -23,6 +23,67 @@ #include #include +#include + +#include + +static int print_json_string(FILE *fp, const char *str, size_t len) +{ + size_t i; + unsigned char c; + int ret; + + if (fputc('"', fp) == EOF) { + return -1; + } + + for (i = 0; i < len; i++) { + c = (unsigned char) str[i]; + + switch (c) { + case '"': + ret = fputs("\\\"", fp); + break; + case '\\': + ret = fputs("\\\\", fp); + break; + case '\b': + ret = fputs("\\b", fp); + break; + case '\f': + ret = fputs("\\f", fp); + break; + case '\n': + ret = fputs("\\n", fp); + break; + case '\r': + ret = fputs("\\r", fp); + break; + case '\t': + ret = fputs("\\t", fp); + break; + default: + if (c < 0x20) { + ret = fprintf(fp, "\\u%04x", c); + } + else { + ret = fputc(c, fp); + } + break; + } + + if (ret < 0) { + return -1; + } + } + + if (fputc('"', fp) == EOF) { + return -1; + } + + return 0; +} + struct cfl_kvlist *cfl_kvlist_create() { struct cfl_kvlist *list; @@ -43,6 +104,10 @@ void cfl_kvlist_destroy(struct cfl_kvlist *list) struct cfl_list *head; struct cfl_kvpair *pair; + if (list == NULL) { + return; + } + cfl_list_foreach_safe(head, tmp, &list->list) { pair = cfl_list_entry(head, struct cfl_kvpair, _head); @@ -68,6 +133,10 @@ int cfl_kvlist_insert_string_s(struct cfl_kvlist *list, struct cfl_variant *value_instance; int result; + if (list == NULL || key == NULL || key_size > INT_MAX) { + return -1; + } + value_instance = cfl_variant_create_from_string_s(value, value_size, referenced); if (value_instance == NULL) { return -1; @@ -91,6 +160,10 @@ int cfl_kvlist_insert_bytes_s(struct cfl_kvlist *list, struct cfl_variant *value_instance; int result; + if (list == NULL || key == NULL || key_size > INT_MAX) { + return -1; + } + value_instance = cfl_variant_create_from_bytes(value, length, referenced); if (value_instance == NULL) { return -1; @@ -112,6 +185,10 @@ int cfl_kvlist_insert_reference_s(struct cfl_kvlist *list, struct cfl_variant *value_instance; int result; + if (list == NULL || key == NULL || key_size > INT_MAX) { + return -1; + } + value_instance = cfl_variant_create_from_reference(value); if (value_instance == NULL) { @@ -135,6 +212,10 @@ int cfl_kvlist_insert_bool_s(struct cfl_kvlist *list, struct cfl_variant *value_instance; int result; + if (list == NULL || key == NULL || key_size > INT_MAX) { + return -1; + } + value_instance = cfl_variant_create_from_bool(value); if (value_instance == NULL) { @@ -158,6 +239,10 @@ int cfl_kvlist_insert_int64_s(struct cfl_kvlist *list, struct cfl_variant *value_instance; int result; + if (list == NULL || key == NULL || key_size > INT_MAX) { + return -1; + } + value_instance = cfl_variant_create_from_int64(value); if (value_instance == NULL) { @@ -181,6 +266,10 @@ int cfl_kvlist_insert_uint64_s(struct cfl_kvlist *list, struct cfl_variant *value_instance; int result; + if (list == NULL || key == NULL || key_size > INT_MAX) { + return -1; + } + value_instance = cfl_variant_create_from_uint64(value); if (value_instance == NULL) { @@ -204,6 +293,10 @@ int cfl_kvlist_insert_double_s(struct cfl_kvlist *list, struct cfl_variant *value_instance; int result; + if (list == NULL || key == NULL || key_size > INT_MAX) { + return -1; + } + value_instance = cfl_variant_create_from_double(value); if (value_instance == NULL) { @@ -227,6 +320,15 @@ int cfl_kvlist_insert_array_s(struct cfl_kvlist *list, struct cfl_variant *value_instance; int result; + if (list == NULL || key == NULL || key_size > INT_MAX || value == NULL) { + return -1; + } + + if (cfl_container_kvlist_contains_array(list, value) || + cfl_container_array_contains_kvlist(value, list)) { + return -1; + } + value_instance = cfl_variant_create_from_array(value); if (value_instance == NULL) { @@ -251,6 +353,10 @@ int cfl_kvlist_insert_new_array_s(struct cfl_kvlist *list, int result; struct cfl_array *value; + if (list == NULL || key == NULL || key_size > INT_MAX) { + return -1; + } + value = cfl_array_create(size); if (value == NULL) { @@ -258,8 +364,7 @@ int cfl_kvlist_insert_new_array_s(struct cfl_kvlist *list, } result = cfl_kvlist_insert_array_s(list, key, key_size, value); - - if (result) { + if (result == -1) { cfl_array_destroy(value); } @@ -272,6 +377,15 @@ int cfl_kvlist_insert_kvlist_s(struct cfl_kvlist *list, struct cfl_variant *value_instance; int result; + if (list == NULL || key == NULL || key_size > INT_MAX || value == NULL) { + return -1; + } + + if (cfl_container_kvlist_contains_kvlist(list, value) || + cfl_container_kvlist_contains_kvlist(value, list)) { + return -1; + } + value_instance = cfl_variant_create_from_kvlist(value); if (value_instance == NULL) { return -1; @@ -294,17 +408,38 @@ int cfl_kvlist_insert_s(struct cfl_kvlist *list, { struct cfl_kvpair *pair; - if (list == NULL || key == NULL || value == NULL) { + if (list == NULL || key == NULL || value == NULL || key_size > INT_MAX) { return -1; } + if (cfl_container_kvlist_contains_variant(list, value)) { + return -1; + } + + /* Only container-valued variants can participate in container cycles. */ + if (value->type == CFL_VARIANT_ARRAY || value->type == CFL_VARIANT_KVLIST) { + if (cfl_container_variant_contains_kvlist(value, list)) { + return -1; + } + + if (value->type == CFL_VARIANT_ARRAY && + cfl_container_kvlist_contains_array(list, value->data.as_array)) { + return -1; + } + + if (value->type == CFL_VARIANT_KVLIST && + cfl_container_kvlist_contains_kvlist(list, value->data.as_kvlist)) { + return -1; + } + } + pair = malloc(sizeof(struct cfl_kvpair)); if (pair == NULL) { cfl_report_runtime_error(); return -1; } - pair->key = cfl_sds_create_len(key, key_size); + pair->key = cfl_sds_create_len(key, (int) key_size); if (pair->key == NULL) { free(pair); @@ -322,6 +457,10 @@ struct cfl_variant *cfl_kvlist_fetch_s(struct cfl_kvlist *list, char *key, size_ struct cfl_list *head; struct cfl_kvpair *pair; + if (list == NULL || key == NULL) { + return NULL; + } + cfl_list_foreach(head, &list->list) { pair = cfl_list_entry(head, struct cfl_kvpair, _head); @@ -341,15 +480,18 @@ struct cfl_variant *cfl_kvlist_fetch_s(struct cfl_kvlist *list, char *key, size_ int cfl_kvlist_insert_string(struct cfl_kvlist *list, char *key, char *value) { - int key_len; - int val_len; + size_t key_len; + size_t val_len; - if (!key || !value) { + if (!list || !key || !value) { return -1; } key_len = strlen(key); val_len = strlen(value); + if (key_len > INT_MAX || val_len > INT_MAX) { + return -1; + } return cfl_kvlist_insert_string_s(list, key, key_len, value, val_len, CFL_FALSE); } @@ -358,78 +500,126 @@ int cfl_kvlist_insert_bytes(struct cfl_kvlist *list, char *key, char *value, size_t length, int referenced) { + if (!list || !key || (value == NULL && length > 0)) { + return -1; + } + return cfl_kvlist_insert_bytes_s(list, key, strlen(key), value, length, referenced); } int cfl_kvlist_insert_reference(struct cfl_kvlist *list, char *key, void *value) { + if (!list || !key) { + return -1; + } + return cfl_kvlist_insert_reference_s(list, key, strlen(key), value); } int cfl_kvlist_insert_bool(struct cfl_kvlist *list, char *key, int value) { + if (!list || !key) { + return -1; + } + return cfl_kvlist_insert_bool_s(list, key, strlen(key), value); } int cfl_kvlist_insert_int64(struct cfl_kvlist *list, char *key, int64_t value) { + if (!list || !key) { + return -1; + } + return cfl_kvlist_insert_int64_s(list, key, strlen(key), value); } int cfl_kvlist_insert_uint64(struct cfl_kvlist *list, char *key, uint64_t value) { + if (!list || !key) { + return -1; + } + return cfl_kvlist_insert_uint64_s(list, key, strlen(key), value); } int cfl_kvlist_insert_double(struct cfl_kvlist *list, char *key, double value) { + if (!list || !key) { + return -1; + } + return cfl_kvlist_insert_double_s(list, key, strlen(key), value); } int cfl_kvlist_insert_array(struct cfl_kvlist *list, char *key, struct cfl_array *value) { + if (!list || !key || !value) { + return -1; + } + return cfl_kvlist_insert_array_s(list, key, strlen(key), value); } int cfl_kvlist_insert_new_array(struct cfl_kvlist *list, char *key, size_t size) { + if (!list || !key) { + return -1; + } + return cfl_kvlist_insert_new_array_s(list, key, strlen(key), size); } int cfl_kvlist_insert_kvlist(struct cfl_kvlist *list, char *key, struct cfl_kvlist *value) { + if (!list || !key || !value) { + return -1; + } + return cfl_kvlist_insert_kvlist_s(list, key, strlen(key), value); } int cfl_kvlist_insert(struct cfl_kvlist *list, char *key, struct cfl_variant *value) { + if (!list || !key || !value) { + return -1; + } + return cfl_kvlist_insert_s(list, key, strlen(key), value); } struct cfl_variant *cfl_kvlist_fetch(struct cfl_kvlist *list, char *key) { + if (!list || !key) { + return NULL; + } + return cfl_kvlist_fetch_s(list, key, strlen(key)); } int cfl_kvlist_count(struct cfl_kvlist *list) { + if (list == NULL) { + return 0; + } + return cfl_list_size(&list->list); } int cfl_kvlist_print(FILE *fp, struct cfl_kvlist *list) { - size_t size; - size_t i; - int ret = -1; + size_t key_size; + int printed; + int ret = 0; struct cfl_list *head = NULL; struct cfl_kvpair *pair = NULL; @@ -438,24 +628,44 @@ int cfl_kvlist_print(FILE *fp, struct cfl_kvlist *list) return -1; } - size = (size_t)cfl_kvlist_count(list); - i = 0; - fputs("{", fp); + printed = CFL_FALSE; + if (fputc('{', fp) == EOF) { + return -1; + } + cfl_list_foreach(head, &list->list) { pair = cfl_list_entry(head, struct cfl_kvpair, _head); if (pair == NULL || pair->key == NULL || pair->val == NULL) { continue; } - fprintf(fp, "\"%s\":", pair->key); - ret = cfl_variant_print(fp, pair->val); + if (printed) { + if (fputc(',', fp) == EOF) { + return -1; + } + } + + key_size = cfl_sds_len(pair->key); + ret = print_json_string(fp, pair->key, key_size); + if (ret < 0) { + return -1; + } + + if (fputc(':', fp) == EOF) { + return -1; + } - i++; - if (i != size) { - fputs(",", fp); + ret = cfl_variant_print(fp, pair->val); + if (ret < 0) { + return -1; } + + printed = CFL_TRUE; + } + + if (fputc('}', fp) == EOF) { + return -1; } - fputs("}", fp); return ret; } @@ -464,12 +674,25 @@ int cfl_kvlist_contains(struct cfl_kvlist *kvlist, char *name) { struct cfl_list *iterator; struct cfl_kvpair *pair; + size_t name_len; + size_t key_len; + + if (kvlist == NULL || name == NULL) { + return CFL_FALSE; + } + + name_len = strlen(name); cfl_list_foreach(iterator, &kvlist->list) { pair = cfl_list_entry(iterator, struct cfl_kvpair, _head); - if (strcasecmp(pair->key, name) == 0) { + key_len = cfl_sds_len(pair->key); + if (key_len != name_len) { + continue; + } + + if (strncasecmp(pair->key, name, name_len) == 0) { return CFL_TRUE; } } @@ -483,17 +706,33 @@ int cfl_kvlist_remove(struct cfl_kvlist *kvlist, char *name) struct cfl_list *iterator_backup; struct cfl_list *iterator; struct cfl_kvpair *pair; + size_t name_len; + size_t key_len; + int removed; + + if (kvlist == NULL || name == NULL) { + return CFL_FALSE; + } + + name_len = strlen(name); + removed = CFL_FALSE; cfl_list_foreach_safe(iterator, iterator_backup, &kvlist->list) { pair = cfl_list_entry(iterator, struct cfl_kvpair, _head); - if (strcasecmp(pair->key, name) == 0) { + key_len = cfl_sds_len(pair->key); + if (key_len != name_len) { + continue; + } + + if (strncasecmp(pair->key, name, name_len) == 0) { cfl_kvpair_destroy(pair); + removed = CFL_TRUE; } } - return CFL_TRUE; + return removed; } @@ -515,4 +754,3 @@ void cfl_kvpair_destroy(struct cfl_kvpair *pair) free(pair); } } - diff --git a/lib/cfl/src/cfl_object.c b/lib/cfl/src/cfl_object.c index 56d13ee2ea8..f840a61c4d6 100644 --- a/lib/cfl/src/cfl_object.c +++ b/lib/cfl/src/cfl_object.c @@ -18,6 +18,7 @@ */ #include "cfl/cfl.h" +#include /* CFL Object * ========== @@ -43,38 +44,207 @@ struct cfl_object *cfl_object_create() return o; } +static int reuses_current_root(struct cfl_object *o, int type, void *ptr) +{ + if (o->variant == NULL) { + return CFL_FALSE; + } + + if (type == CFL_OBJECT_KVLIST && + o->variant->type == CFL_VARIANT_KVLIST && + o->variant->data.as_kvlist == ptr) { + return CFL_TRUE; + } + + if (type == CFL_OBJECT_ARRAY && + o->variant->type == CFL_VARIANT_ARRAY && + o->variant->data.as_array == ptr) { + return CFL_TRUE; + } + + if (type == CFL_OBJECT_VARIANT && o->variant == ptr) { + return CFL_TRUE; + } + + return CFL_FALSE; +} + +static int kvlist_contains_current(struct cfl_kvlist *kvlist, + struct cfl_variant *current) +{ + if (cfl_container_kvlist_contains_variant(kvlist, current)) { + return CFL_TRUE; + } + + if (current->type == CFL_VARIANT_KVLIST && + cfl_container_kvlist_contains_kvlist(kvlist, + current->data.as_kvlist)) { + return CFL_TRUE; + } + + if (current->type == CFL_VARIANT_ARRAY && + cfl_container_kvlist_contains_array(kvlist, + current->data.as_array)) { + return CFL_TRUE; + } + + return CFL_FALSE; +} + +static int array_contains_current(struct cfl_array *array, + struct cfl_variant *current) +{ + if (cfl_container_array_contains_variant(array, current)) { + return CFL_TRUE; + } + + if (current->type == CFL_VARIANT_KVLIST && + cfl_container_array_contains_kvlist(array, + current->data.as_kvlist)) { + return CFL_TRUE; + } + + if (current->type == CFL_VARIANT_ARRAY && + cfl_container_array_contains_array(array, + current->data.as_array)) { + return CFL_TRUE; + } + + return CFL_FALSE; +} + +static int variant_contains_current(struct cfl_variant *variant, + struct cfl_variant *current) +{ + if (cfl_container_variant_contains_variant(variant, current)) { + return CFL_TRUE; + } + + if (current->type == CFL_VARIANT_KVLIST && + cfl_container_variant_contains_kvlist(variant, + current->data.as_kvlist)) { + return CFL_TRUE; + } + + if (current->type == CFL_VARIANT_ARRAY && + cfl_container_variant_contains_array(variant, + current->data.as_array)) { + return CFL_TRUE; + } + + return CFL_FALSE; +} + +static int current_contains_candidate(struct cfl_variant *current, + int type, void *ptr) +{ + struct cfl_variant *variant; + + if (type == CFL_OBJECT_KVLIST) { + return cfl_container_variant_contains_kvlist(current, ptr); + } + + if (type == CFL_OBJECT_ARRAY) { + return cfl_container_variant_contains_array(current, ptr); + } + + if (type == CFL_OBJECT_VARIANT) { + variant = ptr; + + if (cfl_container_variant_contains_variant(current, variant)) { + return CFL_TRUE; + } + + if (variant->type == CFL_VARIANT_KVLIST && + cfl_container_variant_contains_kvlist(current, + variant->data.as_kvlist)) { + return CFL_TRUE; + } + + if (variant->type == CFL_VARIANT_ARRAY && + cfl_container_variant_contains_array(current, + variant->data.as_array)) { + return CFL_TRUE; + } + } + + return CFL_FALSE; +} + +static int candidate_contains_current(struct cfl_variant *current, + int type, void *ptr) +{ + if (type == CFL_OBJECT_KVLIST) { + return kvlist_contains_current(ptr, current); + } + + if (type == CFL_OBJECT_ARRAY) { + return array_contains_current(ptr, current); + } + + if (type == CFL_OBJECT_VARIANT) { + return variant_contains_current(ptr, current); + } + + return CFL_FALSE; +} + /* * Associate a CFL data type to the object. We only support kvlist, array and variant. Note * that everything is held as a variant internally. */ int cfl_object_set(struct cfl_object *o, int type, void *ptr) { - if (!o) { + struct cfl_variant *variant; + + if (!o || !ptr) { return -1; } + if (o->variant != NULL) { + if (reuses_current_root(o, type, ptr)) { + o->type = type; + return 0; + } + + if (current_contains_candidate(o->variant, type, ptr) || + candidate_contains_current(o->variant, type, ptr)) { + return -1; + } + } + if (type == CFL_OBJECT_KVLIST) { - o->type = CFL_OBJECT_KVLIST; - o->variant = cfl_variant_create_from_kvlist(ptr); + variant = cfl_variant_create_from_kvlist(ptr); } else if (type == CFL_OBJECT_VARIANT) { - o->type = CFL_OBJECT_VARIANT; - o->variant = ptr; + variant = ptr; } else if (type == CFL_OBJECT_ARRAY) { - o->type = CFL_OBJECT_ARRAY; - o->variant = cfl_variant_create_from_array(ptr); + variant = cfl_variant_create_from_array(ptr); } else { return -1; } + if (variant == NULL) { + return -1; + } + + if (o->variant != NULL && o->variant != variant) { + cfl_variant_destroy(o->variant); + } + + o->type = type; + o->variant = variant; + return 0; } int cfl_object_print(FILE *stream, struct cfl_object *o) { - if (!o) { + int ret; + + if (stream == NULL || o == NULL) { return -1; } @@ -82,8 +252,14 @@ int cfl_object_print(FILE *stream, struct cfl_object *o) return -1; } - cfl_variant_print(stream, o->variant); - printf("\n"); + ret = cfl_variant_print(stream, o->variant); + if (ret < 0) { + return -1; + } + + if (fputc('\n', stream) == EOF) { + return -1; + } return 0; } @@ -103,4 +279,4 @@ void cfl_object_destroy(struct cfl_object *o) } free(o); -} \ No newline at end of file +} diff --git a/lib/cfl/src/cfl_sds.c b/lib/cfl/src/cfl_sds.c index ca0c01a9739..5422f69d1ec 100644 --- a/lib/cfl/src/cfl_sds.c +++ b/lib/cfl/src/cfl_sds.c @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include @@ -34,6 +36,10 @@ size_t cfl_sds_avail(cfl_sds_t s) { struct cfl_sds *h; + if (s == NULL) { + return 0; + } + h = CFL_SDS_HEADER(s); return (size_t) (h->alloc - h->len); } @@ -44,6 +50,10 @@ static cfl_sds_t sds_alloc(size_t size) cfl_sds_t s; struct cfl_sds *head; + if (size > SIZE_MAX - CFL_SDS_HEADER_SIZE - 1) { + return NULL; + } + buf = malloc(CFL_SDS_HEADER_SIZE + size + 1); if (!buf) { return NULL; @@ -61,6 +71,10 @@ static cfl_sds_t sds_alloc(size_t size) size_t cfl_sds_alloc(cfl_sds_t s) { + if (s == NULL) { + return 0; + } + return (size_t) CFL_SDS_HEADER(s)->alloc; } @@ -71,9 +85,31 @@ cfl_sds_t cfl_sds_increase(cfl_sds_t s, size_t len) cfl_sds_t out; void *tmp; + if (s == NULL) { + return NULL; + } + out = s; - new_size = (CFL_SDS_HEADER_SIZE + cfl_sds_alloc(s) + len + 1); head = CFL_SDS_HEADER(s); + + if (len == 0) { + return s; + } + + if (head->alloc > UINT64_MAX - len) { + return NULL; + } + + if (cfl_sds_alloc(s) > SIZE_MAX - len) { + return NULL; + } + + new_size = cfl_sds_alloc(s) + len; + if (new_size > SIZE_MAX - CFL_SDS_HEADER_SIZE - 1) { + return NULL; + } + new_size += CFL_SDS_HEADER_SIZE + 1; + tmp = realloc(head, new_size); if (!tmp) { return NULL; @@ -87,6 +123,10 @@ cfl_sds_t cfl_sds_increase(cfl_sds_t s, size_t len) size_t cfl_sds_len(cfl_sds_t s) { + if (s == NULL) { + return 0; + } + return (size_t) CFL_SDS_HEADER(s)->len; } @@ -95,6 +135,10 @@ cfl_sds_t cfl_sds_create_len(const char *str, int len) cfl_sds_t s; struct cfl_sds *head; + if (len < 0) { + return NULL; + } + s = sds_alloc(len); if (!s) { return NULL; @@ -119,9 +163,12 @@ cfl_sds_t cfl_sds_create(const char *str) } else { len = strlen(str); + if (len > INT_MAX) { + return NULL; + } } - return cfl_sds_create_len(str, len); + return cfl_sds_create_len(str, (int) len); } void cfl_sds_destroy(cfl_sds_t s) @@ -139,21 +186,72 @@ void cfl_sds_destroy(cfl_sds_t s) cfl_sds_t cfl_sds_cat(cfl_sds_t s, const char *str, int len) { size_t avail; + size_t append_len; + size_t source_offset; + uintptr_t buffer_addr; + uintptr_t source_addr; struct cfl_sds *head; cfl_sds_t tmp = NULL; + const char *source; + int source_in_buffer; + + if (s == NULL || str == NULL || len < 0) { + return NULL; + } + + if (len == 0) { + return s; + } + + append_len = (size_t) len; + head = CFL_SDS_HEADER(s); + if (head->len > head->alloc || head->len > SIZE_MAX - append_len - 1) { + return NULL; + } + + source = str; + source_in_buffer = 0; + source_offset = 0; + + /* + * This flat-address check lets self-appends survive realloc. If the + * source starts inside the SDS buffer, the whole source slice must also + * fit in that allocation. + */ + buffer_addr = (uintptr_t) s; + source_addr = (uintptr_t) str; + if (source_addr >= buffer_addr && + (source_addr - buffer_addr) <= head->alloc) { + source_offset = (size_t) (source_addr - buffer_addr); + + if (append_len - 1 > head->alloc - source_offset) { + return NULL; + } + + source_in_buffer = 1; + } avail = cfl_sds_avail(s); - if (avail < len) { - tmp = cfl_sds_increase(s, len); + if (avail < append_len) { + tmp = cfl_sds_increase(s, append_len - avail); if (!tmp) { return NULL; } s = tmp; } - memcpy((char *) (s + cfl_sds_len(s)), str, len); + + if (source_in_buffer) { + source = s + source_offset; + } head = CFL_SDS_HEADER(s); - head->len += len; + if (head->len > UINT64_MAX - append_len) { + return NULL; + } + + memmove((char *) (s + head->len), source, append_len); + + head->len += append_len; s[head->len] = '\0'; return s; @@ -168,14 +266,27 @@ void cfl_sds_set_len(cfl_sds_t s, size_t len) { struct cfl_sds *head; + if (s == NULL) { + return; + } + head = CFL_SDS_HEADER(s); + if (len > head->alloc) { + return; + } + head->len = len; + s[len] = '\0'; } void cfl_sds_cat_safe(cfl_sds_t *buf, const char *str, int len) { cfl_sds_t tmp; + if (buf == NULL || *buf == NULL) { + return; + } + tmp = cfl_sds_cat(*buf, str, len); if (!tmp) { return; @@ -186,51 +297,54 @@ void cfl_sds_cat_safe(cfl_sds_t *buf, const char *str, int len) cfl_sds_t cfl_sds_printf(cfl_sds_t *sds, const char *fmt, ...) { va_list ap; - int len = strlen(fmt)*2; + size_t avail; + size_t growth; + size_t base_len; int size; cfl_sds_t tmp = NULL; cfl_sds_t s; struct cfl_sds *head; - if (len < 64) len = 64; + if (sds == NULL || *sds == NULL || fmt == NULL) { + return NULL; + } s = *sds; - if (cfl_sds_avail(s)< len) { - tmp = cfl_sds_increase(s, len); - if (!tmp) { - return NULL; - } - *sds = s = tmp; + base_len = cfl_sds_len(s); + if (base_len > cfl_sds_alloc(s)) { + return NULL; } - va_start(ap, fmt); - size = vsnprintf((char *) (s + cfl_sds_len(s)), cfl_sds_avail(s), fmt, ap); - if (size < 0) { + while (1) { + avail = cfl_sds_avail(s); + va_start(ap, fmt); + size = vsnprintf((char *) (s + base_len), avail + 1, fmt, ap); va_end(ap); - return NULL; - } - va_end(ap); - if (size >= cfl_sds_avail(s)) { - tmp = cfl_sds_increase(s, size - cfl_sds_avail(s) + 1); - if (!tmp) { + if (size < 0) { return NULL; } - *sds = s = tmp; - va_start(ap, fmt); - size = vsnprintf((char *) (s + cfl_sds_len(s)), cfl_sds_avail(s), fmt, ap); - if (size > cfl_sds_avail(s)) { - va_end(ap); + if ((size_t) size <= avail) { + break; + } + + growth = (size_t) size - avail; + tmp = cfl_sds_increase(s, growth); + if (!tmp) { return NULL; } - va_end(ap); + + *sds = s = tmp; } head = CFL_SDS_HEADER(s); - head->len += size; + if (head->len > UINT64_MAX - (size_t) size) { + return NULL; + } + + head->len += (size_t) size; s[head->len] = '\0'; return s; } - diff --git a/lib/cfl/src/cfl_utils.c b/lib/cfl/src/cfl_utils.c index 5342134e7e5..ffb7aa36a4f 100644 --- a/lib/cfl/src/cfl_utils.c +++ b/lib/cfl/src/cfl_utils.c @@ -19,15 +19,28 @@ #include +#include +#include + /* Lookup char into string, return position * Based on monkey/monkey's mk_string_char_search. */ static int cfl_string_char_search(const char *string, int c, int len) { char *p; + size_t string_len; + + if (string == NULL) { + return -1; + } if (len < 0) { - len = strlen(string); + string_len = strlen(string); + if (string_len > INT_MAX) { + return -1; + } + + len = (int) string_len; } p = memchr(string, c, len); @@ -43,14 +56,20 @@ static int cfl_string_char_search(const char *string, int c, int len) */ static char *cfl_string_copy_substr(const char *string, int pos_init, int pos_end) { - unsigned int size, bytes; + size_t size; + size_t bytes; char *buffer = 0; - if (pos_init > pos_end) { + if (string == NULL || pos_init < 0 || pos_end < 0 || pos_init > pos_end) { return NULL; } - size = (unsigned int) (pos_end - pos_init) + 1; + bytes = (size_t) (pos_end - pos_init); + if (bytes > SIZE_MAX - 1) { + return NULL; + } + + size = bytes + 1; if (size <= 2) { size = 4; } @@ -61,7 +80,6 @@ static char *cfl_string_copy_substr(const char *string, int pos_init, int pos_en return NULL; } - bytes = pos_end - pos_init; memcpy(buffer, string + pos_init, bytes); buffer[bytes] = '\0'; @@ -74,7 +92,13 @@ static char *cfl_string_copy_substr(const char *string, int pos_init, int pos_en static int quoted_string_len(const char *str) { int len = 0; - char quote = *str++; /* Consume the quote character. */ + char quote; + + if (str == NULL) { + return -1; + } + + quote = *str++; /* Consume the quote character. */ while (quote != 0) { char c = *str++; @@ -98,6 +122,9 @@ static int quoted_string_len(const char *str) default: break; } + if (len == INT_MAX) { + return -1; + } len++; } @@ -117,11 +144,16 @@ static int quoted_string_len(const char *str) static int next_token(const char *str, int separator, char **out, int *out_len, int parse_quotes) { const char *token_in = str; char *token_out; + size_t token_len; int next_separator = 0; int quote = 0; /* Parser state: 0 not inside quoted string, or '"' or '\'' when inside quoted string. */ int len = 0; int i; + if (str == NULL || out == NULL || out_len == NULL) { + return -1; + } + /* Skip leading separators. */ while (*token_in == separator) { token_in++; @@ -129,7 +161,12 @@ static int next_token(const char *str, int separator, char **out, int *out_len, /* Should quotes be parsed? Or is token quoted? If not, copy until separator or the end of string. */ if (parse_quotes == CFL_FALSE || (*token_in != '"' && *token_in != '\'')) { - len = (int)strlen(token_in); + token_len = strlen(token_in); + if (token_len > INT_MAX) { + return -1; + } + + len = (int) token_len; next_separator = cfl_string_char_search(token_in, separator, len); if (next_separator > 0) { len = next_separator; @@ -186,6 +223,7 @@ static struct cfl_list *split(const char *line, int separator, int max_split, in int val_len; int len; int end; + size_t line_len; char *val; struct cfl_list *list; struct cfl_split_entry *new; @@ -201,7 +239,13 @@ static struct cfl_list *split(const char *line, int separator, int max_split, in } cfl_list_init(list); - len = strlen(line); + line_len = strlen(line); + if (line_len > INT_MAX) { + free(list); + return NULL; + } + + len = (int) line_len; while (i < len) { end = next_token(line + i, separator, &val, &val_len, quoted); if (end == -1) { @@ -236,13 +280,19 @@ static struct cfl_list *split(const char *line, int separator, int max_split, in * and last entry. */ if (count >= max_split && max_split > 0 && i < len) { - new = calloc(1, sizeof(struct cfl_split_entry)); + new = calloc(1, sizeof(struct cfl_split_entry)); if (!new) { cfl_errno(); cfl_utils_split_free(list); return NULL; } new->value = cfl_string_copy_substr(line, i, len); + if (new->value == NULL) { + cfl_errno(); + free(new); + cfl_utils_split_free(list); + return NULL; + } new->len = len - i; cfl_list_add(&new->_head, list); break; @@ -265,6 +315,10 @@ struct cfl_list *cfl_utils_split(const char *line, int separator, int max_split) void cfl_utils_split_free_entry(struct cfl_split_entry *entry) { + if (entry == NULL) { + return; + } + cfl_list_del(&entry->_head); free(entry->value); free(entry); @@ -276,6 +330,10 @@ void cfl_utils_split_free(struct cfl_list *list) struct cfl_list *head; struct cfl_split_entry *entry; + if (list == NULL) { + return; + } + cfl_list_foreach_safe(head, tmp, list) { entry = cfl_list_entry(head, struct cfl_split_entry, _head); cfl_utils_split_free_entry(entry); diff --git a/lib/cfl/src/cfl_variant.c b/lib/cfl/src/cfl_variant.c index 3e7065e3737..44b3fa2fdd1 100644 --- a/lib/cfl/src/cfl_variant.c +++ b/lib/cfl/src/cfl_variant.c @@ -23,11 +23,95 @@ #include #include -#if defined(__MINGW32__) || defined(__MINGW64__) -#define HEXDUMPFORMAT "%#x" +#include +#include +#if defined(_MSC_VER) +#include +#endif + +static int double_is_finite(double value) +{ +#if defined(_MSC_VER) + return _finite(value); #else -#define HEXDUMPFORMAT "%p" + return isfinite(value); #endif +} + +static int print_json_string(FILE *fp, const char *str, size_t len) +{ + size_t i; + size_t written; + unsigned char c; + int ret; + + if (fputc('"', fp) == EOF) { + return -1; + } + written = 1; + + if (str != NULL) { + for (i = 0; i < len; i++) { + c = (unsigned char) str[i]; + + switch (c) { + case '"': + ret = fputs("\\\"", fp); + written += 2; + break; + case '\\': + ret = fputs("\\\\", fp); + written += 2; + break; + case '\b': + ret = fputs("\\b", fp); + written += 2; + break; + case '\f': + ret = fputs("\\f", fp); + written += 2; + break; + case '\n': + ret = fputs("\\n", fp); + written += 2; + break; + case '\r': + ret = fputs("\\r", fp); + written += 2; + break; + case '\t': + ret = fputs("\\t", fp); + written += 2; + break; + default: + if (c < 0x20) { + ret = fprintf(fp, "\\u%04x", c); + written += 6; + } + else { + ret = fputc(c, fp); + written++; + } + break; + } + + if (ret < 0) { + return -1; + } + } + } + + if (fputc('"', fp) == EOF) { + return -1; + } + written++; + + if (written > INT_MAX) { + return INT_MAX; + } + + return (int) written; +} int cfl_variant_print(FILE *fp, struct cfl_variant *val) { @@ -41,7 +125,10 @@ int cfl_variant_print(FILE *fp, struct cfl_variant *val) switch (val->type) { case CFL_VARIANT_STRING: - ret = fprintf(fp, "\"%s\"", val->data.as_string); + if (val->data.as_string == NULL && val->size > 0) { + return -1; + } + ret = print_json_string(fp, val->data.as_string, val->size); break; case CFL_VARIANT_BOOL: if (val->data.as_bool) { @@ -58,20 +145,33 @@ int cfl_variant_print(FILE *fp, struct cfl_variant *val) ret = fprintf(fp, "%" PRIu64, val->data.as_uint64); break; case CFL_VARIANT_DOUBLE: - ret = fprintf(fp, "%lf", val->data.as_double); + if (!double_is_finite(val->data.as_double)) { + ret = fputs("null", fp); + } + else { + ret = fprintf(fp, "%lf", val->data.as_double); + } break; case CFL_VARIANT_NULL: ret = fprintf(fp, "null"); break; case CFL_VARIANT_BYTES: - size = cfl_sds_len(val->data.as_bytes); - for (i=0; idata.as_bytes == NULL && val->size > 0) { + return -1; + } + + size = val->size; + ret = 0; + for (i = 0; i < size; i++) { ret = fprintf(fp, "%02x", (unsigned char)val->data.as_bytes[i]); + if (ret < 0) { + return -1; + } } break; case CFL_VARIANT_REFERENCE: - ret = fprintf(fp, HEXDUMPFORMAT, val->data.as_reference); + ret = fputs("null", fp); break; case CFL_VARIANT_ARRAY: ret = cfl_array_print(fp, val->data.as_array); @@ -91,17 +191,26 @@ struct cfl_variant *cfl_variant_create_from_string_s(char *value, size_t value_s { struct cfl_variant *instance; + if (value == NULL && value_size > 0) { + return NULL; + } + instance = cfl_variant_create(); if (!instance) { return NULL; } - instance->referenced = referenced; + instance->referenced = referenced ? CFL_TRUE : CFL_FALSE; if (referenced) { instance->data.as_string = value; } else { - instance->data.as_string = cfl_sds_create_len(value, value_size); + if (value_size > INT_MAX) { + free(instance); + return NULL; + } + + instance->data.as_string = cfl_sds_create_len(value, (int) value_size); if (instance->data.as_string == NULL) { free(instance); return NULL; @@ -116,6 +225,10 @@ struct cfl_variant *cfl_variant_create_from_string_s(char *value, size_t value_s struct cfl_variant *cfl_variant_create_from_string(char *value) { + if (value == NULL) { + return NULL; + } + return cfl_variant_create_from_string_s(value, strlen(value), CFL_FALSE); } @@ -123,17 +236,26 @@ struct cfl_variant *cfl_variant_create_from_bytes(char *value, size_t length, in { struct cfl_variant *instance; + if (value == NULL && length > 0) { + return NULL; + } + instance = cfl_variant_create(); if (!instance){ return NULL; } - instance->referenced = referenced; + instance->referenced = referenced ? CFL_TRUE : CFL_FALSE; if (referenced) { instance->data.as_bytes = value; } else { - instance->data.as_bytes = cfl_sds_create_len(value, length); + if (length > INT_MAX) { + free(instance); + return NULL; + } + + instance->data.as_bytes = cfl_sds_create_len(value, (int) length); if (instance->data.as_bytes == NULL) { free(instance); return NULL; @@ -286,10 +408,18 @@ void cfl_variant_destroy(struct cfl_variant *instance) void cfl_variant_size_set(struct cfl_variant *var, size_t size) { + if (var == NULL) { + return; + } + var->size = size; } size_t cfl_variant_size_get(struct cfl_variant *var) { + if (var == NULL) { + return 0; + } + return var->size; } diff --git a/lib/cfl/tests/CMakeLists.txt b/lib/cfl/tests/CMakeLists.txt index 4dba27058fb..6bd3cc6f7bb 100644 --- a/lib/cfl/tests/CMakeLists.txt +++ b/lib/cfl/tests/CMakeLists.txt @@ -1,6 +1,9 @@ include_directories(lib/acutest) set(UNIT_TESTS_FILES + atomic_operations.c + checksum.c + headers.c kv.c kvlist.c array.c @@ -13,6 +16,32 @@ set(UNIT_TESTS_FILES utils.c ) +if(NOT CFL_SYSTEM_WINDOWS) + find_package(Threads REQUIRED) +endif() + +set(PUBLIC_HEADERS + cfl.h + cfl_array.h + cfl_atomic.h + cfl_checksum.h + cfl_compat.h + cfl_container.h + cfl_found.h + cfl_hash.h + cfl_info.h + cfl_kv.h + cfl_kvlist.h + cfl_list.h + cfl_log.h + cfl_object.h + cfl_sds.h + cfl_time.h + cfl_utils.h + cfl_variant.h + cfl_version.h + ) + configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/cfl_tests_internal.h.in" "${CMAKE_CURRENT_SOURCE_DIR}/cfl_tests_internal.h" @@ -28,9 +57,33 @@ foreach(source_file ${UNIT_TESTS_FILES}) ) target_link_libraries(${source_file_we} cfl-static) + if(source_file STREQUAL "atomic_operations.c" AND NOT CFL_SYSTEM_WINDOWS) + target_link_libraries(${source_file_we} Threads::Threads) + endif() + if (CFL_SANITIZE_ADDRESS) add_sanitizers(${source_file_we}) endif() add_test(${source_file_we} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${source_file_we}) endforeach() + +foreach(public_header ${PUBLIC_HEADERS}) + string(REPLACE "." "_" public_header_target ${public_header}) + set(public_header_source "${CMAKE_CURRENT_BINARY_DIR}/header_${public_header_target}.c") + file(WRITE "${public_header_source}" "#include \nint main(void) { return 0; }\n") + + add_executable( + cfl-test-header-${public_header_target} + ${public_header_source} + ) + + target_link_libraries(cfl-test-header-${public_header_target} cfl-static) + + if (CFL_SANITIZE_ADDRESS) + add_sanitizers(cfl-test-header-${public_header_target}) + endif() + + add_test(cfl-test-header-${public_header_target} + ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/cfl-test-header-${public_header_target}) +endforeach() diff --git a/lib/cfl/tests/array.c b/lib/cfl/tests/array.c index 90332bbfa22..31d50b9b22c 100644 --- a/lib/cfl/tests/array.c +++ b/lib/cfl/tests/array.c @@ -317,6 +317,94 @@ static void append_kvlist() cfl_array_destroy(arr); } +static void append_array_rejects_cycles() +{ + int ret; + struct cfl_array *arr; + struct cfl_array *child; + + arr = cfl_array_create(2); + TEST_CHECK(arr != NULL); + + ret = cfl_array_append_array(arr, arr); + TEST_CHECK(ret == -1); + + child = cfl_array_create(1); + TEST_CHECK(child != NULL); + + ret = cfl_array_append_array(arr, child); + TEST_CHECK(ret == 0); + + ret = cfl_array_append_array(arr, child); + TEST_CHECK(ret == -1); + + ret = cfl_array_append_array(child, arr); + TEST_CHECK(ret == -1); + + cfl_array_destroy(arr); +} + +static void append_variant_rejects_cycles() +{ + int ret; + struct cfl_array *arr; + struct cfl_variant *variant; + + arr = cfl_array_create(1); + TEST_CHECK(arr != NULL); + + variant = cfl_variant_create_from_array(arr); + TEST_CHECK(variant != NULL); + + ret = cfl_array_append(arr, variant); + TEST_CHECK(ret == -1); + + cfl_variant_destroy(variant); +} + +static void append_rejects_duplicate_variant() +{ + int ret; + struct cfl_array *arr; + struct cfl_variant *variant; + + arr = cfl_array_create(2); + TEST_CHECK(arr != NULL); + + variant = cfl_variant_create_from_string("value"); + TEST_CHECK(variant != NULL); + + ret = cfl_array_append(arr, variant); + TEST_CHECK(ret == 0); + + ret = cfl_array_append(arr, variant); + TEST_CHECK(ret == -1); + TEST_CHECK(cfl_array_size(arr) == 1); + + cfl_array_destroy(arr); +} + +static void append_kvlist_rejects_cycles() +{ + int ret; + struct cfl_array *arr; + struct cfl_kvlist *kvlist; + + arr = cfl_array_create(1); + TEST_CHECK(arr != NULL); + + kvlist = cfl_kvlist_create(); + TEST_CHECK(kvlist != NULL); + + ret = cfl_array_append_kvlist(arr, kvlist); + TEST_CHECK(ret == 0); + + ret = cfl_kvlist_insert_array(kvlist, "cycle", arr); + TEST_CHECK(ret == -1); + + cfl_array_destroy(arr); +} + static void remove_by_index() { int ret; @@ -363,6 +451,70 @@ static void remove_by_reference() cfl_array_destroy(arr); } +static void print_write_error() +{ +#ifdef __linux__ + int ret; + FILE *fp; + struct cfl_array *arr; + + arr = cfl_array_create(0); + TEST_CHECK(arr != NULL); + + fp = fopen("/dev/full", "w"); + if (fp == NULL) { + cfl_array_destroy(arr); + return; + } + + setvbuf(fp, NULL, _IONBF, 0); + + ret = cfl_array_print(fp, arr); + TEST_CHECK(ret == -1); + + fclose(fp); + cfl_array_destroy(arr); +#endif +} + +static void null_inputs() +{ + int ret; + struct cfl_variant *var; + + TEST_CHECK(cfl_array_size(NULL) == 0); + + var = cfl_array_fetch_by_index(NULL, 0); + TEST_CHECK(var == NULL); + + ret = cfl_array_resizable(NULL, CFL_TRUE); + TEST_CHECK(ret == -1); + + ret = cfl_array_append(NULL, NULL); + TEST_CHECK(ret == -1); + + ret = cfl_array_append_string(NULL, "value"); + TEST_CHECK(ret < 0); + + ret = cfl_array_append_string(NULL, NULL); + TEST_CHECK(ret == -1); + + ret = cfl_array_append_bytes(NULL, NULL, 1, CFL_TRUE); + TEST_CHECK(ret == -1); + + ret = cfl_array_append_array(NULL, NULL); + TEST_CHECK(ret == -1); + + ret = cfl_array_append_kvlist(NULL, NULL); + TEST_CHECK(ret == -1); + + ret = cfl_array_remove_by_index(NULL, 0); + TEST_CHECK(ret == -1); + + ret = cfl_array_remove_by_reference(NULL, NULL); + TEST_CHECK(ret == -1); +} + TEST_LIST = { {"create", create}, {"resizable", resizable}, @@ -380,7 +532,13 @@ TEST_LIST = { {"append_array", append_array}, {"append_new_array", append_new_array}, {"append_kvlist", append_kvlist}, + {"append_array_rejects_cycles", append_array_rejects_cycles}, + {"append_variant_rejects_cycles", append_variant_rejects_cycles}, + {"append_rejects_duplicate_variant", append_rejects_duplicate_variant}, + {"append_kvlist_rejects_cycles", append_kvlist_rejects_cycles}, {"remove_by_index", remove_by_index}, {"remove_by_reference", remove_by_reference}, + {"print_write_error", print_write_error}, + {"null_inputs", null_inputs}, { 0 } }; diff --git a/lib/cfl/tests/atomic_operations.c b/lib/cfl/tests/atomic_operations.c new file mode 100644 index 00000000000..add799ec663 --- /dev/null +++ b/lib/cfl/tests/atomic_operations.c @@ -0,0 +1,177 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2022 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#if defined (_WIN32) || defined (_WIN64) +#include +#else +#include +#endif + +#include "cfl_tests_internal.h" + +#define THREAD_COUNT 100 +#define CYCLE_COUNT 10000 +#define EXPECTED_VALUE (THREAD_COUNT * CYCLE_COUNT) + +static uint64_t global_counter; + +static void test_atomic_initialize() +{ + TEST_CHECK(cfl_atomic_initialize() == 0); + TEST_CHECK(cfl_atomic_initialize() == 0); + TEST_CHECK(cfl_init() == 0); +} + +static void test_atomic_basic_operations() +{ + TEST_CHECK(cfl_init() == 0); + + cfl_atomic_store(&global_counter, 10); + TEST_CHECK(cfl_atomic_load(&global_counter) == 10); + + TEST_CHECK(cfl_atomic_compare_exchange(&global_counter, 5, 20) == 0); + TEST_CHECK(cfl_atomic_load(&global_counter) == 10); + + TEST_CHECK(cfl_atomic_compare_exchange(&global_counter, 10, 20) == 1); + TEST_CHECK(cfl_atomic_load(&global_counter) == 20); +} + +static void test_atomic_full_width_values() +{ + uint64_t value; + uint64_t high_bit; + + high_bit = UINT64_C(1) << 63; + + TEST_CHECK(cfl_atomic_initialize() == 0); + + value = 0; + cfl_atomic_store(&value, UINT64_MAX); + TEST_CHECK(cfl_atomic_load(&value) == UINT64_MAX); + + TEST_CHECK(cfl_atomic_compare_exchange(&value, high_bit, 1) == 0); + TEST_CHECK(cfl_atomic_load(&value) == UINT64_MAX); + + TEST_CHECK(cfl_atomic_compare_exchange(&value, UINT64_MAX, high_bit) == 1); + TEST_CHECK(cfl_atomic_load(&value) == high_bit); + + TEST_CHECK(cfl_atomic_compare_exchange(&value, high_bit, 0) == 1); + TEST_CHECK(cfl_atomic_load(&value) == 0); +} + +static void add_through_compare_exchange(uint64_t val) +{ + uint64_t old; + uint64_t new; + int result; + + do { + old = cfl_atomic_load(&global_counter); + new = old + val; + + result = cfl_atomic_compare_exchange(&global_counter, old, new); + } + while (result == 0); +} + +#if defined (_WIN32) || defined (_WIN64) +static DWORD WINAPI worker_thread_add_through_compare_exchange(LPVOID ptr) +#else +static void *worker_thread_add_through_compare_exchange(void *ptr) +#endif +{ + int local_counter; + + (void) ptr; + + for (local_counter = 0; local_counter < CYCLE_COUNT; local_counter++) { + add_through_compare_exchange(1); + } + +#if defined (_WIN32) || defined (_WIN64) + return 0; +#else + return NULL; +#endif +} + +#if defined (_WIN32) || defined (_WIN64) + +static void test_atomic_operations() +{ + HANDLE threads[THREAD_COUNT]; + DWORD thread_ids[THREAD_COUNT]; + int thread_index; + DWORD result; + + TEST_CHECK(cfl_init() == 0); + + cfl_atomic_store(&global_counter, 0); + + for (thread_index = 0; thread_index < THREAD_COUNT; thread_index++) { + threads[thread_index] = CreateThread(NULL, 0, + worker_thread_add_through_compare_exchange, + NULL, 0, &thread_ids[thread_index]); + } + + for (thread_index = 0; thread_index < THREAD_COUNT; thread_index++) { + result = WaitForSingleObject(threads[thread_index], INFINITE); + TEST_CHECK(result == WAIT_OBJECT_0); + CloseHandle(threads[thread_index]); + } + + TEST_CHECK(cfl_atomic_load(&global_counter) == EXPECTED_VALUE); +} + +#else + +static void test_atomic_operations() +{ + pthread_t threads[THREAD_COUNT]; + int thread_index; + int result; + + TEST_CHECK(cfl_init() == 0); + + cfl_atomic_store(&global_counter, 0); + + for (thread_index = 0; thread_index < THREAD_COUNT; thread_index++) { + result = pthread_create(&threads[thread_index], NULL, + worker_thread_add_through_compare_exchange, NULL); + TEST_CHECK(result == 0); + } + + for (thread_index = 0; thread_index < THREAD_COUNT; thread_index++) { + result = pthread_join(threads[thread_index], NULL); + TEST_CHECK(result == 0); + } + + TEST_CHECK(cfl_atomic_load(&global_counter) == EXPECTED_VALUE); +} +#endif + +TEST_LIST = { + { "atomic_initialize", test_atomic_initialize }, + { "atomic_basic_operations", test_atomic_basic_operations }, + { "atomic_full_width_values", test_atomic_full_width_values }, + { "atomic_operations", test_atomic_operations }, + { 0 } +}; diff --git a/lib/cfl/tests/checksum.c b/lib/cfl/tests/checksum.c new file mode 100644 index 00000000000..32dc3e5d04d --- /dev/null +++ b/lib/cfl/tests/checksum.c @@ -0,0 +1,38 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2022 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "cfl_tests_internal.h" + +static void crc32c_null_input() +{ + uint32_t crc; + + crc = cfl_checksum_crc32c(NULL, 0); + TEST_CHECK(crc == 0); + + crc = cfl_checksum_crc32c(NULL, 1); + TEST_CHECK(crc == 0); +} + +TEST_LIST = { + {"crc32c_null_input", crc32c_null_input}, + { 0 } +}; diff --git a/lib/cfl/tests/headers.c b/lib/cfl/tests/headers.c new file mode 100644 index 00000000000..5b5f890ea91 --- /dev/null +++ b/lib/cfl/tests/headers.c @@ -0,0 +1,49 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CFL + * === + * Copyright (C) 2022 The CFL Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cfl_tests_internal.h" + +static void public_headers_compile() +{ + TEST_CHECK(cfl_found() == 0); +} + +TEST_LIST = { + {"public_headers_compile", public_headers_compile}, + { 0 } +}; diff --git a/lib/cfl/tests/kv.c b/lib/cfl/tests/kv.c index f49ade9d63c..531925b38dc 100644 --- a/lib/cfl/tests/kv.c +++ b/lib/cfl/tests/kv.c @@ -76,7 +76,30 @@ static void regular_operation() cfl_kv_release(&entry_list); } +static void null_inputs() +{ + struct cfl_kv *entry; + + cfl_kv_init(NULL); + + entry = cfl_kv_item_create(NULL, "key", "value"); + TEST_CHECK(entry == NULL); + + entry = cfl_kv_item_create_len(NULL, "key", 3, "value", 5); + TEST_CHECK(entry == NULL); + + entry = cfl_kv_item_create_len(NULL, "key", 3, NULL, 1); + TEST_CHECK(entry == NULL); + + TEST_CHECK(cfl_kv_get_key_value(NULL, NULL) == NULL); + TEST_CHECK(cfl_kv_get_key_value("key", NULL) == NULL); + + cfl_kv_item_destroy(NULL); + cfl_kv_release(NULL); +} + TEST_LIST = { {"regular_operation", regular_operation}, + {"null_inputs", null_inputs}, { 0 } }; diff --git a/lib/cfl/tests/kvlist.c b/lib/cfl/tests/kvlist.c index 87a55932361..20f1d0e407f 100644 --- a/lib/cfl/tests/kvlist.c +++ b/lib/cfl/tests/kvlist.c @@ -24,6 +24,30 @@ #include "cfl_tests_internal.h" +static int compare(FILE *fp, char *expect) +{ + size_t len; + size_t ret_fp; + char buf[256] = {0}; + + len = strlen(expect); + + if (fseek(fp, 0, SEEK_SET) != 0) { + return -1; + } + + ret_fp = fread(&buf[0], 1, sizeof(buf) - 1, fp); + if (ret_fp == 0 && ferror(fp)) { + return -1; + } + + if (strlen(buf) != len) { + return -1; + } + + return strncmp(expect, &buf[0], len); +} + static void create_destroy() { struct cfl_kvlist *list = NULL; @@ -1166,6 +1190,237 @@ static void test_basics() cfl_kvlist_destroy(list); } +static void null_inputs() +{ + int ret; + struct cfl_kvlist *list; + struct cfl_variant *variant; + + cfl_kvlist_destroy(NULL); + + ret = cfl_kvlist_count(NULL); + TEST_CHECK(ret == 0); + + variant = cfl_kvlist_fetch(NULL, "key"); + TEST_CHECK(variant == NULL); + + variant = cfl_kvlist_fetch_s(NULL, "key", 3); + TEST_CHECK(variant == NULL); + + ret = cfl_kvlist_contains(NULL, "key"); + TEST_CHECK(ret == CFL_FALSE); + + ret = cfl_kvlist_remove(NULL, "key"); + TEST_CHECK(ret == CFL_FALSE); + + ret = cfl_kvlist_insert_string(NULL, "key", "value"); + TEST_CHECK(ret == -1); + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + ret = cfl_kvlist_insert_string(list, NULL, "value"); + TEST_CHECK(ret == -1); + + ret = cfl_kvlist_insert_bytes(list, "key", NULL, 1, CFL_TRUE); + TEST_CHECK(ret == -1); + + ret = cfl_kvlist_insert_array(list, "key", NULL); + TEST_CHECK(ret == -1); + + ret = cfl_kvlist_insert_kvlist(list, "key", NULL); + TEST_CHECK(ret == -1); + + ret = cfl_kvlist_insert(list, "key", NULL); + TEST_CHECK(ret == -1); + + variant = cfl_kvlist_fetch(list, NULL); + TEST_CHECK(variant == NULL); + + cfl_kvpair_destroy(NULL); + cfl_kvlist_destroy(list); +} + +static void print_escaped_keys() +{ + int ret; + FILE *fp; + struct cfl_kvlist *list; + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + ret = cfl_kvlist_insert_string(list, "a\"b\n", "v\n"); + TEST_CHECK(ret == 0); + + fp = tmpfile(); + TEST_CHECK(fp != NULL); + + ret = cfl_kvlist_print(fp, list); + TEST_CHECK(ret > 0); + + ret = compare(fp, "{\"a\\\"b\\n\":\"v\\n\"}"); + TEST_CHECK(ret == 0); + + fclose(fp); + cfl_kvlist_destroy(list); +} + +static void embedded_nul_keys_do_not_match_short_name() +{ + int ret; + char key[] = {'a', 'd', 'm', 'i', 'n', '\0', 'x'}; + struct cfl_kvlist *list; + struct cfl_variant *variant; + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + ret = cfl_kvlist_insert_string(list, "admin", "plain"); + TEST_CHECK(ret == 0); + + ret = cfl_kvlist_insert_string_s(list, key, sizeof(key), + "hidden", 6, CFL_FALSE); + TEST_CHECK(ret == 0); + + ret = cfl_kvlist_contains(list, "admin"); + TEST_CHECK(ret == CFL_TRUE); + + ret = cfl_kvlist_remove(list, "admin"); + TEST_CHECK(ret == CFL_TRUE); + + ret = cfl_kvlist_count(list); + TEST_CHECK(ret == 1); + + variant = cfl_kvlist_fetch_s(list, key, sizeof(key)); + TEST_CHECK(variant != NULL); + + ret = cfl_kvlist_contains(list, "admin"); + TEST_CHECK(ret == CFL_FALSE); + + ret = cfl_kvlist_remove(list, "admin"); + TEST_CHECK(ret == CFL_FALSE); + + ret = cfl_kvlist_count(list); + TEST_CHECK(ret == 1); + + cfl_kvlist_destroy(list); +} + +static void print_write_error() +{ +#ifdef __linux__ + int ret; + FILE *fp; + struct cfl_kvlist *list; + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + fp = fopen("/dev/full", "w"); + if (fp == NULL) { + cfl_kvlist_destroy(list); + return; + } + + setvbuf(fp, NULL, _IONBF, 0); + + ret = cfl_kvlist_print(fp, list); + TEST_CHECK(ret == -1); + + fclose(fp); + cfl_kvlist_destroy(list); +#endif +} + +static void reject_kvlist_cycles() +{ + int ret; + struct cfl_kvlist *list; + struct cfl_kvlist *child; + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + ret = cfl_kvlist_insert_kvlist(list, "self", list); + TEST_CHECK(ret == -1); + + child = cfl_kvlist_create(); + TEST_CHECK(child != NULL); + + ret = cfl_kvlist_insert_kvlist(list, "child", child); + TEST_CHECK(ret == 0); + + ret = cfl_kvlist_insert_kvlist(list, "child-again", child); + TEST_CHECK(ret == -1); + + ret = cfl_kvlist_insert_kvlist(child, "parent", list); + TEST_CHECK(ret == -1); + + cfl_kvlist_destroy(list); +} + +static void reject_variant_cycles() +{ + int ret; + struct cfl_kvlist *list; + struct cfl_variant *variant; + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + variant = cfl_variant_create_from_kvlist(list); + TEST_CHECK(variant != NULL); + + ret = cfl_kvlist_insert(list, "self", variant); + TEST_CHECK(ret == -1); + + cfl_variant_destroy(variant); +} + +static void reject_duplicate_variant() +{ + int ret; + struct cfl_kvlist *list; + struct cfl_variant *variant; + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + variant = cfl_variant_create_from_string("value"); + TEST_CHECK(variant != NULL); + + ret = cfl_kvlist_insert(list, "one", variant); + TEST_CHECK(ret == 0); + + ret = cfl_kvlist_insert(list, "two", variant); + TEST_CHECK(ret == -1); + TEST_CHECK(cfl_kvlist_count(list) == 1); + + cfl_kvlist_destroy(list); +} + +static void reject_array_cycles() +{ + int ret; + struct cfl_array *array; + struct cfl_kvlist *list; + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + array = cfl_array_create(1); + TEST_CHECK(array != NULL); + + ret = cfl_kvlist_insert_array(list, "array", array); + TEST_CHECK(ret == 0); + + ret = cfl_array_append_kvlist(array, list); + TEST_CHECK(ret == -1); + + cfl_kvlist_destroy(list); +} + TEST_LIST = { {"create_destroy", create_destroy}, {"count", count}, @@ -1193,5 +1448,13 @@ TEST_LIST = { {"insert_empty_array_s", insert_empty_array_s}, {"insert_empty_kvlist_s", insert_empty_kvlist_s}, {"basics", test_basics}, + {"null_inputs", null_inputs}, + {"print_escaped_keys", print_escaped_keys}, + {"embedded_nul_keys_do_not_match_short_name", embedded_nul_keys_do_not_match_short_name}, + {"print_write_error", print_write_error}, + {"reject_kvlist_cycles", reject_kvlist_cycles}, + {"reject_variant_cycles", reject_variant_cycles}, + {"reject_duplicate_variant", reject_duplicate_variant}, + {"reject_array_cycles", reject_array_cycles}, { 0 } }; diff --git a/lib/cfl/tests/object.c b/lib/cfl/tests/object.c index 4c372e3e1df..7b5f2b557a5 100644 --- a/lib/cfl/tests/object.c +++ b/lib/cfl/tests/object.c @@ -21,6 +21,30 @@ #include #include "cfl_tests_internal.h" +static int compare(FILE *fp, char *expect) +{ + size_t len; + size_t ret_fp; + char buf[128] = {0}; + + len = strlen(expect); + + if (fseek(fp, 0, SEEK_SET) != 0) { + return -1; + } + + ret_fp = fread(&buf[0], 1, sizeof(buf) - 1, fp); + if (ret_fp == 0 && ferror(fp)) { + return -1; + } + + if (strlen(buf) != len) { + return -1; + } + + return strncmp(expect, &buf[0], len); +} + static void test_basics() { int ret; @@ -52,7 +76,242 @@ static void test_basics() cfl_object_destroy(object); } +static void test_replace_and_print() +{ + int ret; + FILE *fp; + struct cfl_object *object; + struct cfl_variant *variant; + + object = cfl_object_create(); + TEST_CHECK(object != NULL); + + variant = cfl_variant_create_from_string("first"); + TEST_CHECK(variant != NULL); + + ret = cfl_object_set(object, CFL_OBJECT_VARIANT, variant); + TEST_CHECK(ret == 0); + + variant = cfl_variant_create_from_string("second"); + TEST_CHECK(variant != NULL); + + ret = cfl_object_set(object, CFL_OBJECT_VARIANT, variant); + TEST_CHECK(ret == 0); + + ret = cfl_object_set(object, CFL_OBJECT_VARIANT, NULL); + TEST_CHECK(ret == -1); + + fp = tmpfile(); + TEST_CHECK(fp != NULL); + + ret = cfl_object_print(fp, object); + TEST_CHECK(ret == 0); + + ret = compare(fp, "\"second\"\n"); + TEST_CHECK(ret == 0); + + fclose(fp); + cfl_object_destroy(object); +} + +static void test_reuse_owned_kvlist() +{ + int ret; + FILE *fp; + struct cfl_object *object; + struct cfl_kvlist *list; + + object = cfl_object_create(); + TEST_CHECK(object != NULL); + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + ret = cfl_kvlist_insert_string(list, "key", "value"); + TEST_CHECK(ret == 0); + + ret = cfl_object_set(object, CFL_OBJECT_KVLIST, list); + TEST_CHECK(ret == 0); + + list = object->variant->data.as_kvlist; + + ret = cfl_object_set(object, CFL_OBJECT_KVLIST, list); + TEST_CHECK(ret == 0); + + fp = tmpfile(); + TEST_CHECK(fp != NULL); + + ret = cfl_object_print(fp, object); + TEST_CHECK(ret == 0); + + ret = compare(fp, "{\"key\":\"value\"}\n"); + TEST_CHECK(ret == 0); + + fclose(fp); + cfl_object_destroy(object); +} + +static void test_reuse_owned_array() +{ + int ret; + FILE *fp; + struct cfl_object *object; + struct cfl_array *array; + + object = cfl_object_create(); + TEST_CHECK(object != NULL); + + array = cfl_array_create(1); + TEST_CHECK(array != NULL); + + ret = cfl_array_append_string(array, "value"); + TEST_CHECK(ret == 0); + + ret = cfl_object_set(object, CFL_OBJECT_ARRAY, array); + TEST_CHECK(ret == 0); + + array = object->variant->data.as_array; + + ret = cfl_object_set(object, CFL_OBJECT_ARRAY, array); + TEST_CHECK(ret == 0); + + fp = tmpfile(); + TEST_CHECK(fp != NULL); + + ret = cfl_object_print(fp, object); + TEST_CHECK(ret == 0); + + ret = compare(fp, "[\"value\"]\n"); + TEST_CHECK(ret == 0); + + fclose(fp); + cfl_object_destroy(object); +} + +static void test_reject_nested_kvlist_reuse() +{ + int ret; + FILE *fp; + struct cfl_object *object; + struct cfl_kvlist *outer; + struct cfl_kvlist *inner; + + object = cfl_object_create(); + TEST_CHECK(object != NULL); + + outer = cfl_kvlist_create(); + TEST_CHECK(outer != NULL); + + inner = cfl_kvlist_create(); + TEST_CHECK(inner != NULL); + + ret = cfl_kvlist_insert_kvlist(outer, "child", inner); + TEST_CHECK(ret == 0); + + ret = cfl_object_set(object, CFL_OBJECT_KVLIST, outer); + TEST_CHECK(ret == 0); + + ret = cfl_object_set(object, CFL_OBJECT_KVLIST, inner); + TEST_CHECK(ret == -1); + + fp = tmpfile(); + TEST_CHECK(fp != NULL); + + ret = cfl_object_print(fp, object); + TEST_CHECK(ret == 0); + + ret = compare(fp, "{\"child\":{}}\n"); + TEST_CHECK(ret == 0); + + fclose(fp); + cfl_object_destroy(object); +} + +static void test_reject_nested_array_reuse() +{ + int ret; + FILE *fp; + struct cfl_object *object; + struct cfl_array *outer; + struct cfl_array *inner; + + object = cfl_object_create(); + TEST_CHECK(object != NULL); + + outer = cfl_array_create(1); + TEST_CHECK(outer != NULL); + + inner = cfl_array_create(0); + TEST_CHECK(inner != NULL); + + ret = cfl_array_append_array(outer, inner); + TEST_CHECK(ret == 0); + + ret = cfl_object_set(object, CFL_OBJECT_ARRAY, outer); + TEST_CHECK(ret == 0); + + ret = cfl_object_set(object, CFL_OBJECT_ARRAY, inner); + TEST_CHECK(ret == -1); + + fp = tmpfile(); + TEST_CHECK(fp != NULL); + + ret = cfl_object_print(fp, object); + TEST_CHECK(ret == 0); + + ret = compare(fp, "[[]]\n"); + TEST_CHECK(ret == 0); + + fclose(fp); + cfl_object_destroy(object); +} + +static void test_reject_nested_variant_reuse() +{ + int ret; + FILE *fp; + struct cfl_object *object; + struct cfl_kvlist *list; + struct cfl_variant *variant; + + object = cfl_object_create(); + TEST_CHECK(object != NULL); + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + + ret = cfl_kvlist_insert_string(list, "key", "value"); + TEST_CHECK(ret == 0); + + variant = cfl_kvlist_fetch(list, "key"); + TEST_CHECK(variant != NULL); + + ret = cfl_object_set(object, CFL_OBJECT_KVLIST, list); + TEST_CHECK(ret == 0); + + ret = cfl_object_set(object, CFL_OBJECT_VARIANT, variant); + TEST_CHECK(ret == -1); + + fp = tmpfile(); + TEST_CHECK(fp != NULL); + + ret = cfl_object_print(fp, object); + TEST_CHECK(ret == 0); + + ret = compare(fp, "{\"key\":\"value\"}\n"); + TEST_CHECK(ret == 0); + + fclose(fp); + cfl_object_destroy(object); +} + TEST_LIST = { { "test_basics", test_basics }, + { "test_replace_and_print", test_replace_and_print }, + { "test_reuse_owned_kvlist", test_reuse_owned_kvlist }, + { "test_reuse_owned_array", test_reuse_owned_array }, + { "test_reject_nested_kvlist_reuse", test_reject_nested_kvlist_reuse }, + { "test_reject_nested_array_reuse", test_reject_nested_array_reuse }, + { "test_reject_nested_variant_reuse", test_reject_nested_variant_reuse }, { 0 } }; diff --git a/lib/cfl/tests/sds.c b/lib/cfl/tests/sds.c index 899725e161f..d54b11a94ec 100644 --- a/lib/cfl/tests/sds.c +++ b/lib/cfl/tests/sds.c @@ -52,8 +52,82 @@ static void test_sds_printf() cfl_sds_destroy(s); } +static void test_sds_invalid_inputs() +{ + cfl_sds_t s; + cfl_sds_t tmp; + + tmp = cfl_sds_create_len("x", -1); + TEST_CHECK(tmp == NULL); + + s = cfl_sds_create("test"); + TEST_CHECK(s != NULL); + TEST_CHECK(cfl_sds_len(s) == 4); + + tmp = cfl_sds_cat(s, "x", -1); + TEST_CHECK(tmp == NULL); + TEST_CHECK(cfl_sds_len(s) == 4); + + tmp = cfl_sds_cat(NULL, "x", 1); + TEST_CHECK(tmp == NULL); + + tmp = cfl_sds_cat(s, NULL, 1); + TEST_CHECK(tmp == NULL); + TEST_CHECK(cfl_sds_len(s) == 4); + + cfl_sds_set_len(s, 100); + TEST_CHECK(cfl_sds_len(s) == 4); + + tmp = cfl_sds_printf(NULL, "%s", "x"); + TEST_CHECK(tmp == NULL); + + tmp = cfl_sds_printf(&s, NULL); + TEST_CHECK(tmp == NULL); + TEST_CHECK(cfl_sds_len(s) == 4); + + cfl_sds_cat_safe(NULL, "x", 1); + cfl_sds_destroy(s); +} + +static void test_sds_self_append() +{ + cfl_sds_t s; + cfl_sds_t tmp; + + s = cfl_sds_create("abcdef"); + TEST_CHECK(s != NULL); + + tmp = cfl_sds_cat(s, s, cfl_sds_len(s)); + TEST_CHECK(tmp != NULL); + s = tmp; + + TEST_CHECK(cfl_sds_len(s) == 12); + TEST_CHECK(strcmp("abcdefabcdef", s) == 0); + + cfl_sds_destroy(s); +} + +static void test_sds_rejects_oversized_in_buffer_slice() +{ + cfl_sds_t s; + cfl_sds_t tmp; + + s = cfl_sds_create("abcdef"); + TEST_CHECK(s != NULL); + + tmp = cfl_sds_cat(s, s + 4, 4); + TEST_CHECK(tmp == NULL); + TEST_CHECK(cfl_sds_len(s) == 6); + TEST_CHECK(strcmp("abcdef", s) == 0); + + cfl_sds_destroy(s); +} + TEST_LIST = { { "sds_usage" , test_sds_usage}, { "sds_printf", test_sds_printf}, + { "sds_invalid_inputs", test_sds_invalid_inputs}, + { "sds_self_append", test_sds_self_append}, + { "sds_rejects_oversized_in_buffer_slice", test_sds_rejects_oversized_in_buffer_slice}, { 0 } }; diff --git a/lib/cfl/tests/utils.c b/lib/cfl/tests/utils.c index a24e0df4b7a..a3869153cf8 100644 --- a/lib/cfl/tests/utils.c +++ b/lib/cfl/tests/utils.c @@ -153,10 +153,25 @@ void test_cfl_utils_split_quoted_errors() TEST_CHECK(split == NULL); } +void test_cfl_utils_null_inputs() +{ + struct cfl_list *split = NULL; + + split = cfl_utils_split(NULL, ',', 1); + TEST_CHECK(split == NULL); + + split = cfl_utils_split_quoted(NULL, ',', 1); + TEST_CHECK(split == NULL); + + cfl_utils_split_free_entry(NULL); + cfl_utils_split_free(NULL); +} + TEST_LIST = { { "test_flb_utils_split", test_cfl_utils_split }, { "test_flb_utils_split_quoted", test_cfl_utils_split_quoted}, { "test_flb_utils_split_quoted_errors", test_cfl_utils_split_quoted_errors}, + { "test_cfl_utils_null_inputs", test_cfl_utils_null_inputs}, { 0 } }; diff --git a/lib/cfl/tests/variant.c b/lib/cfl/tests/variant.c index 1824001cb57..ecc79dcf5f7 100644 --- a/lib/cfl/tests/variant.c +++ b/lib/cfl/tests/variant.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -383,6 +384,48 @@ static void test_variant_print_double() } } +static void test_variant_print_nonfinite_double() +{ + int ret; + int i; + double inputs[] = {HUGE_VAL, -HUGE_VAL, HUGE_VAL - HUGE_VAL}; + char *expect = "null"; + + FILE *fp = NULL; + struct cfl_variant *val = NULL; + + for (i=0; i 0); + + ret = compare(fp, expect, 0); + TEST_CHECK(ret == 0); + + cfl_variant_destroy(val); + fclose(fp); +} + +static void test_variant_print_escaped_string() +{ + int ret; + char input[] = "line\n\"quoted\"\\"; + char *expect = "\"line\\n\\\"quoted\\\"\\\\\""; + FILE *fp = NULL; + struct cfl_variant *val = NULL; + + fp = tmpfile(); + if (!TEST_CHECK(fp != NULL)) { + TEST_MSG("fp is NULL"); + return; + } + + val = cfl_variant_create_from_string(input); + if (!TEST_CHECK(val != NULL)) { + TEST_MSG("cfl_variant_create_from_string failed"); + fclose(fp); + return; + } + + ret = cfl_variant_print(fp, val); + TEST_CHECK(ret > 0); + + ret = compare(fp, expect, 0); + TEST_CHECK(ret == 0); + + cfl_variant_destroy(val); + fclose(fp); +} + static void test_variant_print_bytes() { int ret; @@ -509,12 +614,64 @@ static void test_variant_print_bytes() fclose(fp); } +static void test_variant_print_referenced_bytes() +{ + int ret; + char input[] = {0x1f, 0xaa, 0x0a, 0xff}; + char *expect = "1faa0aff"; + FILE *fp = NULL; + struct cfl_variant *val = NULL; + + fp = tmpfile(); + if (!TEST_CHECK(fp != NULL)) { + TEST_MSG("fp is NULL"); + return; + } + + val = cfl_variant_create_from_bytes(input, 4, CFL_TRUE); + if (!TEST_CHECK(val != NULL)) { + TEST_MSG("cfl_variant_create_from_bytes failed"); + fclose(fp); + return; + } + + ret = cfl_variant_print(fp, val); + TEST_CHECK(ret > 0); + + ret = compare(fp, expect, 0); + TEST_CHECK(ret == 0); + + cfl_variant_destroy(val); + fclose(fp); +} + +static void test_variant_invalid_inputs() +{ + struct cfl_variant *val; + int ret; + + val = cfl_variant_create_from_string(NULL); + TEST_CHECK(val == NULL); + + val = cfl_variant_create_from_string_s(NULL, 1, CFL_FALSE); + TEST_CHECK(val == NULL); + + val = cfl_variant_create_from_bytes(NULL, 1, CFL_TRUE); + TEST_CHECK(val == NULL); + + TEST_CHECK(cfl_variant_size_get(NULL) == 0); + cfl_variant_size_set(NULL, 1); + + ret = cfl_variant_print(NULL, NULL); + TEST_CHECK(ret == -1); +} + static void test_variant_print_reference() { int ret; int *input = (int*)0x12345678; - char expect[] = "0x12345678"; + char expect[] = "null"; FILE *fp = NULL; struct cfl_variant *val = NULL; @@ -533,7 +690,7 @@ static void test_variant_print_reference() } ret = cfl_variant_print(fp, val); - if (!TEST_CHECK(ret > 0)) { + if (!TEST_CHECK(ret != EOF)) { TEST_MSG("cfl_variant_print failed"); fclose(fp); cfl_variant_destroy(val); @@ -589,9 +746,14 @@ TEST_LIST = { {"variant_print_int64", test_variant_print_int64}, {"variant_print_uint64", test_variant_print_uint64}, {"variant_print_double", test_variant_print_double}, + {"variant_print_nonfinite_double", test_variant_print_nonfinite_double}, {"variant_print_string", test_variant_print_string}, {"variant_print_string_s", test_variant_print_string_s}, + {"variant_print_sized_string_without_nul", test_variant_print_sized_string_without_nul}, + {"variant_print_escaped_string", test_variant_print_escaped_string}, {"variant_print_bytes", test_variant_print_bytes}, + {"variant_print_referenced_bytes", test_variant_print_referenced_bytes}, + {"variant_invalid_inputs", test_variant_invalid_inputs}, {"variant_print_array", test_variant_print_array}, {"variant_print_kvlist", test_variant_print_kvlist}, {"variant_print_reference", test_variant_print_reference}, From 4d3d5bc67848ebaeab06d4d9b52649ce7910aac4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 11 May 2026 19:44:33 -0600 Subject: [PATCH 15/18] downstream_worker: use cfl atomics for shutdown flag Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_downstream_worker.h | 5 +++-- src/flb_downstream_worker.c | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/include/fluent-bit/flb_downstream_worker.h b/include/fluent-bit/flb_downstream_worker.h index 9bf511ce2e8..8e9112d7324 100644 --- a/include/fluent-bit/flb_downstream_worker.h +++ b/include/fluent-bit/flb_downstream_worker.h @@ -20,9 +20,10 @@ #ifndef FLB_DOWNSTREAM_WORKER_H #define FLB_DOWNSTREAM_WORKER_H -#include #include +#include + #include #include @@ -56,7 +57,7 @@ struct flb_downstream_worker { pthread_t thread; pthread_mutex_t mutex; pthread_cond_t condition; - atomic_int should_exit; + uint64_t should_exit; int initialized; int thread_created; int startup_result; diff --git a/src/flb_downstream_worker.c b/src/flb_downstream_worker.c index 19f93736804..d9b8baf62a1 100644 --- a/src/flb_downstream_worker.c +++ b/src/flb_downstream_worker.c @@ -82,7 +82,7 @@ static void *downstream_worker_thread(void *data) goto cleanup; } - while (atomic_load(&worker->should_exit) == FLB_FALSE) { + while (cfl_atomic_load(&worker->should_exit) == FLB_FALSE) { mk_event_wait_2(worker->event_loop, 250); mk_event_foreach(event, worker->event_loop) { @@ -194,7 +194,7 @@ void flb_downstream_worker_runtime_stop(struct flb_downstream_worker_runtime *ru } for (i = 0; i < runtime->active_workers; i++) { - atomic_store(&runtime->workers[i].should_exit, FLB_TRUE); + cfl_atomic_store(&runtime->workers[i].should_exit, FLB_TRUE); if (runtime->workers[i].thread_created == FLB_TRUE) { pthread_join(runtime->workers[i].thread, NULL); } From bb5cba8157ed749ee45003c9675a18071513117b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 11 May 2026 19:44:39 -0600 Subject: [PATCH 16/18] http_server: preserve response buffer after append Signed-off-by: Eduardo Silva --- src/http_server/flb_http_server_http1.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/http_server/flb_http_server_http1.c b/src/http_server/flb_http_server_http1.c index a84a9683628..a8e35b81df9 100644 --- a/src/http_server/flb_http_server_http1.c +++ b/src/http_server/flb_http_server_http1.c @@ -420,6 +420,8 @@ int flb_http1_response_commit(struct flb_http_response *response) return -9; } + response_buffer = sds_result; + if (response->body != NULL) { sds_result = cfl_sds_cat(response_buffer, response->body, From fd077b003b0b5e26965d8c1e1ff29c90f931e5fa Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 11 May 2026 20:23:03 -0600 Subject: [PATCH 17/18] downstream_worker: dispatch callbacks on worker threads Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_downstream_worker.h | 10 ++- src/flb_downstream_worker.c | 98 ++++++++++++++++++++-- 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/include/fluent-bit/flb_downstream_worker.h b/include/fluent-bit/flb_downstream_worker.h index 8e9112d7324..486bf139cc3 100644 --- a/include/fluent-bit/flb_downstream_worker.h +++ b/include/fluent-bit/flb_downstream_worker.h @@ -20,11 +20,11 @@ #ifndef FLB_DOWNSTREAM_WORKER_H #define FLB_DOWNSTREAM_WORKER_H -#include - #include #include +#include +#include #include @@ -49,6 +49,8 @@ typedef void (*flb_downstream_worker_foreach_cb)(struct flb_downstream_worker *w struct flb_downstream_worker { struct flb_downstream_worker_runtime *runtime; struct mk_event_loop *event_loop; + struct mk_event control_event; + flb_pipefd_t control_channel[2]; void *context; void *parent; int worker_id; @@ -58,8 +60,12 @@ struct flb_downstream_worker { pthread_mutex_t mutex; pthread_cond_t condition; uint64_t should_exit; + flb_downstream_worker_foreach_cb control_callback; + void *control_data; int initialized; int thread_created; + int control_channel_created; + int control_done; int startup_result; }; diff --git a/src/flb_downstream_worker.c b/src/flb_downstream_worker.c index d9b8baf62a1..850e4e26185 100644 --- a/src/flb_downstream_worker.c +++ b/src/flb_downstream_worker.c @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -48,6 +49,37 @@ static void downstream_worker_context_cleanup(struct flb_downstream_worker *work pthread_cond_destroy(&worker->condition); } +static int downstream_worker_control_event(struct flb_downstream_worker *worker) +{ + ssize_t bytes; + uint64_t signal; + flb_downstream_worker_foreach_cb callback; + void *data; + + bytes = flb_pipe_r(worker->control_channel[0], &signal, sizeof(signal)); + if (bytes <= 0) { + return -1; + } + + pthread_mutex_lock(&worker->mutex); + callback = worker->control_callback; + data = worker->control_data; + pthread_mutex_unlock(&worker->mutex); + + if (callback != NULL && worker->context != NULL) { + callback(worker, worker->context, data); + } + + pthread_mutex_lock(&worker->mutex); + worker->control_callback = NULL; + worker->control_data = NULL; + worker->control_done = FLB_TRUE; + pthread_cond_signal(&worker->condition); + pthread_mutex_unlock(&worker->mutex); + + return 0; +} + static void *downstream_worker_thread(void *data) { int ret; @@ -65,6 +97,17 @@ static void *downstream_worker_thread(void *data) goto signal_and_exit; } + MK_EVENT_NEW(&worker->control_event); + ret = mk_event_channel_create(worker->event_loop, + &worker->control_channel[0], + &worker->control_channel[1], + &worker->control_event); + if (ret != 0) { + ret = -1; + goto signal_and_exit; + } + worker->control_channel_created = FLB_TRUE; + flb_engine_evl_set(worker->event_loop); flb_net_ctx_init(&dns_ctx); flb_net_dns_ctx_set(&dns_ctx); @@ -85,9 +128,19 @@ static void *downstream_worker_thread(void *data) while (cfl_atomic_load(&worker->should_exit) == FLB_FALSE) { mk_event_wait_2(worker->event_loop, 250); - mk_event_foreach(event, worker->event_loop) { - if (event->type == FLB_ENGINE_EV_CUSTOM) { - event->handler(event); + { + mk_event_foreach(event, worker->event_loop) { + if (event->type == FLB_ENGINE_EV_CUSTOM) { + event->handler(event); + } + } + } + + { + mk_event_foreach(event, worker->event_loop) { + if (event == &worker->control_event) { + downstream_worker_control_event(worker); + } } } @@ -102,6 +155,14 @@ static void *downstream_worker_thread(void *data) worker->context = NULL; } + if (worker->control_channel_created == FLB_TRUE) { + mk_event_channel_destroy(worker->event_loop, + worker->control_channel[0], + worker->control_channel[1], + &worker->control_event); + worker->control_channel_created = FLB_FALSE; + } + if (worker->event_loop != NULL) { mk_event_loop_destroy(worker->event_loop); worker->event_loop = NULL; @@ -210,16 +271,39 @@ void flb_downstream_worker_runtime_foreach(struct flb_downstream_worker_runtime void *data) { int i; + ssize_t bytes; + uint64_t signal = 1; + struct flb_downstream_worker *worker; if (runtime == NULL || callback == NULL) { return; } for (i = 0; i < runtime->worker_count; i++) { - if (runtime->workers[i].context != NULL) { - callback(&runtime->workers[i], - runtime->workers[i].context, - data); + worker = &runtime->workers[i]; + + if (worker->context == NULL || worker->thread_created != FLB_TRUE || + worker->control_channel_created != FLB_TRUE) { + continue; + } + + pthread_mutex_lock(&worker->mutex); + worker->control_callback = callback; + worker->control_data = data; + worker->control_done = FLB_FALSE; + + bytes = flb_pipe_w(worker->control_channel[1], &signal, sizeof(signal)); + if (bytes <= 0) { + worker->control_callback = NULL; + worker->control_data = NULL; + worker->control_done = FLB_TRUE; + pthread_mutex_unlock(&worker->mutex); + continue; + } + + while (worker->control_done == FLB_FALSE) { + pthread_cond_wait(&worker->condition, &worker->mutex); } + pthread_mutex_unlock(&worker->mutex); } } From 33d65436cb65dcd9db6fb459d494045e24bfe483 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 11 May 2026 20:23:06 -0600 Subject: [PATCH 18/18] in_forward: close worker connections on pause Signed-off-by: Eduardo Silva --- plugins/in_forward/fw.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/in_forward/fw.c b/plugins/in_forward/fw.c index 8a0928446b0..d41bad8367b 100644 --- a/plugins/in_forward/fw.c +++ b/plugins/in_forward/fw.c @@ -470,6 +470,7 @@ static void in_fw_worker_pause(struct flb_downstream_worker *worker, flb_downstream_pause(ctx->downstream); ctx->is_paused = FLB_TRUE; ctx->state = FW_INSTANCE_STATE_PAUSED; + fw_conn_del_all(ctx); } }