From 38240b1b3f6f11824870a5a6044729ba074452c4 Mon Sep 17 00:00:00 2001 From: Domenic Bottini Date: Thu, 25 Jun 2026 19:40:59 +0900 Subject: [PATCH 1/3] dockerfile: install libzstd-dev for kafka zstd compression The container builder stages did not install libzstd-dev, so librdkafka's find_package(ZSTD) failed and WITH_ZSTD was disabled. Kafka producers in the resulting images could not use zstd compression. Add libzstd-dev to the builder and debug builder stages so librdkafka is compiled with zstd support. Signed-off-by: Domenic Bottini --- dockerfiles/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 9e96c846991..7c4c5ce940a 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -49,6 +49,7 @@ RUN apt-get update && \ pkg-config \ libsystemd-dev \ zlib1g-dev \ + libzstd-dev \ libpq-dev \ postgresql-server-dev-all \ flex \ @@ -263,7 +264,7 @@ RUN apt-get update && \ openssl \ htop atop strace iotop sysstat ncdu logrotate hdparm pciutils psmisc tree pv \ make tar flex bison \ - libssl-dev libsasl2-dev libsystemd-dev zlib1g-dev libpq-dev libyaml-dev postgresql-server-dev-all \ + libssl-dev libsasl2-dev libsystemd-dev zlib1g-dev libzstd-dev libpq-dev libyaml-dev postgresql-server-dev-all \ && apt-get satisfy -y cmake "cmake (<< 4.0)" \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From 99acbc8514272056274946a80eaf0f18f487576d Mon Sep 17 00:00:00 2001 From: Domenic Bottini Date: Thu, 25 Jun 2026 19:41:39 +0900 Subject: [PATCH 2/3] tests: runtime: cover librdkafka zstd compression support Add a runtime test that sets compression.codec=zstd on a librdkafka conf. rd_kafka_conf_set only accepts the zstd codec when librdkafka was built with WITH_ZSTD, so this guards against images shipping without zstd support. Signed-off-by: Domenic Bottini --- tests/runtime/out_kafka.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/runtime/out_kafka.c b/tests/runtime/out_kafka.c index 4c5c852bc44..d18ff7f5643 100644 --- a/tests/runtime/out_kafka.c +++ b/tests/runtime/out_kafka.c @@ -3,9 +3,34 @@ #include #include "flb_tests_runtime.h" +#include "rdkafka.h" + /* Test data */ #include "data/td/json_td.h" +/* + * Ensure librdkafka was compiled with zstd support so Kafka producers can + * negotiate zstd compression. Setting compression.codec to zstd only succeeds + * when WITH_ZSTD was enabled at build time, otherwise rd_kafka_conf_set + * reports the codec as not built in. + */ +void flb_test_zstd_compression_available() +{ + rd_kafka_conf_t *conf; + rd_kafka_conf_res_t res; + char errstr[512] = {0}; + + conf = rd_kafka_conf_new(); + TEST_CHECK(conf != NULL); + + res = rd_kafka_conf_set(conf, "compression.codec", "zstd", + errstr, sizeof(errstr)); + TEST_CHECK(res == RD_KAFKA_CONF_OK); + TEST_MSG("compression.codec=zstd rejected: %s", errstr); + + rd_kafka_conf_destroy(conf); +} + void flb_test_raw_format() { @@ -44,6 +69,7 @@ void flb_test_raw_format() } TEST_LIST = { + { "zstd_compression_available", flb_test_zstd_compression_available }, { "raw_format", flb_test_raw_format }, { NULL, NULL }, }; From d51a13dd0fe8cf52b4b2b5089a20e6d9e678e057 Mon Sep 17 00:00:00 2001 From: Domenic Bottini Date: Mon, 29 Jun 2026 16:44:35 +0900 Subject: [PATCH 3/3] tests: runtime: add librdkafka include path for out_kafka The out_kafka runtime test includes librdkafka's public rdkafka.h header directly, but the generic runtime test build loop does not add the kafka include path the in_kafka/out_kafka plugins rely on. This breaks the build with 'rdkafka.h: No such file or directory'. Add the librdkafka include directory to the flb-rt-out_kafka target, covering both the system librdkafka (pkg-config) and bundled librdkafka builds. Co-authored-by: Rovo Dev (Claude) Signed-off-by: Domenic Bottini --- tests/runtime/CMakeLists.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/runtime/CMakeLists.txt b/tests/runtime/CMakeLists.txt index 104750b456e..1e5d4b7d571 100644 --- a/tests/runtime/CMakeLists.txt +++ b/tests/runtime/CMakeLists.txt @@ -345,6 +345,17 @@ foreach(source_file ${CHECK_PROGRAMS}) endif() endforeach() +# The out_kafka runtime test includes librdkafka's public header directly, so it +# needs the same include path the in_kafka/out_kafka plugins use. Cover both the +# system librdkafka (pkg-config) and the bundled librdkafka build. +if(TARGET flb-rt-out_kafka) + if(DEFINED KAFKA_INCLUDEDIR) + target_include_directories(flb-rt-out_kafka PRIVATE ${KAFKA_INCLUDEDIR}/librdkafka) + endif() + target_include_directories(flb-rt-out_kafka PRIVATE + ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RDKAFKA}/src) +endif() + function(flb_runtime_lock_tests resource_name) foreach(test_name ${ARGN}) if (TEST ${test_name})