From 495618a9580d3f8ea31274471c0622144a95a6f0 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 02:58:18 -0600 Subject: [PATCH 01/29] opentelemetry: add OTLP JSON and protobuf encoders Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_opentelemetry.h | 71 + src/CMakeLists.txt | 11 +- .../flb_opentelemetry_otlp_json.c | 3616 +++++++++++++++++ .../flb_opentelemetry_otlp_proto.c | 1656 ++++++++ 4 files changed, 5353 insertions(+), 1 deletion(-) create mode 100644 src/opentelemetry/flb_opentelemetry_otlp_json.c create mode 100644 src/opentelemetry/flb_opentelemetry_otlp_proto.c diff --git a/include/fluent-bit/flb_opentelemetry.h b/include/fluent-bit/flb_opentelemetry.h index 4e005d041c5..e621b531272 100644 --- a/include/fluent-bit/flb_opentelemetry.h +++ b/include/fluent-bit/flb_opentelemetry.h @@ -21,6 +21,7 @@ #define FLB_OPENTELEMETRY_H #include +#include #include #include #include @@ -92,6 +93,32 @@ struct flb_otel_error_map { int code; }; +struct cmt; +struct ctrace; +struct flb_log_event; + +enum flb_opentelemetry_otlp_json_result { + FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS = 0, + FLB_OPENTELEMETRY_OTLP_JSON_INVALID_ARGUMENT = -1, + FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED = -2, + FLB_OPENTELEMETRY_OTLP_JSON_INVALID_LOG_EVENT = -3 +}; + +enum flb_opentelemetry_otlp_proto_result { + FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS = 0, + FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_ARGUMENT = -1, + FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED = -2, + FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_LOG_EVENT = -3 +}; + +struct flb_opentelemetry_otlp_json_options { + int logs_require_otel_metadata; + const char *logs_body_key; + const char **logs_body_keys; + size_t logs_body_key_count; + int logs_body_key_attributes; +}; + static struct flb_otel_error_map otel_error_map[] = { {"FLB_OTEL_RESOURCE_INVALID_ATTRIBUTE", FLB_OTEL_RESOURCE_INVALID_ATTRIBUTE}, {"FLB_OTEL_LOGS_ERR_UNEXPECTED_ROOT_OBJECT_TYPE", FLB_OTEL_LOGS_ERR_UNEXPECTED_ROOT_OBJECT_TYPE}, @@ -180,6 +207,50 @@ int flb_opentelemetry_metrics_json_to_cmt(struct cfl_list *context_list, struct ctrace *flb_opentelemetry_json_traces_to_ctrace(const char *body, size_t len, int *error_status); +/* + * OTLP JSON encoding entry points shared by outputs and processors. + * Traces and metrics are intentionally typed to keep transport plugins + * independent from OpenTelemetry schema details. + */ +flb_sds_t flb_opentelemetry_traces_to_otlp_json(struct ctrace *context, + int *result); + +flb_sds_t flb_opentelemetry_traces_msgpack_to_otlp_json(const void *data, + size_t size, + int *result); + +flb_sds_t flb_opentelemetry_metrics_to_otlp_json(struct cmt *context, + int *result); + +flb_sds_t flb_opentelemetry_metrics_msgpack_to_otlp_json(const void *data, + size_t size, + int *result); + +flb_sds_t flb_opentelemetry_logs_to_otlp_json(const void *event_chunk_data, + size_t event_chunk_size, + struct flb_opentelemetry_otlp_json_options *options, + int *result); + +flb_sds_t flb_opentelemetry_traces_to_otlp_proto(struct ctrace *context, + int *result); + +flb_sds_t flb_opentelemetry_metrics_to_otlp_proto(struct cmt *context, + int *result); + +flb_sds_t flb_opentelemetry_metrics_msgpack_to_otlp_proto(const void *data, + size_t size, + int *result); + +flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, + size_t event_chunk_size, + struct flb_opentelemetry_otlp_json_options *options, + int *result); + +int flb_opentelemetry_log_is_otlp(struct flb_log_event *log_event); + +int flb_opentelemetry_logs_chunk_is_otlp(const void *event_chunk_data, + size_t event_chunk_size); + /* OpenTelemetry utils */ int flb_otel_utils_find_map_entry_by_key(msgpack_object_map *map, char *key, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f179a76e52a..a47a5d9d658 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,6 +18,7 @@ set(src flb_pack.c flb_pack_json.c flb_pack_gelf.c + flb_json.c flb_sds.c flb_sds_list.c flb_pipe.c @@ -118,6 +119,8 @@ set(src ${src} opentelemetry/flb_opentelemetry_logs.c opentelemetry/flb_opentelemetry_metrics.c + opentelemetry/flb_opentelemetry_otlp_json.c + opentelemetry/flb_opentelemetry_otlp_proto.c opentelemetry/flb_opentelemetry_traces.c opentelemetry/flb_opentelemetry_utils.c ) @@ -410,7 +413,6 @@ set(FLB_DEPS ctraces-static mk_core jsmn - yyjson ${MSGPACK_LIBRARIES} mpack-static chunkio-static @@ -426,6 +428,13 @@ set(FLB_DEPS ${NGHTTP2_LIBRARIES} ) +if(FLB_YYJSON) + set(FLB_DEPS + ${FLB_DEPS} + yyjson + ) +endif() + if(OPENSSL_FOUND) set(FLB_DEPS ${FLB_DEPS} diff --git a/src/opentelemetry/flb_opentelemetry_otlp_json.c b/src/opentelemetry/flb_opentelemetry_otlp_json.c new file mode 100644 index 00000000000..7064cba1062 --- /dev/null +++ b/src/opentelemetry/flb_opentelemetry_otlp_json.c @@ -0,0 +1,3616 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#define FLB_OTEL_LOGS_SCHEMA_KEY "schema" +#define FLB_OTEL_LOGS_SCHEMA_OTLP "otlp" +#define FLB_OTEL_LOGS_METADATA_KEY "otlp" + +struct otlp_logs_scope_state { + int64_t scope_id; + struct flb_json_mut_val *scope_log; + struct flb_json_mut_val *log_records; +}; + +struct otlp_logs_resource_state { + int64_t resource_id; + struct flb_json_mut_val *resource_log; + struct flb_json_mut_val *scope_logs; + struct otlp_logs_scope_state *scopes; + size_t scope_count; +}; + +struct otlp_metrics_scope_state { + struct flb_json_mut_val *metrics; +}; + +static msgpack_object *msgpack_map_get_object(msgpack_object_map *map, + const char *key); + +static void set_result(int *result, int value) +{ + if (result != NULL) { + *result = value; + } +} + +static void set_error(int *result, int value, int err) +{ + set_result(result, value); + errno = err; +} + +static flb_sds_t otlp_doc_to_sds(struct flb_json_mut_doc *doc) +{ + char *json_buffer; + size_t json_size; + flb_sds_t json; + + json_buffer = flb_json_mut_write(doc, &json_size); + if (json_buffer == NULL) { + return NULL; + } + + json = flb_sds_create_len(json_buffer, json_size); + free(json_buffer); + + return json; +} + +static int append_rendered_root_array_content(flb_sds_t *target, + int *first_entry, + flb_sds_t rendered, + const char *root_key) +{ + flb_sds_t prefix; + char *content_start; + char *suffix; + size_t content_length; + + if (target == NULL || *target == NULL || first_entry == NULL || + rendered == NULL || root_key == NULL) { + return -1; + } + + prefix = flb_sds_create_size(strlen(root_key) + 8); + if (prefix == NULL) { + return -1; + } + + prefix = flb_sds_printf(&prefix, "{\"%s\":[", root_key); + if (prefix == NULL) { + return -1; + } + + if (flb_sds_len(rendered) < flb_sds_len(prefix) + 2 || + strncmp(rendered, prefix, flb_sds_len(prefix)) != 0 || + strcmp(rendered + flb_sds_len(rendered) - 2, "]}") != 0) { + flb_sds_destroy(prefix); + return -1; + } + + content_start = rendered + flb_sds_len(prefix); + suffix = rendered + flb_sds_len(rendered) - 2; + content_length = (size_t) (suffix - content_start); + + flb_sds_destroy(prefix); + + if (content_length == 0) { + return 0; + } + + if (!*first_entry) { + *target = flb_sds_cat(*target, ",", 1); + if (*target == NULL) { + return -1; + } + } + + *target = flb_sds_cat(*target, content_start, content_length); + if (*target == NULL) { + return -1; + } + + *first_entry = FLB_FALSE; + + return 0; +} + +static int json_add_uint64_string(struct flb_json_mut_doc *doc, + struct flb_json_mut_val *obj, + const char *key, + uint64_t value) +{ + char buffer[32]; + int length; + + length = snprintf(buffer, sizeof(buffer), "%" PRIu64, value); + if (length <= 0 || (size_t) length >= sizeof(buffer)) { + return -1; + } + + return flb_json_mut_obj_add_strncpy(doc, obj, key, buffer, length) ? 0 : -1; +} + +static int json_add_int64_string(struct flb_json_mut_doc *doc, + struct flb_json_mut_val *obj, + const char *key, + int64_t value) +{ + char buffer[32]; + int length; + + length = snprintf(buffer, sizeof(buffer), "%" PRId64, value); + if (length <= 0 || (size_t) length >= sizeof(buffer)) { + return -1; + } + + return flb_json_mut_obj_add_strncpy(doc, obj, key, buffer, length) ? 0 : -1; +} + +static int binary_to_base64_sds(const char *input, size_t length, flb_sds_t *output) +{ + int result; + size_t encoded_length; + flb_sds_t encoded; + + *output = NULL; + + result = flb_base64_encode(NULL, 0, &encoded_length, + (const unsigned char *) input, length); + if (result != 0 && result != -0x002A) { + return -1; + } + + encoded = flb_sds_create_size(encoded_length + 1); + if (encoded == NULL) { + flb_errno(); + return -1; + } + + result = flb_base64_encode((unsigned char *) encoded, + flb_sds_alloc(encoded), + &encoded_length, + (const unsigned char *) input, + length); + if (result != 0) { + flb_sds_destroy(encoded); + return -1; + } + + cfl_sds_set_len((cfl_sds_t) encoded, encoded_length); + *output = encoded; + + return 0; +} + +static int binary_to_hex(char *output, size_t output_size, + const char *input, size_t input_size) +{ + static const char hex[] = "0123456789abcdef"; + size_t index; + + if (output_size < (input_size * 2) + 1) { + return -1; + } + + for (index = 0; index < input_size; index++) { + output[index * 2] = hex[((unsigned char) input[index]) >> 4]; + output[(index * 2) + 1] = hex[((unsigned char) input[index]) & 0x0f]; + } + + output[input_size * 2] = '\0'; + + return 0; +} + +static int otlp_uint64_field_value(msgpack_object_map *map, + const char *key, + uint64_t *value) +{ + msgpack_object *field; + + if (map == NULL || key == NULL || value == NULL) { + return -1; + } + + field = msgpack_map_get_object(map, (char *) key); + if (field == NULL) { + return -1; + } + + if (field->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *value = field->via.u64; + return 0; + } + + if (field->type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + field->via.i64 >= 0) { + *value = (uint64_t) field->via.i64; + return 0; + } + + return -1; +} + +static int valid_sds_reference(cfl_sds_t value) +{ + return ((uintptr_t) value) > 1; +} + +static struct flb_json_mut_val *create_binary_value(struct flb_json_mut_doc *doc, + const char *input, + size_t length) +{ + flb_sds_t encoded; + struct flb_json_mut_val *value; + + if (binary_to_base64_sds(input, length, &encoded) != 0) { + return NULL; + } + + value = flb_json_mut_strncpy(doc, encoded, flb_sds_len(encoded)); + flb_sds_destroy(encoded); + + return value; +} + +static struct flb_json_mut_val *msgpack_object_to_otlp_any_value(struct flb_json_mut_doc *doc, + msgpack_object *object); + +static struct flb_json_mut_val *msgpack_array_to_otlp_any_value(struct flb_json_mut_doc *doc, + msgpack_object_array *array) +{ + size_t index; + struct flb_json_mut_val *entry; + struct flb_json_mut_val *result; + struct flb_json_mut_val *values; + + result = flb_json_mut_obj(doc); + values = flb_json_mut_arr(doc); + + if (result == NULL || values == NULL) { + return NULL; + } + + if (!flb_json_mut_obj_add_val(doc, result, "values", values)) { + return NULL; + } + + for (index = 0; index < array->size; index++) { + entry = msgpack_object_to_otlp_any_value(doc, &array->ptr[index]); + if (entry == NULL || !flb_json_mut_arr_add_val(values, entry)) { + return NULL; + } + } + + return result; +} + +static struct flb_json_mut_val *msgpack_map_to_otlp_kv_array(struct flb_json_mut_doc *doc, + msgpack_object_map *map) +{ + int key_index; + size_t index; + struct flb_json_mut_val *entry; + struct flb_json_mut_val *result; + struct flb_json_mut_val *value; + + result = flb_json_mut_arr(doc); + if (result == NULL) { + return NULL; + } + + for (index = 0; index < map->size; index++) { + if (map->ptr[index].key.type != MSGPACK_OBJECT_STR) { + return NULL; + } + + entry = flb_json_mut_obj(doc); + value = msgpack_object_to_otlp_any_value(doc, &map->ptr[index].val); + if (entry == NULL || value == NULL) { + return NULL; + } + + key_index = flb_json_mut_obj_add_strncpy(doc, + entry, + "key", + map->ptr[index].key.via.str.ptr, + map->ptr[index].key.via.str.size); + if (!key_index || + !flb_json_mut_obj_add_val(doc, entry, "value", value) || + !flb_json_mut_arr_add_val(result, entry)) { + return NULL; + } + } + + return result; +} + +static struct flb_json_mut_val *msgpack_map_to_otlp_kv_array_filtered( + struct flb_json_mut_doc *doc, + msgpack_object_map *map, + const char *skip_key, + size_t skip_key_length) +{ + int key_index; + size_t index; + struct flb_json_mut_val *entry; + struct flb_json_mut_val *result; + struct flb_json_mut_val *value; + msgpack_object *key; + + result = flb_json_mut_arr(doc); + if (result == NULL) { + return NULL; + } + + for (index = 0; index < map->size; index++) { + key = &map->ptr[index].key; + + if (key->type != MSGPACK_OBJECT_STR) { + return NULL; + } + + if (skip_key != NULL && + key->via.str.size == skip_key_length && + strncmp(key->via.str.ptr, skip_key, skip_key_length) == 0) { + continue; + } + + entry = flb_json_mut_obj(doc); + value = msgpack_object_to_otlp_any_value(doc, &map->ptr[index].val); + if (entry == NULL || value == NULL) { + return NULL; + } + + key_index = flb_json_mut_obj_add_strncpy(doc, + entry, + "key", + key->via.str.ptr, + key->via.str.size); + if (!key_index || + !flb_json_mut_obj_add_val(doc, entry, "value", value) || + !flb_json_mut_arr_add_val(result, entry)) { + return NULL; + } + } + + return result; +} + +static struct flb_json_mut_val *msgpack_map_to_otlp_any_value(struct flb_json_mut_doc *doc, + msgpack_object_map *map) +{ + struct flb_json_mut_val *result; + struct flb_json_mut_val *values; + + result = flb_json_mut_obj(doc); + values = msgpack_map_to_otlp_kv_array(doc, map); + + if (result == NULL || values == NULL) { + return NULL; + } + + if (!flb_json_mut_obj_add_val(doc, result, "values", values)) { + return NULL; + } + + return result; +} + +static struct flb_json_mut_val *msgpack_object_to_otlp_any_value(struct flb_json_mut_doc *doc, + msgpack_object *object) +{ + struct flb_json_mut_val *root; + struct flb_json_mut_val *value; + + if (object == NULL || object->type == MSGPACK_OBJECT_NIL) { + return NULL; + } + + root = flb_json_mut_obj(doc); + if (root == NULL) { + return NULL; + } + + switch (object->type) { + case MSGPACK_OBJECT_BOOLEAN: + return flb_json_mut_obj_add_bool(doc, root, "boolValue", + object->via.boolean) ? root : NULL; + case MSGPACK_OBJECT_POSITIVE_INTEGER: + return json_add_uint64_string(doc, root, "intValue", + object->via.u64) == 0 ? root : NULL; + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + return json_add_int64_string(doc, root, "intValue", + object->via.i64) == 0 ? root : NULL; + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + return flb_json_mut_obj_add_real(doc, root, "doubleValue", + object->via.f64) ? root : NULL; + case MSGPACK_OBJECT_STR: + return flb_json_mut_obj_add_strncpy(doc, root, "stringValue", + object->via.str.ptr, + object->via.str.size) ? root : NULL; + case MSGPACK_OBJECT_BIN: + value = create_binary_value(doc, object->via.bin.ptr, object->via.bin.size); + return (value != NULL && + flb_json_mut_obj_add_val(doc, root, "bytesValue", value)) ? root : NULL; + case MSGPACK_OBJECT_ARRAY: + value = msgpack_array_to_otlp_any_value(doc, &object->via.array); + return (value != NULL && + flb_json_mut_obj_add_val(doc, root, "arrayValue", value)) ? root : NULL; + case MSGPACK_OBJECT_MAP: + value = msgpack_map_to_otlp_any_value(doc, &object->via.map); + return (value != NULL && + flb_json_mut_obj_add_val(doc, root, "kvlistValue", value)) ? root : NULL; + default: + return NULL; + } +} + +static struct flb_json_mut_val *cfl_variant_to_otlp_any_value(struct flb_json_mut_doc *doc, + struct cfl_variant *variant); + +static struct flb_json_mut_val *cfl_array_to_otlp_any_value(struct flb_json_mut_doc *doc, + struct cfl_array *array) +{ + size_t index; + struct flb_json_mut_val *entry; + struct flb_json_mut_val *result; + struct flb_json_mut_val *values; + + result = flb_json_mut_obj(doc); + values = flb_json_mut_arr(doc); + + if (result == NULL || values == NULL) { + return NULL; + } + + if (!flb_json_mut_obj_add_val(doc, result, "values", values)) { + return NULL; + } + + for (index = 0; index < array->entry_count; index++) { + entry = cfl_variant_to_otlp_any_value(doc, array->entries[index]); + if (entry == NULL || !flb_json_mut_arr_add_val(values, entry)) { + return NULL; + } + } + + return result; +} + +static struct flb_json_mut_val *cfl_kvlist_to_otlp_kv_array(struct flb_json_mut_doc *doc, + struct cfl_kvlist *kvlist) +{ + struct cfl_kvpair *pair; + struct cfl_list *head; + struct flb_json_mut_val *entry; + struct flb_json_mut_val *result; + struct flb_json_mut_val *value; + + result = flb_json_mut_arr(doc); + if (result == NULL) { + return NULL; + } + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + + entry = flb_json_mut_obj(doc); + value = cfl_variant_to_otlp_any_value(doc, pair->val); + if (entry == NULL || value == NULL) { + return NULL; + } + + if (!flb_json_mut_obj_add_strncpy(doc, entry, "key", + pair->key, cfl_sds_len(pair->key)) || + !flb_json_mut_obj_add_val(doc, entry, "value", value) || + !flb_json_mut_arr_add_val(result, entry)) { + return NULL; + } + } + + return result; +} + +static struct flb_json_mut_val *cfl_kvlist_to_otlp_any_value(struct flb_json_mut_doc *doc, + struct cfl_kvlist *kvlist) +{ + struct flb_json_mut_val *result; + struct flb_json_mut_val *values; + + result = flb_json_mut_obj(doc); + values = cfl_kvlist_to_otlp_kv_array(doc, kvlist); + + if (result == NULL || values == NULL) { + return NULL; + } + + if (!flb_json_mut_obj_add_val(doc, result, "values", values)) { + return NULL; + } + + return result; +} + +static struct flb_json_mut_val *cfl_variant_to_otlp_any_value(struct flb_json_mut_doc *doc, + struct cfl_variant *variant) +{ + struct flb_json_mut_val *root; + struct flb_json_mut_val *value; + + if (variant == NULL) { + return NULL; + } + + root = flb_json_mut_obj(doc); + if (root == NULL) { + return NULL; + } + + switch (variant->type) { + case CFL_VARIANT_BOOL: + return flb_json_mut_obj_add_bool(doc, root, "boolValue", + variant->data.as_bool) ? root : NULL; + case CFL_VARIANT_INT: + return json_add_int64_string(doc, root, "intValue", + variant->data.as_int64) == 0 ? root : NULL; + case CFL_VARIANT_UINT: + return json_add_uint64_string(doc, root, "intValue", + variant->data.as_uint64) == 0 ? root : NULL; + case CFL_VARIANT_DOUBLE: + return flb_json_mut_obj_add_real(doc, root, "doubleValue", + variant->data.as_double) ? root : NULL; + case CFL_VARIANT_STRING: + return flb_json_mut_obj_add_strncpy(doc, root, "stringValue", + variant->data.as_string, + cfl_sds_len(variant->data.as_string)) ? root : NULL; + case CFL_VARIANT_BYTES: + value = create_binary_value(doc, variant->data.as_bytes, + cfl_sds_len(variant->data.as_bytes)); + return (value != NULL && + flb_json_mut_obj_add_val(doc, root, "bytesValue", value)) ? root : NULL; + case CFL_VARIANT_ARRAY: + value = cfl_array_to_otlp_any_value(doc, variant->data.as_array); + return (value != NULL && + flb_json_mut_obj_add_val(doc, root, "arrayValue", value)) ? root : NULL; + case CFL_VARIANT_KVLIST: + value = cfl_kvlist_to_otlp_any_value(doc, variant->data.as_kvlist); + return (value != NULL && + flb_json_mut_obj_add_val(doc, root, "kvlistValue", value)) ? root : NULL; + default: + return NULL; + } +} + +static int msgpack_map_entry_is_string(msgpack_object_map *map, + const char *key, + const char *value) +{ + int index; + msgpack_object *entry_value; + size_t value_length; + + if (map == NULL || key == NULL || value == NULL) { + return FLB_FALSE; + } + + index = flb_otel_utils_find_map_entry_by_key(map, (char *) key, 0, FLB_TRUE); + if (index < 0) { + return FLB_FALSE; + } + + entry_value = &map->ptr[index].val; + value_length = strlen(value); + + if (entry_value->type != MSGPACK_OBJECT_STR || + entry_value->via.str.size != value_length) { + return FLB_FALSE; + } + + return strncmp(entry_value->via.str.ptr, value, value_length) == 0; +} + +static int msgpack_map_contains_key(msgpack_object_map *map, const char *key) +{ + if (map == NULL || key == NULL) { + return FLB_FALSE; + } + + return flb_otel_utils_find_map_entry_by_key(map, (char *) key, 0, FLB_TRUE) >= 0; +} + +int flb_opentelemetry_log_is_otlp(struct flb_log_event *log_event) +{ + if (log_event == NULL) { + return FLB_FALSE; + } + + if (log_event->group_metadata != NULL && + log_event->group_metadata->type == MSGPACK_OBJECT_MAP && + msgpack_map_entry_is_string(&log_event->group_metadata->via.map, + FLB_OTEL_LOGS_SCHEMA_KEY, + FLB_OTEL_LOGS_SCHEMA_OTLP) == FLB_TRUE) { + return FLB_TRUE; + } + + if (log_event->metadata != NULL && + log_event->metadata->type == MSGPACK_OBJECT_MAP && + msgpack_map_contains_key(&log_event->metadata->via.map, + FLB_OTEL_LOGS_METADATA_KEY) == FLB_TRUE) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +int flb_opentelemetry_logs_chunk_is_otlp(const void *event_chunk_data, + size_t event_chunk_size) +{ + int ret; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + + if (event_chunk_data == NULL || event_chunk_size == 0) { + return FLB_FALSE; + } + + ret = flb_log_event_decoder_init(&decoder, + (char *) event_chunk_data, + event_chunk_size); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + return FLB_FALSE; + } + + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + + while ((ret = flb_log_event_decoder_next(&decoder, &event)) == + FLB_EVENT_DECODER_SUCCESS) { + if (flb_opentelemetry_log_is_otlp(&event) == FLB_TRUE) { + flb_log_event_decoder_destroy(&decoder); + return FLB_TRUE; + } + } + + flb_log_event_decoder_destroy(&decoder); + + return FLB_FALSE; +} + +static int msgpack_map_get_int64(msgpack_object_map *map, + const char *key, + int64_t *output) +{ + int index; + msgpack_object *value; + + index = flb_otel_utils_find_map_entry_by_key(map, (char *) key, 0, FLB_TRUE); + if (index < 0) { + return -1; + } + + value = &map->ptr[index].val; + + if (value->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *output = (int64_t) value->via.u64; + return 0; + } + else if (value->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *output = value->via.i64; + return 0; + } + + return -1; +} + +static msgpack_object *msgpack_map_get_object(msgpack_object_map *map, + const char *key) +{ + int index; + + index = flb_otel_utils_find_map_entry_by_key(map, (char *) key, 0, FLB_TRUE); + if (index < 0) { + return NULL; + } + + return &map->ptr[index].val; +} + +static struct otlp_logs_resource_state *find_logs_resource_state( + struct otlp_logs_resource_state *states, + size_t state_count, + int64_t resource_id) +{ + size_t index; + + for (index = 0; index < state_count; index++) { + if (states[index].resource_id == resource_id) { + return &states[index]; + } + } + + return NULL; +} + +static struct otlp_logs_scope_state *find_logs_scope_state( + struct otlp_logs_resource_state *resource, + int64_t scope_id) +{ + size_t index; + + for (index = 0; index < resource->scope_count; index++) { + if (resource->scopes[index].scope_id == scope_id) { + return &resource->scopes[index]; + } + } + + return NULL; +} + +static int add_msgpack_attributes_array(struct flb_json_mut_doc *doc, + struct flb_json_mut_val *obj, + const char *key, + msgpack_object *source) +{ + struct flb_json_mut_val *value; + + if (source == NULL || source->type != MSGPACK_OBJECT_MAP) { + return 0; + } + + value = msgpack_map_to_otlp_kv_array(doc, &source->via.map); + if (value == NULL) { + return -1; + } + + return flb_json_mut_obj_add_val(doc, obj, key, value) ? 0 : -1; +} + +static struct flb_json_mut_val *logs_group_resource_to_json(struct flb_json_mut_doc *doc, + msgpack_object *resource_object) +{ + msgpack_object *attributes; + msgpack_object *metadata; + struct flb_json_mut_val *resource; + + resource = flb_json_mut_obj(doc); + if (resource == NULL) { + return NULL; + } + + if (resource_object == NULL || resource_object->type != MSGPACK_OBJECT_MAP) { + return resource; + } + + attributes = msgpack_map_get_object(&resource_object->via.map, "attributes"); + if (attributes != NULL && + add_msgpack_attributes_array(doc, resource, "attributes", attributes) != 0) { + return NULL; + } + + metadata = msgpack_map_get_object(&resource_object->via.map, + "dropped_attributes_count"); + if (metadata != NULL) { + if (metadata->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (!flb_json_mut_obj_add_uint(doc, resource, "droppedAttributesCount", + metadata->via.u64)) { + return NULL; + } + } + else if (metadata->type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + metadata->via.i64 >= 0) { + if (!flb_json_mut_obj_add_uint(doc, resource, "droppedAttributesCount", + (uint64_t) metadata->via.i64)) { + return NULL; + } + } + } + + return resource; +} + +static struct flb_json_mut_val *logs_group_scope_to_json(struct flb_json_mut_doc *doc, + msgpack_object *scope_object) +{ + msgpack_object *field; + struct flb_json_mut_val *scope; + + scope = flb_json_mut_obj(doc); + if (scope == NULL) { + return NULL; + } + + if (scope_object == NULL || scope_object->type != MSGPACK_OBJECT_MAP) { + return scope; + } + + field = msgpack_map_get_object(&scope_object->via.map, "name"); + if (field != NULL && field->type == MSGPACK_OBJECT_STR) { + if (!flb_json_mut_obj_add_strncpy(doc, scope, "name", + field->via.str.ptr, + field->via.str.size)) { + return NULL; + } + } + + field = msgpack_map_get_object(&scope_object->via.map, "version"); + if (field != NULL && field->type == MSGPACK_OBJECT_STR) { + if (!flb_json_mut_obj_add_strncpy(doc, scope, "version", + field->via.str.ptr, + field->via.str.size)) { + return NULL; + } + } + + field = msgpack_map_get_object(&scope_object->via.map, "attributes"); + if (field != NULL && + add_msgpack_attributes_array(doc, scope, "attributes", field) != 0) { + return NULL; + } + + field = msgpack_map_get_object(&scope_object->via.map, "dropped_attributes_count"); + if (field != NULL && field->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (!flb_json_mut_obj_add_uint(doc, scope, "droppedAttributesCount", + field->via.u64)) { + return NULL; + } + } + + return scope; +} + +static struct otlp_logs_resource_state *append_logs_resource_state( + struct flb_json_mut_doc *doc, + struct flb_json_mut_val *resource_logs, + struct otlp_logs_resource_state **states, + size_t *state_count, + int64_t resource_id, + msgpack_object *resource_object, + msgpack_object *resource_body) +{ + struct otlp_logs_resource_state *new_states; + struct otlp_logs_resource_state *state; + msgpack_object *schema_url; + struct flb_json_mut_val *resource_log; + struct flb_json_mut_val *resource; + struct flb_json_mut_val *scope_logs; + + resource_log = flb_json_mut_obj(doc); + scope_logs = flb_json_mut_arr(doc); + resource = logs_group_resource_to_json(doc, resource_object); + + if (resource_log == NULL || scope_logs == NULL || resource == NULL) { + return NULL; + } + + if (!flb_json_mut_obj_add_val(doc, resource_log, "resource", resource) || + !flb_json_mut_obj_add_val(doc, resource_log, "scopeLogs", scope_logs) || + !flb_json_mut_arr_add_val(resource_logs, resource_log)) { + return NULL; + } + + if (resource_body != NULL && resource_body->type == MSGPACK_OBJECT_MAP) { + schema_url = msgpack_map_get_object(&resource_body->via.map, "schema_url"); + if (schema_url != NULL && schema_url->type == MSGPACK_OBJECT_STR) { + if (!flb_json_mut_obj_add_strncpy(doc, + resource_log, + "schemaUrl", + schema_url->via.str.ptr, + schema_url->via.str.size)) { + return NULL; + } + } + } + + new_states = flb_realloc(*states, + sizeof(struct otlp_logs_resource_state) * + (*state_count + 1)); + if (new_states == NULL) { + flb_errno(); + return NULL; + } + + *states = new_states; + state = &new_states[*state_count]; + memset(state, 0, sizeof(struct otlp_logs_resource_state)); + + state->resource_id = resource_id; + state->resource_log = resource_log; + state->scope_logs = scope_logs; + + (*state_count)++; + + return state; +} + +static struct otlp_logs_scope_state *append_logs_scope_state( + struct flb_json_mut_doc *doc, + struct otlp_logs_resource_state *resource_state, + int64_t scope_id, + msgpack_object *scope_object) +{ + struct otlp_logs_scope_state *new_scopes; + struct otlp_logs_scope_state *state; + msgpack_object *schema_url; + struct flb_json_mut_val *log_records; + struct flb_json_mut_val *scope; + struct flb_json_mut_val *scope_log; + + scope_log = flb_json_mut_obj(doc); + log_records = flb_json_mut_arr(doc); + scope = logs_group_scope_to_json(doc, scope_object); + + if (scope_log == NULL || log_records == NULL || scope == NULL) { + return NULL; + } + + if (!flb_json_mut_obj_add_val(doc, scope_log, "scope", scope) || + !flb_json_mut_obj_add_val(doc, scope_log, "logRecords", log_records) || + !flb_json_mut_arr_add_val(resource_state->scope_logs, scope_log)) { + return NULL; + } + + if (scope_object != NULL && scope_object->type == MSGPACK_OBJECT_MAP) { + schema_url = msgpack_map_get_object(&scope_object->via.map, "schema_url"); + if (schema_url != NULL && schema_url->type == MSGPACK_OBJECT_STR) { + if (!flb_json_mut_obj_add_strncpy(doc, + scope_log, + "schemaUrl", + schema_url->via.str.ptr, + schema_url->via.str.size)) { + return NULL; + } + } + } + + new_scopes = flb_realloc(resource_state->scopes, + sizeof(struct otlp_logs_scope_state) * + (resource_state->scope_count + 1)); + if (new_scopes == NULL) { + flb_errno(); + return NULL; + } + + resource_state->scopes = new_scopes; + state = &new_scopes[resource_state->scope_count]; + memset(state, 0, sizeof(struct otlp_logs_scope_state)); + + state->scope_id = scope_id; + state->scope_log = scope_log; + state->log_records = log_records; + + resource_state->scope_count++; + + return state; +} + +static int ensure_default_logs_scope_state( + struct flb_json_mut_doc *doc, + struct flb_json_mut_val *resource_logs, + struct otlp_logs_resource_state **resource_states, + size_t *resource_state_count, + struct otlp_logs_resource_state **current_resource, + struct otlp_logs_scope_state **current_scope) +{ + *current_resource = find_logs_resource_state(*resource_states, + *resource_state_count, + 0); + if (*current_resource == NULL) { + *current_resource = append_logs_resource_state(doc, + resource_logs, + resource_states, + resource_state_count, + 0, + NULL, + NULL); + if (*current_resource == NULL) { + return -1; + } + } + + *current_scope = find_logs_scope_state(*current_resource, 0); + if (*current_scope == NULL) { + *current_scope = append_logs_scope_state(doc, + *current_resource, + 0, + NULL); + if (*current_scope == NULL) { + return -1; + } + } + + return 0; +} + +static msgpack_object *find_log_body_candidate(msgpack_object *body, + const char **logs_body_keys, + size_t logs_body_key_count, + const char **matched_key, + size_t *matched_key_length) +{ + size_t index; + msgpack_object *candidate; + + if (body == NULL || body->type == MSGPACK_OBJECT_NIL) { + return NULL; + } + + if (body->type != MSGPACK_OBJECT_MAP) { + return body; + } + + for (index = 0; index < logs_body_key_count; index++) { + if (logs_body_keys[index] == NULL) { + continue; + } + + candidate = msgpack_map_get_object(&body->via.map, logs_body_keys[index]); + if (candidate != NULL) { + if (matched_key != NULL) { + *matched_key = logs_body_keys[index]; + } + if (matched_key_length != NULL) { + *matched_key_length = strlen(logs_body_keys[index]); + } + return candidate; + } + } + + return body; +} + +static struct flb_json_mut_val *extract_log_body_value(struct flb_json_mut_doc *doc, + msgpack_object *body, + const char **logs_body_keys, + size_t logs_body_key_count, + const char **matched_key, + size_t *matched_key_length) +{ + msgpack_object *candidate; + + candidate = find_log_body_candidate(body, + logs_body_keys, + logs_body_key_count, + matched_key, + matched_key_length); + + return msgpack_object_to_otlp_any_value(doc, candidate); +} + +static int add_binary_id_field(struct flb_json_mut_doc *doc, + struct flb_json_mut_val *obj, + const char *key, + msgpack_object *value, + size_t expected_size) +{ + char hex_buffer[33]; + flb_sds_t encoded; + + if (value == NULL || value->type != MSGPACK_OBJECT_BIN) { + return 0; + } + + if (value->via.bin.size == expected_size && + binary_to_hex(hex_buffer, sizeof(hex_buffer), + value->via.bin.ptr, value->via.bin.size) == 0) { + return flb_json_mut_obj_add_strcpy(doc, obj, key, hex_buffer) ? 0 : -1; + } + + if (binary_to_base64_sds(value->via.bin.ptr, value->via.bin.size, &encoded) != 0) { + return -1; + } + + if (!flb_json_mut_obj_add_strn(doc, obj, key, encoded, flb_sds_len(encoded))) { + flb_sds_destroy(encoded); + return -1; + } + + flb_sds_destroy(encoded); + + return 0; +} + +static struct flb_json_mut_val *log_record_to_json(struct flb_json_mut_doc *doc, + struct flb_log_event *event, + const char **logs_body_keys, + size_t logs_body_key_count, + int logs_body_key_attributes) +{ + int attributes_added; + const char *matched_body_key; + msgpack_object *metadata; + msgpack_object *field; + msgpack_object *otlp_metadata; + struct flb_json_mut_val *attributes; + struct flb_json_mut_val *body; + struct flb_json_mut_val *record; + size_t matched_body_key_length; + uint64_t timestamp; + + record = flb_json_mut_obj(doc); + if (record == NULL) { + return NULL; + } + + metadata = event->metadata; + otlp_metadata = NULL; + attributes = NULL; + attributes_added = FLB_FALSE; + matched_body_key = NULL; + matched_body_key_length = 0; + + if (metadata != NULL && metadata->type == MSGPACK_OBJECT_MAP) { + otlp_metadata = msgpack_map_get_object(&metadata->via.map, FLB_OTEL_LOGS_METADATA_KEY); + } + + if (otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP && + otlp_uint64_field_value(&otlp_metadata->via.map, "timestamp", ×tamp) == 0) { + /* preserve the exact OTLP log timestamp when the normalized chunk carries it */ + } + else if (event->raw_timestamp != NULL) { + if (event->raw_timestamp->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + timestamp = event->raw_timestamp->via.u64; + } + else if (event->raw_timestamp->type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + event->raw_timestamp->via.i64 >= 0) { + timestamp = (uint64_t) event->raw_timestamp->via.i64; + } + else { + timestamp = flb_time_to_nanosec(&event->timestamp); + } + } + else { + timestamp = flb_time_to_nanosec(&event->timestamp); + } + if (timestamp > 0 && + json_add_uint64_string(doc, record, "timeUnixNano", timestamp) != 0) { + return NULL; + } + + if (otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP) { + field = msgpack_map_get_object(&otlp_metadata->via.map, "observed_timestamp"); + if (field != NULL) { + if (field->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (json_add_uint64_string(doc, record, "observedTimeUnixNano", + field->via.u64) != 0) { + return NULL; + } + } + else if (field->type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + field->via.i64 >= 0) { + if (json_add_uint64_string(doc, record, "observedTimeUnixNano", + (uint64_t) field->via.i64) != 0) { + return NULL; + } + } + } + + field = msgpack_map_get_object(&otlp_metadata->via.map, "severity_number"); + if (field != NULL) { + if (field->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (!flb_json_mut_obj_add_uint(doc, record, "severityNumber", + field->via.u64)) { + return NULL; + } + } + else if (field->type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + field->via.i64 >= 0) { + if (!flb_json_mut_obj_add_uint(doc, record, "severityNumber", + (uint64_t) field->via.i64)) { + return NULL; + } + } + } + + field = msgpack_map_get_object(&otlp_metadata->via.map, "severity_text"); + if (field != NULL && field->type == MSGPACK_OBJECT_STR) { + if (!flb_json_mut_obj_add_strncpy(doc, record, "severityText", + field->via.str.ptr, + field->via.str.size)) { + return NULL; + } + } + + field = msgpack_map_get_object(&otlp_metadata->via.map, "attributes"); + if (field != NULL && field->type == MSGPACK_OBJECT_MAP) { + attributes = msgpack_map_to_otlp_kv_array(doc, &field->via.map); + if (attributes == NULL) { + return NULL; + } + + if (flb_json_mut_arr_size(attributes) > 0 && + !flb_json_mut_obj_add_val(doc, record, "attributes", attributes)) { + return NULL; + } + + attributes_added = FLB_TRUE; + } + + if (add_binary_id_field(doc, record, "traceId", + msgpack_map_get_object(&otlp_metadata->via.map, "trace_id"), + 16) != 0 || + add_binary_id_field(doc, record, "spanId", + msgpack_map_get_object(&otlp_metadata->via.map, "span_id"), + 8) != 0) { + return NULL; + } + } + + body = extract_log_body_value(doc, + event->body, + logs_body_keys, + logs_body_key_count, + &matched_body_key, + &matched_body_key_length); + + if (logs_body_key_attributes == FLB_TRUE && + matched_body_key != NULL && + event->body != NULL && + event->body->type == MSGPACK_OBJECT_MAP && + attributes == NULL) { + attributes = msgpack_map_to_otlp_kv_array_filtered(doc, + &event->body->via.map, + matched_body_key, + matched_body_key_length); + if (attributes == NULL) { + return NULL; + } + } + + if (attributes != NULL && + attributes_added == FLB_FALSE && + flb_json_mut_arr_size(attributes) > 0 && + !flb_json_mut_obj_add_val(doc, record, "attributes", attributes)) { + return NULL; + } + + if (body != NULL && !flb_json_mut_obj_add_val(doc, record, "body", body)) { + return NULL; + } + + return record; +} + +static void destroy_logs_resource_states(struct otlp_logs_resource_state *states, + size_t count) +{ + size_t index; + + if (states == NULL) { + return; + } + + for (index = 0; index < count; index++) { + if (states[index].scopes != NULL) { + flb_free(states[index].scopes); + } + } + + flb_free(states); +} + +flb_sds_t flb_opentelemetry_logs_to_otlp_json(const void *event_chunk_data, + size_t event_chunk_size, + struct flb_opentelemetry_otlp_json_options *options, + int *result) +{ + int ret; + int32_t record_type; + int logs_body_key_attributes; + int require_otel_metadata; + int64_t resource_id; + int64_t scope_id; + const char **logs_body_keys; + const char *logs_body_key; + struct flb_json_mut_doc *doc; + struct flb_json_mut_val *resource_logs; + struct flb_json_mut_val *root; + struct flb_json_mut_val *record; + msgpack_object *group_body; + msgpack_object *group_metadata; + msgpack_object *resource_object; + msgpack_object *scope_object; + flb_sds_t json; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + struct otlp_logs_scope_state *current_scope; + struct otlp_logs_resource_state *current_resource; + struct otlp_logs_resource_state *resource_states; + size_t logs_body_key_count; + size_t resource_state_count; + static const char *default_logs_body_keys[] = {"log", "message"}; + + if (event_chunk_data == NULL || event_chunk_size == 0) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + require_otel_metadata = FLB_TRUE; + logs_body_key = "log"; + logs_body_keys = default_logs_body_keys; + logs_body_key_count = 2; + logs_body_key_attributes = FLB_FALSE; + + if (options != NULL) { + require_otel_metadata = options->logs_require_otel_metadata; + logs_body_key_attributes = options->logs_body_key_attributes; + if (options->logs_body_keys != NULL && options->logs_body_key_count > 0) { + logs_body_keys = options->logs_body_keys; + logs_body_key_count = options->logs_body_key_count; + } + else if (options->logs_body_key != NULL) { + logs_body_key = options->logs_body_key; + logs_body_keys = &logs_body_key; + logs_body_key_count = 1; + } + } + + if (require_otel_metadata == FLB_TRUE && + flb_opentelemetry_logs_chunk_is_otlp(event_chunk_data, + event_chunk_size) != FLB_TRUE) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_LOG_EVENT, EINVAL); + return NULL; + } + + ret = flb_log_event_decoder_init(&decoder, + (char *) event_chunk_data, + event_chunk_size); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + + doc = flb_json_mut_doc_create(); + root = flb_json_mut_obj(doc); + resource_logs = flb_json_mut_arr(doc); + resource_states = NULL; + resource_state_count = 0; + current_resource = NULL; + current_scope = NULL; + json = NULL; + + if (doc == NULL || root == NULL || resource_logs == NULL || + !flb_json_mut_obj_add_val(doc, root, "resourceLogs", resource_logs)) { + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + if (doc != NULL) { + flb_json_mut_doc_destroy(doc); + } + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + flb_json_mut_doc_set_root(doc, root); + + while ((ret = flb_log_event_decoder_next(&decoder, &event)) == + FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &record_type); + if (ret != 0) { + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_LOG_EVENT, EINVAL); + return NULL; + } + + if (record_type == FLB_LOG_EVENT_GROUP_START) { + group_metadata = event.group_metadata != NULL ? event.group_metadata : event.metadata; + group_body = event.body; + + if (group_metadata == NULL || + group_metadata->type != MSGPACK_OBJECT_MAP || + msgpack_map_entry_is_string(&group_metadata->via.map, + FLB_OTEL_LOGS_SCHEMA_KEY, + FLB_OTEL_LOGS_SCHEMA_OTLP) != FLB_TRUE || + msgpack_map_get_int64(&group_metadata->via.map, "resource_id", &resource_id) != 0 || + msgpack_map_get_int64(&group_metadata->via.map, "scope_id", &scope_id) != 0) { + if (require_otel_metadata == FLB_TRUE) { + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_LOG_EVENT, EINVAL); + return NULL; + } + + current_resource = NULL; + current_scope = NULL; + continue; + } + + resource_object = NULL; + scope_object = NULL; + + if (group_body != NULL && group_body->type == MSGPACK_OBJECT_MAP) { + resource_object = msgpack_map_get_object(&group_body->via.map, "resource"); + scope_object = msgpack_map_get_object(&group_body->via.map, "scope"); + } + + current_resource = find_logs_resource_state(resource_states, + resource_state_count, + resource_id); + if (current_resource == NULL) { + current_resource = append_logs_resource_state(doc, + resource_logs, + &resource_states, + &resource_state_count, + resource_id, + resource_object, + group_body); + if (current_resource == NULL) { + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + current_scope = find_logs_scope_state(current_resource, scope_id); + if (current_scope == NULL) { + current_scope = append_logs_scope_state(doc, + current_resource, + scope_id, + scope_object); + if (current_scope == NULL) { + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + continue; + } + else if (record_type == FLB_LOG_EVENT_GROUP_END) { + current_resource = NULL; + current_scope = NULL; + continue; + } + + if (current_scope == NULL) { + if (require_otel_metadata == FLB_TRUE) { + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_LOG_EVENT, EINVAL); + return NULL; + } + + if (ensure_default_logs_scope_state(doc, + resource_logs, + &resource_states, + &resource_state_count, + ¤t_resource, + ¤t_scope) != 0) { + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + record = log_record_to_json(doc, + &event, + logs_body_keys, + logs_body_key_count, + logs_body_key_attributes); + if (record == NULL || + !flb_json_mut_arr_add_val(current_scope->log_records, record)) { + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + + if (ret != FLB_EVENT_DECODER_SUCCESS && + ret != FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA) { + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_LOG_EVENT, EINVAL); + return NULL; + } + + json = otlp_doc_to_sds(doc); + flb_json_mut_doc_destroy(doc); + + if (json == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + set_result(result, FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS); + + return json; +} + +static struct cfl_kvlist *fetch_metadata_kvlist_key(struct cfl_kvlist *kvlist, + const char *key) +{ + struct cfl_variant *entry; + + if (kvlist == NULL) { + return NULL; + } + + entry = cfl_kvlist_fetch(kvlist, (char *) key); + if (entry == NULL || entry->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + return entry->data.as_kvlist; +} + +static struct cfl_array *fetch_metadata_array_key(struct cfl_kvlist *kvlist, + const char *key) +{ + struct cfl_variant *entry; + + if (kvlist == NULL) { + return NULL; + } + + entry = cfl_kvlist_fetch(kvlist, (char *) key); + if (entry == NULL || entry->type != CFL_VARIANT_ARRAY) { + return NULL; + } + + return entry->data.as_array; +} + +static struct cfl_kvlist *fetch_array_kvlist_entry(struct cfl_array *array, + size_t index) +{ + struct cfl_variant *entry; + + if (array == NULL || index >= array->entry_count) { + return NULL; + } + + entry = cfl_array_fetch_by_index(array, index); + if (entry == NULL || entry->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + return entry->data.as_kvlist; +} + +static int kvlist_fetch_uint64(struct cfl_kvlist *kvlist, + const char *key, + uint64_t *output) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(kvlist, (char *) key); + if (value == NULL) { + return -1; + } + + if (value->type == CFL_VARIANT_UINT) { + *output = value->data.as_uint64; + return 0; + } + + if (value->type == CFL_VARIANT_INT && value->data.as_int64 >= 0) { + *output = (uint64_t) value->data.as_int64; + return 0; + } + + return -1; +} + +static int kvlist_fetch_int64(struct cfl_kvlist *kvlist, + const char *key, + int64_t *output) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(kvlist, (char *) key); + if (value == NULL) { + return -1; + } + + if (value->type == CFL_VARIANT_INT) { + *output = value->data.as_int64; + return 0; + } + + if (value->type == CFL_VARIANT_UINT && value->data.as_uint64 <= INT64_MAX) { + *output = (int64_t) value->data.as_uint64; + return 0; + } + + return -1; +} + +static int kvlist_fetch_double(struct cfl_kvlist *kvlist, + const char *key, + double *output) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(kvlist, (char *) key); + if (value == NULL) { + return -1; + } + + if (value->type == CFL_VARIANT_DOUBLE) { + *output = value->data.as_double; + return 0; + } + if (value->type == CFL_VARIANT_UINT) { + *output = (double) value->data.as_uint64; + return 0; + } + if (value->type == CFL_VARIANT_INT) { + *output = (double) value->data.as_int64; + return 0; + } + + return -1; +} + +static int kvlist_fetch_bool(struct cfl_kvlist *kvlist, + const char *key, + int *output) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(kvlist, (char *) key); + if (value == NULL) { + return -1; + } + + if (value->type == CFL_VARIANT_BOOL) { + *output = value->data.as_bool ? CMT_TRUE : CMT_FALSE; + return 0; + } + if (value->type == CFL_VARIANT_UINT) { + *output = value->data.as_uint64 != 0 ? CMT_TRUE : CMT_FALSE; + return 0; + } + if (value->type == CFL_VARIANT_INT) { + *output = value->data.as_int64 != 0 ? CMT_TRUE : CMT_FALSE; + return 0; + } + + return -1; +} + +static char *kvlist_fetch_string(struct cfl_kvlist *kvlist, const char *key) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(kvlist, (char *) key); + if (value == NULL || value->type != CFL_VARIANT_STRING) { + return NULL; + } + + return value->data.as_string; +} + +static char *metrics_map_type_to_key(int type) +{ + switch (type) { + case CMT_COUNTER: + return "counter"; + case CMT_GAUGE: + return "gauge"; + case CMT_UNTYPED: + return "untyped"; + case CMT_SUMMARY: + return "summary"; + case CMT_HISTOGRAM: + return "histogram"; + case CMT_EXP_HISTOGRAM: + return "exp_histogram"; + default: + return "unknown"; + } +} + +static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, + struct cmt_map *map) +{ + struct cfl_kvlist *otlp_root; + struct cfl_kvlist *metrics_root; + struct cfl_kvlist *type_root; + + if (cmt == NULL || map == NULL || map->opts == NULL || map->opts->fqname == NULL) { + return NULL; + } + + otlp_root = fetch_metadata_kvlist_key(cmt->external_metadata, "otlp"); + metrics_root = fetch_metadata_kvlist_key(otlp_root, "metrics"); + type_root = fetch_metadata_kvlist_key(metrics_root, metrics_map_type_to_key(map->type)); + + if (type_root == NULL) { + return NULL; + } + + return fetch_metadata_kvlist_key(type_root, map->opts->fqname); +} + +static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample) +{ + char key[128]; + struct cfl_kvlist *metric_context; + struct cfl_kvlist *datapoints_context; + + metric_context = get_metric_otlp_metadata_context(cmt, map); + datapoints_context = fetch_metadata_kvlist_key(metric_context, "datapoints"); + + if (datapoints_context == NULL) { + return NULL; + } + + snprintf(key, sizeof(key) - 1, "%" PRIx64 ":%" PRIu64, + sample != NULL ? sample->hash : 0, + sample != NULL ? cmt_metric_get_timestamp(sample) : 0); + + return fetch_metadata_kvlist_key(datapoints_context, key); +} + +static int compute_flat_scope_index(size_t *scope_counts, + size_t resource_count, + size_t resource_index, + size_t scope_index) +{ + size_t index; + size_t offset; + + offset = 0; + + for (index = 0; index < resource_index && index < resource_count; index++) { + offset += scope_counts[index]; + } + + return (int) (offset + scope_index); +} + +static size_t resolve_target_scope_index(struct cmt *cmt, + struct cmt_map *map, + size_t *scope_counts, + size_t resource_count, + size_t total_scope_count) +{ + uint64_t flat_scope_index; + uint64_t resource_index; + uint64_t scope_index; + struct cfl_kvlist *metadata; + struct cfl_kvlist *metric_context; + + metric_context = get_metric_otlp_metadata_context(cmt, map); + metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + + if (metadata == NULL) { + return 0; + } + + if (kvlist_fetch_uint64(metadata, "scope_flat_index", &flat_scope_index) == 0 && + flat_scope_index < total_scope_count) { + return (size_t) flat_scope_index; + } + + if (kvlist_fetch_uint64(metadata, "resource_index", &resource_index) != 0 || + kvlist_fetch_uint64(metadata, "scope_index", &scope_index) != 0) { + return 0; + } + + return (size_t) compute_flat_scope_index(scope_counts, + resource_count, + (size_t) resource_index, + (size_t) scope_index); +} + +static struct flb_json_mut_val *create_otlp_attributes_from_kvlist(struct flb_json_mut_doc *doc, + struct cfl_kvlist *kvlist) +{ + if (kvlist == NULL) { + return flb_json_mut_arr(doc); + } + + return cfl_kvlist_to_otlp_kv_array(doc, kvlist); +} + +static struct flb_json_mut_val *create_metrics_resource_json(struct flb_json_mut_doc *doc, + struct cfl_kvlist *resource_root) +{ + struct cfl_kvlist *attributes; + struct cfl_kvlist *metadata; + struct flb_json_mut_val *resource; + struct flb_json_mut_val *attribute_array; + int64_t dropped; + + resource = flb_json_mut_obj(doc); + if (resource == NULL) { + return NULL; + } + + if (resource_root == NULL) { + return resource; + } + + attributes = fetch_metadata_kvlist_key(resource_root, "attributes"); + metadata = fetch_metadata_kvlist_key(resource_root, "metadata"); + + if (attributes != NULL && cfl_kvlist_count(attributes) > 0) { + attribute_array = create_otlp_attributes_from_kvlist(doc, attributes); + if (attribute_array == NULL || + !flb_json_mut_obj_add_val(doc, resource, "attributes", attribute_array)) { + return NULL; + } + } + + if (metadata != NULL && + kvlist_fetch_int64(metadata, "dropped_attributes_count", &dropped) == 0 && + dropped >= 0) { + if (!flb_json_mut_obj_add_uint(doc, resource, "droppedAttributesCount", + (uint64_t) dropped)) { + return NULL; + } + } + + return resource; +} + +static struct flb_json_mut_val *create_metrics_scope_json(struct flb_json_mut_doc *doc, + struct cfl_kvlist *scope_root) +{ + struct cfl_kvlist *attributes; + struct cfl_kvlist *metadata; + struct flb_json_mut_val *scope; + struct flb_json_mut_val *attribute_array; + char *string; + int64_t dropped; + + scope = flb_json_mut_obj(doc); + if (scope == NULL) { + return NULL; + } + + if (scope_root == NULL) { + return scope; + } + + attributes = fetch_metadata_kvlist_key(scope_root, "attributes"); + metadata = fetch_metadata_kvlist_key(scope_root, "metadata"); + + if (attributes != NULL && cfl_kvlist_count(attributes) > 0) { + attribute_array = create_otlp_attributes_from_kvlist(doc, attributes); + if (attribute_array == NULL || + !flb_json_mut_obj_add_val(doc, scope, "attributes", attribute_array)) { + return NULL; + } + } + + if (metadata != NULL) { + string = kvlist_fetch_string(metadata, "name"); + if (string != NULL && + !flb_json_mut_obj_add_strn(doc, scope, "name", string, cfl_sds_len(string))) { + return NULL; + } + + string = kvlist_fetch_string(metadata, "version"); + if (string != NULL && + !flb_json_mut_obj_add_strn(doc, scope, "version", string, cfl_sds_len(string))) { + return NULL; + } + + if (kvlist_fetch_int64(metadata, "dropped_attributes_count", &dropped) == 0 && + dropped >= 0 && + !flb_json_mut_obj_add_uint(doc, scope, "droppedAttributesCount", + (uint64_t) dropped)) { + return NULL; + } + } + + return scope; +} + +static int add_metric_metadata_json(struct flb_json_mut_doc *doc, + struct flb_json_mut_val *metric, + struct cmt *cmt, + struct cmt_map *map) +{ + struct cfl_kvlist *context; + struct cfl_kvlist *metadata; + struct flb_json_mut_val *entries; + + context = get_metric_otlp_metadata_context(cmt, map); + metadata = fetch_metadata_kvlist_key(context, "metadata"); + + if (metadata == NULL || cfl_kvlist_count(metadata) == 0) { + return 0; + } + + entries = cfl_kvlist_to_otlp_kv_array(doc, metadata); + if (entries == NULL) { + return -1; + } + + return flb_json_mut_obj_add_val(doc, metric, "metadata", entries) ? 0 : -1; +} + +static struct flb_json_mut_val *create_exemplar_json(struct flb_json_mut_doc *doc, + struct cfl_kvlist *kvlist) +{ + char hex_buffer[33]; + flb_sds_t encoded; + struct cfl_kvlist *filtered_attributes; + struct cfl_variant *variant; + struct flb_json_mut_val *attributes; + struct flb_json_mut_val *exemplar; + int64_t sint; + uint64_t uint_value; + double double_value; + + exemplar = flb_json_mut_obj(doc); + if (exemplar == NULL) { + return NULL; + } + + if (kvlist_fetch_uint64(kvlist, "time_unix_nano", &uint_value) == 0 && + json_add_uint64_string(doc, exemplar, "timeUnixNano", uint_value) != 0) { + return NULL; + } + + variant = cfl_kvlist_fetch(kvlist, "span_id"); + if (variant != NULL && variant->type == CFL_VARIANT_BYTES) { + if (cfl_sds_len(variant->data.as_bytes) == 8 && + binary_to_hex(hex_buffer, sizeof(hex_buffer), + variant->data.as_bytes, + cfl_sds_len(variant->data.as_bytes)) == 0) { + if (!flb_json_mut_obj_add_strcpy(doc, exemplar, "spanId", hex_buffer)) { + return NULL; + } + } + else { + if (binary_to_base64_sds(variant->data.as_bytes, + cfl_sds_len(variant->data.as_bytes), + &encoded) != 0) { + return NULL; + } + if (!flb_json_mut_obj_add_strn(doc, exemplar, "spanId", + encoded, flb_sds_len(encoded))) { + flb_sds_destroy(encoded); + return NULL; + } + flb_sds_destroy(encoded); + } + } + + variant = cfl_kvlist_fetch(kvlist, "trace_id"); + if (variant != NULL && variant->type == CFL_VARIANT_BYTES) { + if (cfl_sds_len(variant->data.as_bytes) == 16 && + binary_to_hex(hex_buffer, sizeof(hex_buffer), + variant->data.as_bytes, + cfl_sds_len(variant->data.as_bytes)) == 0) { + if (!flb_json_mut_obj_add_strcpy(doc, exemplar, "traceId", hex_buffer)) { + return NULL; + } + } + else { + if (binary_to_base64_sds(variant->data.as_bytes, + cfl_sds_len(variant->data.as_bytes), + &encoded) != 0) { + return NULL; + } + if (!flb_json_mut_obj_add_strn(doc, exemplar, "traceId", + encoded, flb_sds_len(encoded))) { + flb_sds_destroy(encoded); + return NULL; + } + flb_sds_destroy(encoded); + } + } + + if (kvlist_fetch_double(kvlist, "as_double", &double_value) == 0) { + if (!flb_json_mut_obj_add_real(doc, exemplar, "asDouble", double_value)) { + return NULL; + } + } + else if (kvlist_fetch_int64(kvlist, "as_int", &sint) == 0) { + if (json_add_int64_string(doc, exemplar, "asInt", sint) != 0) { + return NULL; + } + } + else if (kvlist_fetch_uint64(kvlist, "as_int", &uint_value) == 0) { + if (json_add_uint64_string(doc, exemplar, "asInt", uint_value) != 0) { + return NULL; + } + } + + filtered_attributes = fetch_metadata_kvlist_key(kvlist, "filtered_attributes"); + if (filtered_attributes != NULL && cfl_kvlist_count(filtered_attributes) > 0) { + attributes = cfl_kvlist_to_otlp_kv_array(doc, filtered_attributes); + if (attributes == NULL || + !flb_json_mut_obj_add_val(doc, exemplar, "filteredAttributes", attributes)) { + return NULL; + } + } + + return exemplar; +} + +static int add_data_point_common_fields(struct flb_json_mut_doc *doc, + struct flb_json_mut_val *point, + struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample) +{ + int label_index; + int bool_value; + uint64_t uint_value; + struct cfl_list *head; + struct cmt_label *static_label; + struct cmt_map_label *label_name; + struct cmt_map_label *label_value; + struct cfl_kvlist *metadata; + struct cfl_variant *variant; + struct cfl_array *array; + struct flb_json_mut_val *attributes; + struct flb_json_mut_val *attribute; + struct flb_json_mut_val *value; + struct flb_json_mut_val *exemplars; + + attributes = flb_json_mut_arr(doc); + if (attributes == NULL) { + return -1; + } + + cfl_list_foreach(head, &cmt->static_labels->list) { + static_label = cfl_list_entry(head, struct cmt_label, _head); + attribute = flb_json_mut_obj(doc); + value = flb_json_mut_obj(doc); + + if (attribute == NULL || value == NULL || + !flb_json_mut_obj_add_strn(doc, attribute, "key", + static_label->key, + cfl_sds_len(static_label->key)) || + !flb_json_mut_obj_add_strn(doc, value, "stringValue", + static_label->val, + cfl_sds_len(static_label->val)) || + !flb_json_mut_obj_add_val(doc, attribute, "value", value) || + !flb_json_mut_arr_add_val(attributes, attribute)) { + return -1; + } + } + + label_index = 0; + label_name = cfl_list_entry_first(&map->label_keys, struct cmt_map_label, _head); + cfl_list_foreach(head, &sample->labels) { + label_value = cfl_list_entry(head, struct cmt_map_label, _head); + attribute = flb_json_mut_obj(doc); + value = flb_json_mut_obj(doc); + + (void) label_index; + + if (attribute == NULL || value == NULL || + !flb_json_mut_obj_add_strn(doc, attribute, "key", + label_name->name, + cfl_sds_len(label_name->name)) || + !flb_json_mut_obj_add_strn(doc, value, "stringValue", + label_value->name, + cfl_sds_len(label_value->name)) || + !flb_json_mut_obj_add_val(doc, attribute, "value", value) || + !flb_json_mut_arr_add_val(attributes, attribute)) { + return -1; + } + + label_name = cfl_list_entry_next(&label_name->_head, + struct cmt_map_label, + _head, + &map->label_keys); + } + + if (flb_json_mut_arr_size(attributes) > 0 && + !flb_json_mut_obj_add_val(doc, point, "attributes", attributes)) { + return -1; + } + + if (cmt_metric_get_timestamp(sample) > 0 && + json_add_uint64_string(doc, point, "timeUnixNano", + cmt_metric_get_timestamp(sample)) != 0) { + return -1; + } + + metadata = get_data_point_otlp_metadata_context(cmt, map, sample); + + if (cmt_metric_has_start_timestamp(sample)) { + if (json_add_uint64_string(doc, point, "startTimeUnixNano", + cmt_metric_get_start_timestamp(sample)) != 0) { + return -1; + } + } + else if (metadata != NULL && + kvlist_fetch_uint64(metadata, "start_time_unix_nano", &uint_value) == 0 && + json_add_uint64_string(doc, point, "startTimeUnixNano", uint_value) != 0) { + return -1; + } + + if (metadata != NULL && + kvlist_fetch_uint64(metadata, "flags", &uint_value) == 0 && + !flb_json_mut_obj_add_uint(doc, point, "flags", uint_value)) { + return -1; + } + + variant = metadata != NULL ? cfl_kvlist_fetch(metadata, "exemplars") : NULL; + if (variant != NULL && variant->type == CFL_VARIANT_ARRAY) { + array = variant->data.as_array; + exemplars = flb_json_mut_arr(doc); + if (exemplars == NULL) { + return -1; + } + + for (label_index = 0; label_index < (int) array->entry_count; label_index++) { + variant = cfl_array_fetch_by_index(array, label_index); + if (variant == NULL || variant->type != CFL_VARIANT_KVLIST) { + return -1; + } + + attribute = create_exemplar_json(doc, variant->data.as_kvlist); + if (attribute == NULL || !flb_json_mut_arr_add_val(exemplars, attribute)) { + return -1; + } + } + + if (flb_json_mut_arr_size(exemplars) > 0 && + !flb_json_mut_obj_add_val(doc, point, "exemplars", exemplars)) { + return -1; + } + } + + if (metadata != NULL && + kvlist_fetch_bool(metadata, "has_sum", &bool_value) == 0 && + bool_value == CMT_FALSE) { + if (!flb_json_mut_obj_add_bool(doc, point, "hasSum", false)) { + return -1; + } + } + + return 0; +} + +static int add_min_max_fields(struct flb_json_mut_doc *doc, + struct flb_json_mut_val *point, + struct cfl_kvlist *metadata) +{ + int bool_value; + double double_value; + + if (metadata == NULL) { + return 0; + } + + if (kvlist_fetch_bool(metadata, "has_min", &bool_value) == 0 && + bool_value == CMT_TRUE && + kvlist_fetch_double(metadata, "min", &double_value) == 0 && + !flb_json_mut_obj_add_real(doc, point, "min", double_value)) { + return -1; + } + + if (kvlist_fetch_bool(metadata, "has_max", &bool_value) == 0 && + bool_value == CMT_TRUE && + kvlist_fetch_double(metadata, "max", &double_value) == 0 && + !flb_json_mut_obj_add_real(doc, point, "max", double_value)) { + return -1; + } + + return 0; +} + +static struct flb_json_mut_val *create_number_data_point_json(struct flb_json_mut_doc *doc, + struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample) +{ + int value_type; + int64_t int64_value; + uint64_t uint64_value; + char *string; + struct cfl_kvlist *metadata; + struct flb_json_mut_val *point; + + point = flb_json_mut_obj(doc); + if (point == NULL || add_data_point_common_fields(doc, point, cmt, map, sample) != 0) { + return NULL; + } + + metadata = get_data_point_otlp_metadata_context(cmt, map, sample); + value_type = cmt_metric_get_value_type(sample); + + if (value_type == CMT_METRIC_VALUE_INT64) { + int64_value = cmt_metric_get_int64_value(sample); + if (json_add_int64_string(doc, point, "asInt", int64_value) != 0) { + return NULL; + } + } + else if (value_type == CMT_METRIC_VALUE_UINT64) { + uint64_value = cmt_metric_get_uint64_value(sample); + if (json_add_uint64_string(doc, point, "asInt", uint64_value) != 0) { + return NULL; + } + } + else { + string = metadata != NULL ? kvlist_fetch_string(metadata, "number_value_case") : NULL; + if (string != NULL && strcmp(string, "int") == 0) { + int64_value = cmt_metric_get_int64_value(sample); + if (json_add_int64_string(doc, point, "asInt", int64_value) != 0) { + return NULL; + } + } + else { + if (!flb_json_mut_obj_add_real(doc, point, "asDouble", + cmt_metric_get_value(sample))) { + return NULL; + } + } + } + + return point; +} + +static struct flb_json_mut_val *create_summary_data_point_json(struct flb_json_mut_doc *doc, + struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample, + struct cmt_summary *summary) +{ + size_t index; + struct flb_json_mut_val *point; + struct flb_json_mut_val *quantile_values; + struct flb_json_mut_val *entry; + + point = flb_json_mut_obj(doc); + if (point == NULL || add_data_point_common_fields(doc, point, cmt, map, sample) != 0) { + return NULL; + } + + if (json_add_uint64_string(doc, point, "count", + cmt_summary_get_count_value(sample)) != 0 || + !flb_json_mut_obj_add_real(doc, point, "sum", + cmt_summary_get_sum_value(sample))) { + return NULL; + } + + quantile_values = flb_json_mut_arr(doc); + if (quantile_values == NULL) { + return NULL; + } + + for (index = 0; index < summary->quantiles_count; index++) { + entry = flb_json_mut_obj(doc); + if (entry == NULL || + !flb_json_mut_obj_add_real(doc, entry, "quantile", summary->quantiles[index]) || + !flb_json_mut_obj_add_real(doc, entry, "value", + cmt_summary_quantile_get_value(sample, index)) || + !flb_json_mut_arr_add_val(quantile_values, entry)) { + return NULL; + } + } + + if (flb_json_mut_arr_size(quantile_values) > 0 && + !flb_json_mut_obj_add_val(doc, point, "quantileValues", quantile_values)) { + return NULL; + } + + return point; +} + +static struct flb_json_mut_val *create_histogram_data_point_json(struct flb_json_mut_doc *doc, + struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample, + struct cmt_histogram *histogram) +{ + size_t index; + int bool_value; + struct cfl_kvlist *metadata; + struct flb_json_mut_val *bucket_counts; + struct flb_json_mut_val *explicit_bounds; + struct flb_json_mut_val *point; + + point = flb_json_mut_obj(doc); + if (point == NULL || add_data_point_common_fields(doc, point, cmt, map, sample) != 0) { + return NULL; + } + + metadata = get_data_point_otlp_metadata_context(cmt, map, sample); + + if (json_add_uint64_string(doc, point, "count", + cmt_metric_hist_get_count_value(sample)) != 0) { + return NULL; + } + + if (!(metadata != NULL && + kvlist_fetch_bool(metadata, "has_sum", &bool_value) == 0 && + bool_value == CMT_FALSE)) { + if (!flb_json_mut_obj_add_real(doc, point, "sum", + cmt_metric_hist_get_sum_value(sample))) { + return NULL; + } + } + + if (add_min_max_fields(doc, point, metadata) != 0) { + return NULL; + } + + bucket_counts = flb_json_mut_arr(doc); + explicit_bounds = flb_json_mut_arr(doc); + if (bucket_counts == NULL || explicit_bounds == NULL) { + return NULL; + } + + for (index = 0; index < histogram->buckets->count + 1; index++) { + char buffer[32]; + int length; + + length = snprintf(buffer, sizeof(buffer), "%" PRIu64, + sample->hist_buckets[index]); + if (length <= 0 || (size_t) length >= sizeof(buffer) || + !flb_json_mut_arr_add_strncpy(doc, bucket_counts, buffer, length)) { + return NULL; + } + } + + for (index = 0; index < histogram->buckets->count; index++) { + if (!flb_json_mut_arr_add_real(doc, explicit_bounds, + histogram->buckets->upper_bounds[index])) { + return NULL; + } + } + + return (flb_json_mut_obj_add_val(doc, point, "bucketCounts", bucket_counts) && + flb_json_mut_obj_add_val(doc, point, "explicitBounds", explicit_bounds)) ? + point : NULL; +} + +static struct flb_json_mut_val *create_exp_histogram_data_point_json( + struct flb_json_mut_doc *doc, + struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample) +{ + size_t index; + int bool_value; + struct cfl_kvlist *metadata; + struct cmt_exp_histogram_snapshot snapshot; + struct flb_json_mut_val *negative; + struct flb_json_mut_val *point; + struct flb_json_mut_val *positive; + struct flb_json_mut_val *bucket_counts; + + if (cmt_metric_exp_hist_get_snapshot(sample, &snapshot) != 0) { + return NULL; + } + + point = flb_json_mut_obj(doc); + if (point == NULL || add_data_point_common_fields(doc, point, cmt, map, sample) != 0) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + + metadata = get_data_point_otlp_metadata_context(cmt, map, sample); + + if (json_add_uint64_string(doc, point, "count", snapshot.count) != 0 || + !flb_json_mut_obj_add_int(doc, point, "scale", snapshot.scale) || + json_add_uint64_string(doc, point, "zeroCount", snapshot.zero_count) != 0 || + !flb_json_mut_obj_add_real(doc, point, "zeroThreshold", snapshot.zero_threshold)) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + + if (!(metadata != NULL && + kvlist_fetch_bool(metadata, "has_sum", &bool_value) == 0 && + bool_value == CMT_FALSE)) { + if (snapshot.sum_set && + !flb_json_mut_obj_add_real(doc, point, "sum", + cmt_math_uint64_to_d64(snapshot.sum))) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + } + + if (add_min_max_fields(doc, point, metadata) != 0) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + + if (snapshot.positive_count > 0) { + positive = flb_json_mut_obj(doc); + bucket_counts = flb_json_mut_arr(doc); + if (positive == NULL || bucket_counts == NULL || + !flb_json_mut_obj_add_int(doc, positive, "offset", snapshot.positive_offset)) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + + for (index = 0; index < snapshot.positive_count; index++) { + char buffer[32]; + int length; + + length = snprintf(buffer, sizeof(buffer), "%" PRIu64, + snapshot.positive_buckets[index]); + if (length <= 0 || (size_t) length >= sizeof(buffer) || + !flb_json_mut_arr_add_strncpy(doc, bucket_counts, buffer, length)) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + } + + if (!flb_json_mut_obj_add_val(doc, positive, "bucketCounts", bucket_counts) || + !flb_json_mut_obj_add_val(doc, point, "positive", positive)) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + } + + if (snapshot.negative_count > 0) { + negative = flb_json_mut_obj(doc); + bucket_counts = flb_json_mut_arr(doc); + if (negative == NULL || bucket_counts == NULL || + !flb_json_mut_obj_add_int(doc, negative, "offset", snapshot.negative_offset)) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + + for (index = 0; index < snapshot.negative_count; index++) { + char buffer[32]; + int length; + + length = snprintf(buffer, sizeof(buffer), "%" PRIu64, + snapshot.negative_buckets[index]); + if (length <= 0 || (size_t) length >= sizeof(buffer) || + !flb_json_mut_arr_add_strncpy(doc, bucket_counts, buffer, length)) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + } + + if (!flb_json_mut_obj_add_val(doc, negative, "bucketCounts", bucket_counts) || + !flb_json_mut_obj_add_val(doc, point, "negative", negative)) { + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + return NULL; + } + } + + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + + return point; +} + +static struct flb_json_mut_val *create_metric_json(struct flb_json_mut_doc *doc, + struct cmt *cmt, + struct cmt_map *map) +{ + size_t sample_count; + int temporality; + int monotonism; + struct cfl_list *head; + struct cmt_metric *sample; + struct cmt_counter *counter; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; + struct cmt_summary *summary; + struct flb_json_mut_val *container; + struct flb_json_mut_val *metric; + struct flb_json_mut_val *points; + struct flb_json_mut_val *point; + + sample_count = (map->metric_static_set ? 1 : 0) + cfl_list_size(&map->metrics); + if (sample_count == 0) { + return NULL; + } + + metric = flb_json_mut_obj(doc); + if (metric == NULL || + !flb_json_mut_obj_add_strn(doc, metric, "name", + map->opts->fqname, + cfl_sds_len(map->opts->fqname))) { + return NULL; + } + + if (map->opts->description != NULL && + !flb_json_mut_obj_add_str(doc, metric, "description", + map->opts->description)) { + return NULL; + } + + if (map->unit != NULL && + !flb_json_mut_obj_add_strn(doc, metric, "unit", + map->unit, cfl_sds_len(map->unit))) { + return NULL; + } + + if (add_metric_metadata_json(doc, metric, cmt, map) != 0) { + return NULL; + } + + points = flb_json_mut_arr(doc); + if (points == NULL) { + return NULL; + } + + temporality = 0; + monotonism = CMT_FALSE; + + if (map->type == CMT_COUNTER) { + counter = (struct cmt_counter *) map->parent; + if (counter != NULL) { + temporality = counter->aggregation_type; + monotonism = !counter->allow_reset; + } + + container = flb_json_mut_obj(doc); + if (container == NULL || + !flb_json_mut_obj_add_val(doc, container, "dataPoints", points) || + !flb_json_mut_obj_add_uint(doc, container, "aggregationTemporality", temporality) || + !flb_json_mut_obj_add_bool(doc, container, "isMonotonic", monotonism) || + !flb_json_mut_obj_add_val(doc, metric, "sum", container)) { + return NULL; + } + } + else if (map->type == CMT_GAUGE || map->type == CMT_UNTYPED) { + container = flb_json_mut_obj(doc); + if (container == NULL || + !flb_json_mut_obj_add_val(doc, container, "dataPoints", points) || + !flb_json_mut_obj_add_val(doc, metric, "gauge", container)) { + return NULL; + } + } + else if (map->type == CMT_SUMMARY) { + container = flb_json_mut_obj(doc); + if (container == NULL || + !flb_json_mut_obj_add_val(doc, container, "dataPoints", points) || + !flb_json_mut_obj_add_val(doc, metric, "summary", container)) { + return NULL; + } + } + else if (map->type == CMT_HISTOGRAM) { + histogram = (struct cmt_histogram *) map->parent; + if (histogram != NULL) { + temporality = histogram->aggregation_type; + } + + container = flb_json_mut_obj(doc); + if (container == NULL || + !flb_json_mut_obj_add_val(doc, container, "dataPoints", points) || + !flb_json_mut_obj_add_uint(doc, container, "aggregationTemporality", temporality) || + !flb_json_mut_obj_add_val(doc, metric, "histogram", container)) { + return NULL; + } + } + else if (map->type == CMT_EXP_HISTOGRAM) { + exp_histogram = (struct cmt_exp_histogram *) map->parent; + if (exp_histogram != NULL) { + temporality = exp_histogram->aggregation_type; + } + + container = flb_json_mut_obj(doc); + if (container == NULL || + !flb_json_mut_obj_add_val(doc, container, "dataPoints", points) || + !flb_json_mut_obj_add_uint(doc, container, "aggregationTemporality", temporality) || + !flb_json_mut_obj_add_val(doc, metric, "exponentialHistogram", container)) { + return NULL; + } + } + else { + return NULL; + } + + if (map->metric_static_set) { + if (map->type == CMT_COUNTER || map->type == CMT_GAUGE || map->type == CMT_UNTYPED) { + point = create_number_data_point_json(doc, cmt, map, &map->metric); + } + else if (map->type == CMT_SUMMARY) { + summary = (struct cmt_summary *) map->parent; + point = create_summary_data_point_json(doc, cmt, map, &map->metric, summary); + } + else if (map->type == CMT_HISTOGRAM) { + histogram = (struct cmt_histogram *) map->parent; + point = create_histogram_data_point_json(doc, cmt, map, &map->metric, histogram); + } + else { + point = create_exp_histogram_data_point_json(doc, cmt, map, &map->metric); + } + + if (point == NULL || !flb_json_mut_arr_add_val(points, point)) { + return NULL; + } + } + + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + + if (map->type == CMT_COUNTER || map->type == CMT_GAUGE || map->type == CMT_UNTYPED) { + point = create_number_data_point_json(doc, cmt, map, sample); + } + else if (map->type == CMT_SUMMARY) { + summary = (struct cmt_summary *) map->parent; + point = create_summary_data_point_json(doc, cmt, map, sample, summary); + } + else if (map->type == CMT_HISTOGRAM) { + histogram = (struct cmt_histogram *) map->parent; + point = create_histogram_data_point_json(doc, cmt, map, sample, histogram); + } + else { + point = create_exp_histogram_data_point_json(doc, cmt, map, sample); + } + + if (point == NULL || !flb_json_mut_arr_add_val(points, point)) { + return NULL; + } + } + + return metric; +} + +flb_sds_t flb_opentelemetry_metrics_to_otlp_json(struct cmt *context, + int *result) +{ + size_t index; + size_t resource_count; + size_t resource_index; + size_t scope_index; + size_t total_scope_count; + size_t *scope_counts; + struct cfl_array *resource_metrics_list; + struct cfl_array *scope_metrics_list; + struct cfl_kvlist *resource_entry; + struct cfl_kvlist *resource_metrics_root; + struct cfl_kvlist *resource_root; + struct cfl_kvlist *scope_entry; + struct cfl_kvlist *scope_metrics_root; + struct cfl_kvlist *scope_root; + struct cfl_list *head; + struct cmt_counter *counter; + struct cmt_exp_histogram *exp_histogram; + struct cmt_gauge *gauge; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_untyped *untyped; + struct flb_json_mut_doc *doc; + struct flb_json_mut_val *root; + struct flb_json_mut_val *resource_metrics; + struct flb_json_mut_val *resource_metric; + struct flb_json_mut_val *scope_metrics; + struct flb_json_mut_val *scope_metric; + struct flb_json_mut_val *scope; + struct flb_json_mut_val *resource; + struct flb_json_mut_val *metric; + struct flb_json_mut_val **scope_metric_arrays; + flb_sds_t json; + char *string; + + if (context == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + resource_metrics_list = fetch_metadata_array_key(context->external_metadata, + "resource_metrics_list"); + + resource_count = (resource_metrics_list != NULL && + resource_metrics_list->entry_count > 0) ? + resource_metrics_list->entry_count : 1; + + scope_counts = flb_calloc(resource_count, sizeof(size_t)); + if (scope_counts == NULL) { + flb_errno(); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + total_scope_count = 0; + + for (resource_index = 0; resource_index < resource_count; resource_index++) { + resource_entry = fetch_array_kvlist_entry(resource_metrics_list, resource_index); + if (resource_entry != NULL) { + scope_metrics_list = fetch_metadata_array_key(resource_entry, + "scope_metrics_list"); + } + else { + scope_metrics_list = fetch_metadata_array_key(context->external_metadata, + "scope_metrics_list"); + } + + scope_counts[resource_index] = (scope_metrics_list != NULL && + scope_metrics_list->entry_count > 0) ? + scope_metrics_list->entry_count : 1; + total_scope_count += scope_counts[resource_index]; + } + + doc = flb_json_mut_doc_create(); + root = flb_json_mut_obj(doc); + resource_metrics = flb_json_mut_arr(doc); + scope_metric_arrays = flb_calloc(total_scope_count ? total_scope_count : 1, + sizeof(struct flb_json_mut_val *)); + json = NULL; + + if (doc == NULL || root == NULL || resource_metrics == NULL || + scope_metric_arrays == NULL || + !flb_json_mut_obj_add_val(doc, root, "resourceMetrics", resource_metrics)) { + if (scope_metric_arrays != NULL) { + flb_free(scope_metric_arrays); + } + flb_free(scope_counts); + if (doc != NULL) { + flb_json_mut_doc_destroy(doc); + } + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + flb_json_mut_doc_set_root(doc, root); + + index = 0; + for (resource_index = 0; resource_index < resource_count; resource_index++) { + resource_entry = fetch_array_kvlist_entry(resource_metrics_list, resource_index); + if (resource_entry != NULL) { + resource_metrics_root = fetch_metadata_kvlist_key(resource_entry, "resource_metrics"); + resource_root = fetch_metadata_kvlist_key(resource_entry, "resource"); + scope_metrics_list = fetch_metadata_array_key(resource_entry, "scope_metrics_list"); + scope_metrics_root = fetch_metadata_kvlist_key(resource_entry, "scope_metrics"); + scope_root = fetch_metadata_kvlist_key(resource_entry, "scope"); + } + else { + resource_metrics_root = fetch_metadata_kvlist_key(context->external_metadata, "resource_metrics"); + resource_root = fetch_metadata_kvlist_key(context->external_metadata, "resource"); + scope_metrics_list = fetch_metadata_array_key(context->external_metadata, "scope_metrics_list"); + scope_metrics_root = fetch_metadata_kvlist_key(context->external_metadata, "scope_metrics"); + scope_root = fetch_metadata_kvlist_key(context->external_metadata, "scope"); + } + + resource_metric = flb_json_mut_obj(doc); + scope_metrics = flb_json_mut_arr(doc); + resource = create_metrics_resource_json(doc, resource_root); + + if (resource_metric == NULL || scope_metrics == NULL || resource == NULL || + !flb_json_mut_obj_add_val(doc, resource_metric, "resource", resource) || + !flb_json_mut_obj_add_val(doc, resource_metric, "scopeMetrics", scope_metrics) || + !flb_json_mut_arr_add_val(resource_metrics, resource_metric)) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + if (resource_metrics_root != NULL) { + string = kvlist_fetch_string(fetch_metadata_kvlist_key(resource_metrics_root, "metadata"), + "schema_url"); + if (string != NULL && + !flb_json_mut_obj_add_strn(doc, resource_metric, "schemaUrl", + string, cfl_sds_len(string))) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + for (scope_index = 0; scope_index < scope_counts[resource_index]; scope_index++) { + scope_entry = fetch_array_kvlist_entry(scope_metrics_list, scope_index); + if (scope_entry != NULL) { + scope_metrics_root = fetch_metadata_kvlist_key(scope_entry, "scope_metrics"); + scope_root = fetch_metadata_kvlist_key(scope_entry, "scope"); + } + + scope_metric = flb_json_mut_obj(doc); + scope = create_metrics_scope_json(doc, scope_root); + scope_metric_arrays[index++] = flb_json_mut_arr(doc); + + if (scope_metric == NULL || scope == NULL || scope_metric_arrays[index - 1] == NULL || + !flb_json_mut_obj_add_val(doc, scope_metric, "scope", scope) || + !flb_json_mut_obj_add_val(doc, scope_metric, "metrics", scope_metric_arrays[index - 1]) || + !flb_json_mut_arr_add_val(scope_metrics, scope_metric)) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + if (scope_metrics_root != NULL) { + string = kvlist_fetch_string(fetch_metadata_kvlist_key(scope_metrics_root, "metadata"), + "schema_url"); + if (string != NULL && + !flb_json_mut_obj_add_strn(doc, scope_metric, "schemaUrl", + string, cfl_sds_len(string))) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + } + } + + cfl_list_foreach(head, &context->counters) { + counter = cfl_list_entry(head, struct cmt_counter, _head); + metric = create_metric_json(doc, context, counter->map); + if (metric != NULL && + !flb_json_mut_arr_add_val(scope_metric_arrays[ + resolve_target_scope_index(context, counter->map, + scope_counts, resource_count, + total_scope_count)], metric)) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + cfl_list_foreach(head, &context->gauges) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + metric = create_metric_json(doc, context, gauge->map); + if (metric != NULL && + !flb_json_mut_arr_add_val(scope_metric_arrays[ + resolve_target_scope_index(context, gauge->map, + scope_counts, resource_count, + total_scope_count)], metric)) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + cfl_list_foreach(head, &context->untypeds) { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + metric = create_metric_json(doc, context, untyped->map); + if (metric != NULL && + !flb_json_mut_arr_add_val(scope_metric_arrays[ + resolve_target_scope_index(context, untyped->map, + scope_counts, resource_count, + total_scope_count)], metric)) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + cfl_list_foreach(head, &context->summaries) { + summary = cfl_list_entry(head, struct cmt_summary, _head); + metric = create_metric_json(doc, context, summary->map); + if (metric != NULL && + !flb_json_mut_arr_add_val(scope_metric_arrays[ + resolve_target_scope_index(context, summary->map, + scope_counts, resource_count, + total_scope_count)], metric)) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + cfl_list_foreach(head, &context->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + metric = create_metric_json(doc, context, histogram->map); + if (metric != NULL && + !flb_json_mut_arr_add_val(scope_metric_arrays[ + resolve_target_scope_index(context, histogram->map, + scope_counts, resource_count, + total_scope_count)], metric)) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + cfl_list_foreach(head, &context->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + metric = create_metric_json(doc, context, exp_histogram->map); + if (metric != NULL && + !flb_json_mut_arr_add_val(scope_metric_arrays[ + resolve_target_scope_index(context, exp_histogram->map, + scope_counts, resource_count, + total_scope_count)], metric)) { + flb_free(scope_metric_arrays); + flb_free(scope_counts); + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + flb_free(scope_metric_arrays); + flb_free(scope_counts); + + json = otlp_doc_to_sds(doc); + flb_json_mut_doc_destroy(doc); + + if (json == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + set_result(result, FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS); + + return json; +} + +flb_sds_t flb_opentelemetry_metrics_msgpack_to_otlp_json(const void *data, + size_t size, + int *result) +{ + int ret; + int first_entry; + size_t offset; + flb_sds_t rendered; + flb_sds_t json; + flb_sds_t output; + struct cmt *context; + + if (data == NULL || size == 0) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + rendered = NULL; + json = NULL; + output = flb_sds_create("{\"resourceMetrics\":["); + offset = 0; + first_entry = FLB_TRUE; + + if (output == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + while ((ret = cmt_decode_msgpack_create(&context, (char *) data, size, &offset)) == + CMT_DECODE_MSGPACK_SUCCESS) { + rendered = flb_opentelemetry_metrics_to_otlp_json(context, result); + cmt_destroy(context); + + if (rendered == NULL) { + flb_sds_destroy(output); + return NULL; + } + + if (append_rendered_root_array_content(&output, + &first_entry, + rendered, + "resourceMetrics") != 0) { + flb_sds_destroy(rendered); + flb_sds_destroy(output); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, EINVAL); + return NULL; + } + + flb_sds_destroy(rendered); + } + + if (ret != CMT_DECODE_MSGPACK_INSUFFICIENT_DATA && + ret != CMT_DECODE_MSGPACK_SUCCESS) { + flb_sds_destroy(output); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + json = flb_sds_cat(output, "]}", 2); + if (json == NULL) { + flb_sds_destroy(output); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + set_result(result, FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS); + + return json; +} + +static struct flb_json_mut_val *ctr_attributes_to_kv_array(struct flb_json_mut_doc *doc, + struct ctrace_attributes *attributes) +{ + if (attributes == NULL || attributes->kv == NULL) { + return flb_json_mut_arr(doc); + } + + return cfl_kvlist_to_otlp_kv_array(doc, attributes->kv); +} + +static int add_trace_id_field(struct flb_json_mut_doc *doc, + struct flb_json_mut_val *obj, + const char *key, + struct ctrace_id *id) +{ + cfl_sds_t encoded; + int ret; + + if (id == NULL) { + return 0; + } + + encoded = ctr_id_to_lower_base16(id); + if (encoded == NULL) { + return -1; + } + + ret = flb_json_mut_obj_add_strncpy(doc, obj, key, encoded, cfl_sds_len(encoded)) ? 0 : -1; + cfl_sds_destroy(encoded); + + return ret; +} + +static struct flb_json_mut_val *create_trace_status_json(struct flb_json_mut_doc *doc, + struct ctrace_span_status *status) +{ + const char *code_string; + struct flb_json_mut_val *json; + + if (status == NULL) { + return NULL; + } + + if (status->code == CTRACE_SPAN_STATUS_CODE_UNSET && + !valid_sds_reference(status->message)) { + return NULL; + } + + json = flb_json_mut_obj(doc); + if (json == NULL) { + return NULL; + } + + if (status->code == CTRACE_SPAN_STATUS_CODE_OK) { + code_string = "OK"; + } + else if (status->code == CTRACE_SPAN_STATUS_CODE_ERROR) { + code_string = "ERROR"; + } + else { + code_string = "UNSET"; + } + + if (!flb_json_mut_obj_add_str(doc, json, "code", code_string)) { + return NULL; + } + + if (valid_sds_reference(status->message) && + !flb_json_mut_obj_add_strn(doc, json, "message", + status->message, cfl_sds_len(status->message))) { + return NULL; + } + + return json; +} + +static struct flb_json_mut_val *create_trace_event_json(struct flb_json_mut_doc *doc, + struct ctrace_span_event *event) +{ + struct flb_json_mut_val *attributes; + struct flb_json_mut_val *json; + + json = flb_json_mut_obj(doc); + if (json == NULL || + !valid_sds_reference(event->name) || + !flb_json_mut_obj_add_strn(doc, json, "name", + event->name, cfl_sds_len(event->name))) { + return NULL; + } + + if (event->time_unix_nano > 0 && + json_add_uint64_string(doc, json, "timeUnixNano", + event->time_unix_nano) != 0) { + return NULL; + } + + attributes = ctr_attributes_to_kv_array(doc, event->attr); + if (attributes == NULL) { + return NULL; + } + + if (flb_json_mut_arr_size(attributes) > 0 && + !flb_json_mut_obj_add_val(doc, json, "attributes", attributes)) { + return NULL; + } + + if (event->dropped_attr_count > 0 && + !flb_json_mut_obj_add_uint(doc, json, "droppedAttributesCount", + event->dropped_attr_count)) { + return NULL; + } + + return json; +} + +static struct flb_json_mut_val *create_trace_link_json(struct flb_json_mut_doc *doc, + struct ctrace_link *link) +{ + struct flb_json_mut_val *attributes; + struct flb_json_mut_val *json; + + json = flb_json_mut_obj(doc); + if (json == NULL || + add_trace_id_field(doc, json, "traceId", link->trace_id) != 0 || + add_trace_id_field(doc, json, "spanId", link->span_id) != 0) { + return NULL; + } + + if (valid_sds_reference(link->trace_state) && + !flb_json_mut_obj_add_strn(doc, json, "traceState", + link->trace_state, + cfl_sds_len(link->trace_state))) { + return NULL; + } + + attributes = ctr_attributes_to_kv_array(doc, link->attr); + if (attributes == NULL) { + return NULL; + } + + if (flb_json_mut_arr_size(attributes) > 0 && + !flb_json_mut_obj_add_val(doc, json, "attributes", attributes)) { + return NULL; + } + + if (link->dropped_attr_count > 0 && + !flb_json_mut_obj_add_uint(doc, json, "droppedAttributesCount", + link->dropped_attr_count)) { + return NULL; + } + + if (link->flags > 0 && + !flb_json_mut_obj_add_uint(doc, json, "flags", link->flags)) { + return NULL; + } + + return json; +} + +static struct flb_json_mut_val *create_trace_span_json(struct flb_json_mut_doc *doc, + struct ctrace_span *span) +{ + struct cfl_list *head; + struct flb_json_mut_val *attributes; + struct flb_json_mut_val *events; + struct flb_json_mut_val *json; + struct flb_json_mut_val *links; + struct flb_json_mut_val *status; + struct ctrace_link *link; + struct ctrace_span_event *event; + + json = flb_json_mut_obj(doc); + if (json == NULL || + add_trace_id_field(doc, json, "traceId", span->trace_id) != 0 || + add_trace_id_field(doc, json, "spanId", span->span_id) != 0 || + !valid_sds_reference(span->name) || + !flb_json_mut_obj_add_strn(doc, json, "name", + span->name, cfl_sds_len(span->name))) { + return NULL; + } + + if (add_trace_id_field(doc, json, "parentSpanId", span->parent_span_id) != 0) { + return NULL; + } + + if (valid_sds_reference(span->trace_state) && + !flb_json_mut_obj_add_strn(doc, json, "traceState", + span->trace_state, + cfl_sds_len(span->trace_state))) { + return NULL; + } + + if (span->flags > 0 && + !flb_json_mut_obj_add_uint(doc, json, "flags", span->flags)) { + return NULL; + } + + if (span->kind >= 0 && + !flb_json_mut_obj_add_int(doc, json, "kind", span->kind)) { + return NULL; + } + + if (span->start_time_unix_nano > 0 && + json_add_uint64_string(doc, json, "startTimeUnixNano", + span->start_time_unix_nano) != 0) { + return NULL; + } + + if (span->end_time_unix_nano > 0 && + json_add_uint64_string(doc, json, "endTimeUnixNano", + span->end_time_unix_nano) != 0) { + return NULL; + } + + attributes = ctr_attributes_to_kv_array(doc, span->attr); + if (attributes == NULL) { + return NULL; + } + + if (flb_json_mut_arr_size(attributes) > 0 && + !flb_json_mut_obj_add_val(doc, json, "attributes", attributes)) { + return NULL; + } + + if (span->dropped_attr_count > 0 && + !flb_json_mut_obj_add_uint(doc, json, "droppedAttributesCount", + span->dropped_attr_count)) { + return NULL; + } + + events = flb_json_mut_arr(doc); + if (events == NULL) { + return NULL; + } + + cfl_list_foreach(head, &span->events) { + event = cfl_list_entry(head, struct ctrace_span_event, _head); + status = create_trace_event_json(doc, event); + if (status == NULL || !flb_json_mut_arr_add_val(events, status)) { + return NULL; + } + } + + if (flb_json_mut_arr_size(events) > 0 && + !flb_json_mut_obj_add_val(doc, json, "events", events)) { + return NULL; + } + + if (span->dropped_events_count > 0 && + !flb_json_mut_obj_add_uint(doc, json, "droppedEventsCount", + span->dropped_events_count)) { + return NULL; + } + + links = flb_json_mut_arr(doc); + if (links == NULL) { + return NULL; + } + + cfl_list_foreach(head, &span->links) { + link = cfl_list_entry(head, struct ctrace_link, _head); + status = create_trace_link_json(doc, link); + if (status == NULL || !flb_json_mut_arr_add_val(links, status)) { + return NULL; + } + } + + if (flb_json_mut_arr_size(links) > 0 && + !flb_json_mut_obj_add_val(doc, json, "links", links)) { + return NULL; + } + + if (span->dropped_links_count > 0 && + !flb_json_mut_obj_add_uint(doc, json, "droppedLinksCount", + span->dropped_links_count)) { + return NULL; + } + + status = create_trace_status_json(doc, &span->status); + if (status != NULL && + !flb_json_mut_obj_add_val(doc, json, "status", status)) { + return NULL; + } + + return json; +} + +static struct flb_json_mut_val *create_trace_scope_json(struct flb_json_mut_doc *doc, + struct ctrace_instrumentation_scope *scope) +{ + struct flb_json_mut_val *attributes; + struct flb_json_mut_val *json; + + json = flb_json_mut_obj(doc); + if (json == NULL) { + return NULL; + } + + if (scope == NULL) { + return json; + } + + if (valid_sds_reference(scope->name) && + !flb_json_mut_obj_add_strn(doc, json, "name", + scope->name, cfl_sds_len(scope->name))) { + return NULL; + } + + if (valid_sds_reference(scope->version) && + !flb_json_mut_obj_add_strn(doc, json, "version", + scope->version, cfl_sds_len(scope->version))) { + return NULL; + } + + attributes = ctr_attributes_to_kv_array(doc, scope->attr); + if (attributes == NULL) { + return NULL; + } + + if (flb_json_mut_arr_size(attributes) > 0 && + !flb_json_mut_obj_add_val(doc, json, "attributes", attributes)) { + return NULL; + } + + if (scope->dropped_attr_count > 0 && + !flb_json_mut_obj_add_uint(doc, json, "droppedAttributesCount", + scope->dropped_attr_count)) { + return NULL; + } + + return json; +} + +static struct flb_json_mut_val *create_trace_resource_json(struct flb_json_mut_doc *doc, + struct ctrace_resource *resource) +{ + struct flb_json_mut_val *attributes; + struct flb_json_mut_val *json; + + json = flb_json_mut_obj(doc); + if (json == NULL) { + return NULL; + } + + if (resource == NULL) { + return json; + } + + attributes = ctr_attributes_to_kv_array(doc, resource->attr); + if (attributes == NULL) { + return NULL; + } + + if (flb_json_mut_arr_size(attributes) > 0 && + !flb_json_mut_obj_add_val(doc, json, "attributes", attributes)) { + return NULL; + } + + if (resource->dropped_attr_count > 0 && + !flb_json_mut_obj_add_uint(doc, json, "droppedAttributesCount", + resource->dropped_attr_count)) { + return NULL; + } + + return json; +} + +flb_sds_t flb_opentelemetry_traces_to_otlp_json(struct ctrace *context, + int *result) +{ + struct cfl_list *resource_head; + struct cfl_list *scope_head; + struct cfl_list *span_head; + struct ctrace_resource_span *resource_span; + struct ctrace_scope_span *scope_span; + struct ctrace_span *span; + struct flb_json_mut_doc *doc; + struct flb_json_mut_val *json; + struct flb_json_mut_val *resource; + struct flb_json_mut_val *resource_span_array; + struct flb_json_mut_val *resource_span_json; + struct flb_json_mut_val *scope; + struct flb_json_mut_val *scope_span_array; + struct flb_json_mut_val *scope_span_json; + struct flb_json_mut_val *spans; + flb_sds_t output; + + if (context == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + doc = flb_json_mut_doc_create(); + json = flb_json_mut_obj(doc); + resource_span_array = flb_json_mut_arr(doc); + output = NULL; + + if (doc == NULL || json == NULL || resource_span_array == NULL || + !flb_json_mut_obj_add_val(doc, json, "resourceSpans", resource_span_array)) { + if (doc != NULL) { + flb_json_mut_doc_destroy(doc); + } + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + flb_json_mut_doc_set_root(doc, json); + + cfl_list_foreach(resource_head, &context->resource_spans) { + resource_span = cfl_list_entry(resource_head, struct ctrace_resource_span, _head); + + resource_span_json = flb_json_mut_obj(doc); + resource = create_trace_resource_json(doc, resource_span->resource); + scope_span_array = flb_json_mut_arr(doc); + + if (resource_span_json == NULL || resource == NULL || scope_span_array == NULL || + !flb_json_mut_obj_add_val(doc, resource_span_json, "resource", resource) || + !flb_json_mut_obj_add_val(doc, resource_span_json, "scopeSpans", scope_span_array) || + !flb_json_mut_arr_add_val(resource_span_array, resource_span_json)) { + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + if (valid_sds_reference(resource_span->schema_url) && + !flb_json_mut_obj_add_strn(doc, resource_span_json, "schemaUrl", + resource_span->schema_url, + cfl_sds_len(resource_span->schema_url))) { + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + cfl_list_foreach(scope_head, &resource_span->scope_spans) { + scope_span = cfl_list_entry(scope_head, struct ctrace_scope_span, _head); + scope_span_json = flb_json_mut_obj(doc); + scope = create_trace_scope_json(doc, scope_span->instrumentation_scope); + spans = flb_json_mut_arr(doc); + + if (scope_span_json == NULL || scope == NULL || spans == NULL || + !flb_json_mut_obj_add_val(doc, scope_span_json, "scope", scope) || + !flb_json_mut_obj_add_val(doc, scope_span_json, "spans", spans) || + !flb_json_mut_arr_add_val(scope_span_array, scope_span_json)) { + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + if (valid_sds_reference(scope_span->schema_url) && + !flb_json_mut_obj_add_strn(doc, scope_span_json, "schemaUrl", + scope_span->schema_url, + cfl_sds_len(scope_span->schema_url))) { + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + cfl_list_foreach(span_head, &scope_span->spans) { + span = cfl_list_entry(span_head, struct ctrace_span, _head); + json = create_trace_span_json(doc, span); + if (json == NULL || !flb_json_mut_arr_add_val(spans, json)) { + flb_json_mut_doc_destroy(doc); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + } + } + + output = otlp_doc_to_sds(doc); + flb_json_mut_doc_destroy(doc); + + if (output == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + set_result(result, FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS); + + return output; +} + +flb_sds_t flb_opentelemetry_traces_msgpack_to_otlp_json(const void *data, + size_t size, + int *result) +{ + int ret; + int first_entry; + size_t offset; + flb_sds_t rendered; + flb_sds_t json; + flb_sds_t output; + struct ctrace *context; + + if (data == NULL || size == 0) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + rendered = NULL; + json = NULL; + output = flb_sds_create("{\"resourceSpans\":["); + offset = 0; + first_entry = FLB_TRUE; + + if (output == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + while ((ret = ctr_decode_msgpack_create(&context, (char *) data, size, &offset)) == + CTR_DECODE_MSGPACK_SUCCESS) { + rendered = flb_opentelemetry_traces_to_otlp_json(context, result); + ctr_destroy(context); + + if (rendered == NULL) { + flb_sds_destroy(output); + return NULL; + } + + if (append_rendered_root_array_content(&output, + &first_entry, + rendered, + "resourceSpans") != 0) { + flb_sds_destroy(rendered); + flb_sds_destroy(output); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, EINVAL); + return NULL; + } + + flb_sds_destroy(rendered); + } + + if (ret != CTR_DECODE_MSGPACK_SUCCESS && + !(ret == CTR_MPACK_ENGINE_ERROR && offset >= size)) { + flb_sds_destroy(output); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + json = flb_sds_cat(output, "]}", 2); + if (json == NULL) { + flb_sds_destroy(output); + set_error(result, FLB_OPENTELEMETRY_OTLP_JSON_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + set_result(result, FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS); + + return json; +} diff --git a/src/opentelemetry/flb_opentelemetry_otlp_proto.c b/src/opentelemetry/flb_opentelemetry_otlp_proto.c new file mode 100644 index 00000000000..06fd5efbb27 --- /dev/null +++ b/src/opentelemetry/flb_opentelemetry_otlp_proto.c @@ -0,0 +1,1656 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include + +#define FLB_OTEL_LOGS_SCHEMA_KEY "schema" +#define FLB_OTEL_LOGS_SCHEMA_OTLP "otlp" +#define FLB_OTEL_LOGS_METADATA_KEY "otlp" + +struct otlp_proto_logs_scope_state { + int64_t scope_id; + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; +}; + +struct otlp_proto_logs_resource_state { + int64_t resource_id; + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; + struct otlp_proto_logs_scope_state *scopes; + size_t scope_count; +}; + +static void set_result(int *result, int value) +{ + if (result != NULL) { + *result = value; + } +} + +static void set_error(int *result, int value, int err) +{ + set_result(result, value); + errno = err; +} + +static msgpack_object *msgpack_map_get_object(msgpack_object_map *map, + const char *key) +{ + size_t index; + size_t key_length; + msgpack_object_kv *entry; + + if (map == NULL || key == NULL) { + return NULL; + } + + key_length = strlen(key); + + for (index = 0; index < map->size; index++) { + entry = &map->ptr[index]; + + if (entry->key.type != MSGPACK_OBJECT_STR) { + continue; + } + + if (entry->key.via.str.size != key_length) { + continue; + } + + if (strncmp(entry->key.via.str.ptr, key, key_length) == 0) { + return &entry->val; + } + } + + return NULL; +} + +static int msgpack_map_entry_is_string(msgpack_object_map *map, + const char *key, + const char *expected) +{ + msgpack_object *value; + + value = msgpack_map_get_object(map, key); + if (value == NULL || value->type != MSGPACK_OBJECT_STR) { + return FLB_FALSE; + } + + if (value->via.str.size != strlen(expected)) { + return FLB_FALSE; + } + + if (strncmp(value->via.str.ptr, expected, value->via.str.size) != 0) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +static int msgpack_map_get_int64(msgpack_object_map *map, + const char *key, + int64_t *result) +{ + msgpack_object *value; + + value = msgpack_map_get_object(map, key); + if (value == NULL) { + return -1; + } + + if (value->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *result = (int64_t) value->via.u64; + return 0; + } + + if (value->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *result = value->via.i64; + return 0; + } + + return -1; +} + +static void otlp_kvpair_destroy(Opentelemetry__Proto__Common__V1__KeyValue *kvpair); +static void otlp_any_value_destroy(Opentelemetry__Proto__Common__V1__AnyValue *value); + +static Opentelemetry__Proto__Common__V1__ArrayValue *otlp_array_value_initialize(size_t entry_count) +{ + Opentelemetry__Proto__Common__V1__ArrayValue *value; + + value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__ArrayValue)); + if (value == NULL) { + return NULL; + } + + opentelemetry__proto__common__v1__array_value__init(value); + + if (entry_count > 0) { + value->values = flb_calloc(entry_count, + sizeof(Opentelemetry__Proto__Common__V1__AnyValue *)); + if (value->values == NULL) { + flb_free(value); + return NULL; + } + + value->n_values = entry_count; + } + + return value; +} + +static Opentelemetry__Proto__Common__V1__KeyValue *otlp_kvpair_value_initialize() +{ + Opentelemetry__Proto__Common__V1__KeyValue *value; + + value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__KeyValue)); + if (value != NULL) { + opentelemetry__proto__common__v1__key_value__init(value); + } + + return value; +} + +static Opentelemetry__Proto__Common__V1__KeyValueList *otlp_kvlist_value_initialize(size_t entry_count) +{ + Opentelemetry__Proto__Common__V1__KeyValueList *value; + + value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__KeyValueList)); + if (value == NULL) { + return NULL; + } + + opentelemetry__proto__common__v1__key_value_list__init(value); + + if (entry_count > 0) { + value->values = flb_calloc(entry_count, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); + if (value->values == NULL) { + flb_free(value); + return NULL; + } + + value->n_values = entry_count; + } + + return value; +} + +static Opentelemetry__Proto__Common__V1__AnyValue *otlp_any_value_initialize(int data_type, + size_t entry_count) +{ + Opentelemetry__Proto__Common__V1__AnyValue *value; + + value = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__AnyValue)); + if (value == NULL) { + return NULL; + } + + opentelemetry__proto__common__v1__any_value__init(value); + + if (data_type == MSGPACK_OBJECT_STR) { + value->value_case = + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE; + } + else if (data_type == MSGPACK_OBJECT_NIL) { + value->value_case = + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET; + } + else if (data_type == MSGPACK_OBJECT_BOOLEAN) { + value->value_case = + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE; + } + else if (data_type == MSGPACK_OBJECT_POSITIVE_INTEGER || + data_type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + value->value_case = + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE; + } + else if (data_type == MSGPACK_OBJECT_FLOAT32 || + data_type == MSGPACK_OBJECT_FLOAT64) { + value->value_case = + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE; + } + else if (data_type == MSGPACK_OBJECT_ARRAY) { + value->value_case = + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE; + value->array_value = otlp_array_value_initialize(entry_count); + + if (value->array_value == NULL) { + flb_free(value); + return NULL; + } + } + else if (data_type == MSGPACK_OBJECT_MAP) { + value->value_case = + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE; + value->kvlist_value = otlp_kvlist_value_initialize(entry_count); + + if (value->kvlist_value == NULL) { + flb_free(value); + return NULL; + } + } + else if (data_type == MSGPACK_OBJECT_BIN) { + value->value_case = + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE; + } + else { + flb_free(value); + return NULL; + } + + return value; +} + +static void otlp_kvarray_destroy(Opentelemetry__Proto__Common__V1__KeyValue **kvarray, + size_t entry_count) +{ + size_t index; + + if (kvarray == NULL) { + return; + } + + for (index = 0; index < entry_count; index++) { + if (kvarray[index] != NULL) { + otlp_kvpair_destroy(kvarray[index]); + } + } + + flb_free(kvarray); +} + +static void otlp_kvlist_destroy(Opentelemetry__Proto__Common__V1__KeyValueList *kvlist) +{ + size_t index; + + if (kvlist == NULL) { + return; + } + + for (index = 0; index < kvlist->n_values; index++) { + otlp_kvpair_destroy(kvlist->values[index]); + } + + flb_free(kvlist->values); + flb_free(kvlist); +} + +static void otlp_array_destroy(Opentelemetry__Proto__Common__V1__ArrayValue *array) +{ + size_t index; + + if (array == NULL) { + return; + } + + for (index = 0; index < array->n_values; index++) { + otlp_any_value_destroy(array->values[index]); + } + + flb_free(array->values); + flb_free(array); +} + +static void otlp_any_value_destroy(Opentelemetry__Proto__Common__V1__AnyValue *value) +{ + if (value == NULL) { + return; + } + + if (value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE) { + flb_free(value->string_value); + } + else if (value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE) { + otlp_array_destroy(value->array_value); + } + else if (value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { + otlp_kvlist_destroy(value->kvlist_value); + } + else if (value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE) { + flb_free(value->bytes_value.data); + } + + flb_free(value); +} + +static void otlp_kvpair_destroy(Opentelemetry__Proto__Common__V1__KeyValue *kvpair) +{ + if (kvpair == NULL) { + return; + } + + flb_free(kvpair->key); + otlp_any_value_destroy(kvpair->value); + flb_free(kvpair); +} + +static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_object_to_otlp_any_value( + msgpack_object *object); + +static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_array_to_otlp_any_value( + msgpack_object *object) +{ + size_t index; + Opentelemetry__Proto__Common__V1__AnyValue *entry; + Opentelemetry__Proto__Common__V1__AnyValue *value; + + value = otlp_any_value_initialize(MSGPACK_OBJECT_ARRAY, + object->via.array.size); + if (value == NULL) { + return NULL; + } + + for (index = 0; index < object->via.array.size; index++) { + entry = msgpack_object_to_otlp_any_value(&object->via.array.ptr[index]); + if (entry == NULL) { + otlp_any_value_destroy(value); + return NULL; + } + + value->array_value->values[index] = entry; + } + + return value; +} + +static Opentelemetry__Proto__Common__V1__KeyValue *msgpack_kv_to_otlp_any_value( + struct msgpack_object_kv *input_pair) +{ + Opentelemetry__Proto__Common__V1__KeyValue *kv; + + kv = otlp_kvpair_value_initialize(); + if (kv == NULL) { + return NULL; + } + + kv->key = flb_strndup(input_pair->key.via.str.ptr, input_pair->key.via.str.size); + if (kv->key == NULL) { + flb_free(kv); + return NULL; + } + + kv->value = msgpack_object_to_otlp_any_value(&input_pair->val); + if (kv->value == NULL) { + flb_free(kv->key); + flb_free(kv); + return NULL; + } + + return kv; +} + +static Opentelemetry__Proto__Common__V1__KeyValue **msgpack_map_to_otlp_kvarray( + msgpack_object *object, size_t *entry_count) +{ + size_t index; + Opentelemetry__Proto__Common__V1__KeyValue **result; + + *entry_count = object->via.map.size; + + if (*entry_count == 0) { + return NULL; + } + + result = flb_calloc(*entry_count, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *)); + if (result == NULL) { + *entry_count = 0; + return NULL; + } + + for (index = 0; index < *entry_count; index++) { + result[index] = msgpack_kv_to_otlp_any_value(&object->via.map.ptr[index]); + if (result[index] == NULL) { + otlp_kvarray_destroy(result, index); + *entry_count = 0; + return NULL; + } + } + + return result; +} + +static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_map_to_otlp_any_value( + msgpack_object *object) +{ + size_t index; + Opentelemetry__Proto__Common__V1__KeyValue *entry; + Opentelemetry__Proto__Common__V1__AnyValue *value; + + value = otlp_any_value_initialize(MSGPACK_OBJECT_MAP, object->via.map.size); + if (value == NULL) { + return NULL; + } + + for (index = 0; index < object->via.map.size; index++) { + entry = msgpack_kv_to_otlp_any_value(&object->via.map.ptr[index]); + if (entry == NULL) { + otlp_any_value_destroy(value); + return NULL; + } + + value->kvlist_value->values[index] = entry; + } + + return value; +} + +static Opentelemetry__Proto__Common__V1__AnyValue *msgpack_object_to_otlp_any_value( + msgpack_object *object) +{ + Opentelemetry__Proto__Common__V1__AnyValue *value; + + if (object == NULL) { + return NULL; + } + + value = NULL; + + switch (object->type) { + case MSGPACK_OBJECT_NIL: + value = otlp_any_value_initialize(MSGPACK_OBJECT_NIL, 0); + break; + case MSGPACK_OBJECT_BOOLEAN: + value = otlp_any_value_initialize(MSGPACK_OBJECT_BOOLEAN, 0); + if (value != NULL) { + value->bool_value = object->via.boolean; + } + break; + case MSGPACK_OBJECT_POSITIVE_INTEGER: + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + value = otlp_any_value_initialize(object->type, 0); + if (value != NULL) { + if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + value->int_value = (int64_t) object->via.u64; + } + else { + value->int_value = object->via.i64; + } + } + break; + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + value = otlp_any_value_initialize(object->type, 0); + if (value != NULL) { + value->double_value = object->via.f64; + } + break; + case MSGPACK_OBJECT_STR: + value = otlp_any_value_initialize(MSGPACK_OBJECT_STR, 0); + if (value != NULL) { + value->string_value = flb_strndup(object->via.str.ptr, + object->via.str.size); + if (value->string_value == NULL) { + otlp_any_value_destroy(value); + value = NULL; + } + } + break; + case MSGPACK_OBJECT_BIN: + value = otlp_any_value_initialize(MSGPACK_OBJECT_BIN, 0); + if (value != NULL) { + value->bytes_value.len = object->via.bin.size; + value->bytes_value.data = flb_malloc(object->via.bin.size); + if (value->bytes_value.data == NULL) { + otlp_any_value_destroy(value); + value = NULL; + } + else { + memcpy(value->bytes_value.data, + object->via.bin.ptr, + object->via.bin.size); + } + } + break; + case MSGPACK_OBJECT_ARRAY: + value = msgpack_array_to_otlp_any_value(object); + break; + case MSGPACK_OBJECT_MAP: + value = msgpack_map_to_otlp_any_value(object); + break; + default: + break; + } + + return value; +} + +static struct otlp_proto_logs_resource_state *find_logs_resource_state( + struct otlp_proto_logs_resource_state *states, + size_t state_count, + int64_t resource_id) +{ + size_t index; + + for (index = 0; index < state_count; index++) { + if (states[index].resource_id == resource_id) { + return &states[index]; + } + } + + return NULL; +} + +static struct otlp_proto_logs_scope_state *find_logs_scope_state( + struct otlp_proto_logs_resource_state *resource, + int64_t scope_id) +{ + size_t index; + + for (index = 0; index < resource->scope_count; index++) { + if (resource->scopes[index].scope_id == scope_id) { + return &resource->scopes[index]; + } + } + + return NULL; +} + +static void destroy_logs_resource_states( + struct otlp_proto_logs_resource_state *states, + size_t state_count) +{ + size_t index; + + if (states == NULL) { + return; + } + + for (index = 0; index < state_count; index++) { + flb_free(states[index].scopes); + } + + flb_free(states); +} + +static msgpack_object *find_log_body_candidate(msgpack_object *body, + const char **logs_body_keys, + size_t logs_body_key_count, + const char **matched_key, + size_t *matched_key_length) +{ + size_t index; + msgpack_object *candidate; + + if (body == NULL || body->type == MSGPACK_OBJECT_NIL) { + return NULL; + } + + if (body->type != MSGPACK_OBJECT_MAP) { + return body; + } + + for (index = 0; index < logs_body_key_count; index++) { + if (logs_body_keys[index] == NULL) { + continue; + } + + candidate = msgpack_map_get_object(&body->via.map, logs_body_keys[index]); + if (candidate != NULL) { + if (matched_key != NULL) { + *matched_key = logs_body_keys[index]; + } + if (matched_key_length != NULL) { + *matched_key_length = strlen(logs_body_keys[index]); + } + return candidate; + } + } + + return body; +} + +static int append_kvarrays(Opentelemetry__Proto__Common__V1__KeyValue ***base, + size_t *base_count, + Opentelemetry__Proto__Common__V1__KeyValue **extra, + size_t extra_count) +{ + Opentelemetry__Proto__Common__V1__KeyValue **tmp; + + if (extra == NULL || extra_count == 0) { + return 0; + } + + if (*base == NULL) { + *base = extra; + *base_count = extra_count; + return 0; + } + + tmp = flb_realloc(*base, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *) * + (*base_count + extra_count)); + if (tmp == NULL) { + return -1; + } + + *base = tmp; + memcpy(*base + *base_count, extra, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *) * extra_count); + *base_count += extra_count; + flb_free(extra); + + return 0; +} + +static int msgpack_map_to_otlp_kvarray_filtered( + msgpack_object_map *map, + const char *ignored_key, + size_t ignored_key_length, + Opentelemetry__Proto__Common__V1__KeyValue ***out_values, + size_t *out_count) +{ + size_t index; + size_t count; + Opentelemetry__Proto__Common__V1__KeyValue **tmp; + Opentelemetry__Proto__Common__V1__KeyValue **values; + + values = NULL; + count = 0; + + for (index = 0; index < map->size; index++) { + if (ignored_key != NULL && + map->ptr[index].key.type == MSGPACK_OBJECT_STR && + map->ptr[index].key.via.str.size == ignored_key_length && + strncmp(map->ptr[index].key.via.str.ptr, + ignored_key, + ignored_key_length) == 0) { + continue; + } + + tmp = flb_realloc(values, + sizeof(Opentelemetry__Proto__Common__V1__KeyValue *) * + (count + 1)); + if (tmp == NULL) { + otlp_kvarray_destroy(values, count); + *out_values = NULL; + *out_count = 0; + return -1; + } + values = tmp; + + values[count] = msgpack_kv_to_otlp_any_value(&map->ptr[index]); + if (values[count] == NULL) { + otlp_kvarray_destroy(values, count); + *out_values = NULL; + *out_count = 0; + return -1; + } + + count++; + } + + *out_values = values; + *out_count = count; + + return 0; +} + +static int log_record_set_body_and_attributes( + Opentelemetry__Proto__Logs__V1__LogRecord *record, + struct flb_log_event *event, + const char **logs_body_keys, + size_t logs_body_key_count, + int logs_body_key_attributes) +{ + const char *matched_key; + size_t matched_key_length; + msgpack_object *candidate; + Opentelemetry__Proto__Common__V1__KeyValue **attributes; + size_t attribute_count; + + matched_key = NULL; + matched_key_length = 0; + candidate = find_log_body_candidate(event->body, + logs_body_keys, + logs_body_key_count, + &matched_key, + &matched_key_length); + + record->body = msgpack_object_to_otlp_any_value(candidate); + if (candidate != NULL && record->body == NULL) { + return -1; + } + + if (logs_body_key_attributes == FLB_TRUE && + matched_key != NULL && + event->body != NULL && + event->body->type == MSGPACK_OBJECT_MAP) { + if (msgpack_map_to_otlp_kvarray_filtered(&event->body->via.map, + matched_key, + matched_key_length, + &attributes, + &attribute_count) != 0) { + return -1; + } + + if (append_kvarrays(&record->attributes, + &record->n_attributes, + attributes, + attribute_count) != 0) { + otlp_kvarray_destroy(attributes, attribute_count); + return -1; + } + } + + return 0; +} + +static int add_msgpack_attributes_to_resource( + Opentelemetry__Proto__Resource__V1__Resource *resource, + msgpack_object *resource_object) +{ + msgpack_object *field; + + if (resource_object == NULL || resource_object->type != MSGPACK_OBJECT_MAP) { + return 0; + } + + field = msgpack_map_get_object(&resource_object->via.map, "attributes"); + if (field != NULL && field->type == MSGPACK_OBJECT_MAP) { + resource->attributes = msgpack_map_to_otlp_kvarray(field, + &resource->n_attributes); + if (field->via.map.size > 0 && resource->attributes == NULL) { + return -1; + } + } + + field = msgpack_map_get_object(&resource_object->via.map, + "dropped_attributes_count"); + if (field != NULL) { + if (field->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + resource->dropped_attributes_count = field->via.u64; + } + else if (field->type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + field->via.i64 >= 0) { + resource->dropped_attributes_count = (uint32_t) field->via.i64; + } + } + + return 0; +} + +static int add_msgpack_scope_fields( + Opentelemetry__Proto__Common__V1__InstrumentationScope *scope, + msgpack_object *scope_object) +{ + msgpack_object *field; + + if (scope_object == NULL || scope_object->type != MSGPACK_OBJECT_MAP) { + return 0; + } + + field = msgpack_map_get_object(&scope_object->via.map, "name"); + if (field != NULL && field->type == MSGPACK_OBJECT_STR) { + scope->name = flb_strndup(field->via.str.ptr, field->via.str.size); + if (scope->name == NULL) { + return -1; + } + } + + field = msgpack_map_get_object(&scope_object->via.map, "version"); + if (field != NULL && field->type == MSGPACK_OBJECT_STR) { + scope->version = flb_strndup(field->via.str.ptr, field->via.str.size); + if (scope->version == NULL) { + return -1; + } + } + + field = msgpack_map_get_object(&scope_object->via.map, "attributes"); + if (field != NULL && field->type == MSGPACK_OBJECT_MAP) { + scope->attributes = msgpack_map_to_otlp_kvarray(field, + &scope->n_attributes); + if (field->via.map.size > 0 && scope->attributes == NULL) { + return -1; + } + } + + field = msgpack_map_get_object(&scope_object->via.map, + "dropped_attributes_count"); + if (field != NULL && field->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + scope->dropped_attributes_count = field->via.u64; + } + + return 0; +} + +static struct otlp_proto_logs_resource_state *append_logs_resource_state( + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *export_logs, + struct otlp_proto_logs_resource_state **states, + size_t *state_count, + int64_t resource_id, + msgpack_object *resource_object, + msgpack_object *resource_body) +{ + struct otlp_proto_logs_resource_state *new_states; + struct otlp_proto_logs_resource_state *state; + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; + Opentelemetry__Proto__Resource__V1__Resource *resource; + Opentelemetry__Proto__Logs__V1__ResourceLogs **tmp; + msgpack_object *schema_url; + + resource_log = flb_calloc(1, sizeof(Opentelemetry__Proto__Logs__V1__ResourceLogs)); + resource = flb_calloc(1, sizeof(Opentelemetry__Proto__Resource__V1__Resource)); + if (resource_log == NULL || resource == NULL) { + flb_free(resource_log); + flb_free(resource); + return NULL; + } + + opentelemetry__proto__logs__v1__resource_logs__init(resource_log); + opentelemetry__proto__resource__v1__resource__init(resource); + resource_log->resource = resource; + + if (add_msgpack_attributes_to_resource(resource, resource_object) != 0) { + flb_free(resource); + flb_free(resource_log); + return NULL; + } + + if (resource_body != NULL && resource_body->type == MSGPACK_OBJECT_MAP) { + schema_url = msgpack_map_get_object(&resource_body->via.map, "schema_url"); + if (schema_url != NULL && schema_url->type == MSGPACK_OBJECT_STR) { + resource_log->schema_url = flb_strndup(schema_url->via.str.ptr, + schema_url->via.str.size); + if (resource_log->schema_url == NULL) { + otlp_kvarray_destroy(resource->attributes, resource->n_attributes); + flb_free(resource); + flb_free(resource_log); + return NULL; + } + } + } + + tmp = flb_realloc(export_logs->resource_logs, + sizeof(Opentelemetry__Proto__Logs__V1__ResourceLogs *) * + (export_logs->n_resource_logs + 1)); + if (tmp == NULL) { + otlp_kvarray_destroy(resource->attributes, resource->n_attributes); + flb_free(resource_log->schema_url); + flb_free(resource); + flb_free(resource_log); + return NULL; + } + + export_logs->resource_logs = tmp; + export_logs->resource_logs[export_logs->n_resource_logs++] = resource_log; + + new_states = flb_realloc(*states, + sizeof(struct otlp_proto_logs_resource_state) * + (*state_count + 1)); + if (new_states == NULL) { + return NULL; + } + + *states = new_states; + state = &new_states[*state_count]; + memset(state, 0, sizeof(struct otlp_proto_logs_resource_state)); + + state->resource_id = resource_id; + state->resource_log = resource_log; + (*state_count)++; + + return state; +} + +static struct otlp_proto_logs_scope_state *append_logs_scope_state( + struct otlp_proto_logs_resource_state *resource_state, + int64_t scope_id, + msgpack_object *scope_object) +{ + struct otlp_proto_logs_scope_state *new_scopes; + struct otlp_proto_logs_scope_state *state; + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; + Opentelemetry__Proto__Common__V1__InstrumentationScope *scope; + Opentelemetry__Proto__Logs__V1__ScopeLogs **tmp; + msgpack_object *schema_url; + + scope_log = flb_calloc(1, sizeof(Opentelemetry__Proto__Logs__V1__ScopeLogs)); + scope = flb_calloc(1, sizeof(Opentelemetry__Proto__Common__V1__InstrumentationScope)); + if (scope_log == NULL || scope == NULL) { + flb_free(scope_log); + flb_free(scope); + return NULL; + } + + opentelemetry__proto__logs__v1__scope_logs__init(scope_log); + opentelemetry__proto__common__v1__instrumentation_scope__init(scope); + scope_log->scope = scope; + + if (add_msgpack_scope_fields(scope, scope_object) != 0) { + flb_free(scope->name); + flb_free(scope->version); + otlp_kvarray_destroy(scope->attributes, scope->n_attributes); + flb_free(scope); + flb_free(scope_log); + return NULL; + } + + if (scope_object != NULL && scope_object->type == MSGPACK_OBJECT_MAP) { + schema_url = msgpack_map_get_object(&scope_object->via.map, "schema_url"); + if (schema_url != NULL && schema_url->type == MSGPACK_OBJECT_STR) { + scope_log->schema_url = flb_strndup(schema_url->via.str.ptr, + schema_url->via.str.size); + if (scope_log->schema_url == NULL) { + flb_free(scope->name); + flb_free(scope->version); + otlp_kvarray_destroy(scope->attributes, scope->n_attributes); + flb_free(scope); + flb_free(scope_log); + return NULL; + } + } + } + + tmp = flb_realloc(resource_state->resource_log->scope_logs, + sizeof(Opentelemetry__Proto__Logs__V1__ScopeLogs *) * + (resource_state->resource_log->n_scope_logs + 1)); + if (tmp == NULL) { + return NULL; + } + + resource_state->resource_log->scope_logs = tmp; + resource_state->resource_log->scope_logs[ + resource_state->resource_log->n_scope_logs++] = scope_log; + + new_scopes = flb_realloc(resource_state->scopes, + sizeof(struct otlp_proto_logs_scope_state) * + (resource_state->scope_count + 1)); + if (new_scopes == NULL) { + return NULL; + } + + resource_state->scopes = new_scopes; + state = &new_scopes[resource_state->scope_count]; + memset(state, 0, sizeof(struct otlp_proto_logs_scope_state)); + + state->scope_id = scope_id; + state->scope_log = scope_log; + resource_state->scope_count++; + + return state; +} + +static int ensure_default_logs_scope_state( + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *export_logs, + struct otlp_proto_logs_resource_state **resource_states, + size_t *resource_state_count, + struct otlp_proto_logs_resource_state **current_resource, + struct otlp_proto_logs_scope_state **current_scope) +{ + *current_resource = find_logs_resource_state(*resource_states, + *resource_state_count, + 0); + if (*current_resource == NULL) { + *current_resource = append_logs_resource_state(export_logs, + resource_states, + resource_state_count, + 0, + NULL, + NULL); + if (*current_resource == NULL) { + return -1; + } + } + + *current_scope = find_logs_scope_state(*current_resource, 0); + if (*current_scope == NULL) { + *current_scope = append_logs_scope_state(*current_resource, 0, NULL); + if (*current_scope == NULL) { + return -1; + } + } + + return 0; +} + +static int append_binary_id_field(ProtobufCBinaryData *field, + msgpack_object *value, + size_t expected_size) +{ + if (value == NULL) { + return 0; + } + + if (value->type == MSGPACK_OBJECT_BIN) { + field->data = flb_malloc(value->via.bin.size); + if (field->data == NULL) { + return -1; + } + + memcpy(field->data, value->via.bin.ptr, value->via.bin.size); + field->len = value->via.bin.size; + return 0; + } + + if (value->type != MSGPACK_OBJECT_STR) { + return 0; + } + + if (value->via.str.size != expected_size * 2) { + return 0; + } + + field->data = flb_calloc(1, expected_size); + if (field->data == NULL) { + return -1; + } + + for (size_t i = 0; i < expected_size; i++) { + int high; + int low; + char *str = (char *) value->via.str.ptr; + + if (!isxdigit(str[i * 2]) || !isxdigit(str[i * 2 + 1])) { + flb_free(field->data); + field->data = NULL; + return -1; + } + + high = (str[i * 2] >= 'a') ? str[i * 2] - 'a' + 10 : + (str[i * 2] >= 'A') ? str[i * 2] - 'A' + 10 : + str[i * 2] - '0'; + low = (str[i * 2 + 1] >= 'a') ? str[i * 2 + 1] - 'a' + 10 : + (str[i * 2 + 1] >= 'A') ? str[i * 2 + 1] - 'A' + 10 : + str[i * 2 + 1] - '0'; + + ((unsigned char *) field->data)[i] = (high << 4) | low; + } + + field->len = expected_size; + return 0; +} + +static int log_record_to_proto(Opentelemetry__Proto__Logs__V1__LogRecord *record, + struct flb_log_event *event, + const char **logs_body_keys, + size_t logs_body_key_count, + int logs_body_key_attributes) +{ + msgpack_object *metadata; + msgpack_object *otlp_metadata; + msgpack_object *field; + uint64_t timestamp; + size_t count; + Opentelemetry__Proto__Common__V1__KeyValue **attrs; + + metadata = event->metadata; + otlp_metadata = NULL; + + if (metadata != NULL && metadata->type == MSGPACK_OBJECT_MAP) { + otlp_metadata = msgpack_map_get_object(&metadata->via.map, + FLB_OTEL_LOGS_METADATA_KEY); + } + + if (otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP) { + field = msgpack_map_get_object(&otlp_metadata->via.map, "timestamp"); + if (field != NULL && field->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + timestamp = field->via.u64; + } + else if (event->raw_timestamp != NULL && + event->raw_timestamp->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + timestamp = event->raw_timestamp->via.u64; + } + else { + timestamp = flb_time_to_nanosec(&event->timestamp); + } + } + else if (event->raw_timestamp != NULL && + event->raw_timestamp->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + timestamp = event->raw_timestamp->via.u64; + } + else { + timestamp = flb_time_to_nanosec(&event->timestamp); + } + + record->time_unix_nano = timestamp; + + if (otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP) { + field = msgpack_map_get_object(&otlp_metadata->via.map, + "observed_timestamp"); + if (field != NULL && field->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + record->observed_time_unix_nano = field->via.u64; + } + + field = msgpack_map_get_object(&otlp_metadata->via.map, + "severity_number"); + if (field != NULL && field->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + record->severity_number = field->via.u64; + } + + field = msgpack_map_get_object(&otlp_metadata->via.map, + "severity_text"); + if (field != NULL && field->type == MSGPACK_OBJECT_STR) { + record->severity_text = flb_strndup(field->via.str.ptr, + field->via.str.size); + if (record->severity_text == NULL) { + return -1; + } + } + + field = msgpack_map_get_object(&otlp_metadata->via.map, "attributes"); + if (field != NULL && field->type == MSGPACK_OBJECT_MAP) { + attrs = msgpack_map_to_otlp_kvarray(field, &count); + if (field->via.map.size > 0 && attrs == NULL) { + return -1; + } + + if (append_kvarrays(&record->attributes, + &record->n_attributes, + attrs, + count) != 0) { + otlp_kvarray_destroy(attrs, count); + return -1; + } + } + + if (append_binary_id_field(&record->trace_id, + msgpack_map_get_object(&otlp_metadata->via.map, + "trace_id"), + 16) != 0) { + return -1; + } + + if (append_binary_id_field(&record->span_id, + msgpack_map_get_object(&otlp_metadata->via.map, + "span_id"), + 8) != 0) { + return -1; + } + } + + if (log_record_set_body_and_attributes(record, + event, + logs_body_keys, + logs_body_key_count, + logs_body_key_attributes) != 0) { + return -1; + } + + return 0; +} + +static void destroy_log_record(Opentelemetry__Proto__Logs__V1__LogRecord *record) +{ + if (record == NULL) { + return; + } + + otlp_any_value_destroy(record->body); + otlp_kvarray_destroy(record->attributes, record->n_attributes); + if (record->severity_text != NULL && + record->severity_text != protobuf_c_empty_string) { + flb_free(record->severity_text); + } + flb_free(record->span_id.data); + flb_free(record->trace_id.data); + flb_free(record); +} + +static void destroy_export_logs( + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *export_logs) +{ + size_t index; + size_t inner; + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; + + if (export_logs == NULL) { + return; + } + + for (index = 0; index < export_logs->n_resource_logs; index++) { + resource_log = export_logs->resource_logs[index]; + if (resource_log == NULL) { + continue; + } + + for (inner = 0; inner < resource_log->n_scope_logs; inner++) { + scope_log = resource_log->scope_logs[inner]; + if (scope_log == NULL) { + continue; + } + + for (size_t record_index = 0; + record_index < scope_log->n_log_records; + record_index++) { + destroy_log_record(scope_log->log_records[record_index]); + } + + flb_free(scope_log->log_records); + if (scope_log->scope != NULL) { + if (scope_log->scope->name != NULL && + scope_log->scope->name != protobuf_c_empty_string) { + flb_free(scope_log->scope->name); + } + if (scope_log->scope->version != NULL && + scope_log->scope->version != protobuf_c_empty_string) { + flb_free(scope_log->scope->version); + } + otlp_kvarray_destroy(scope_log->scope->attributes, + scope_log->scope->n_attributes); + flb_free(scope_log->scope); + } + if (scope_log->schema_url != NULL && + scope_log->schema_url != protobuf_c_empty_string) { + flb_free(scope_log->schema_url); + } + flb_free(scope_log); + } + + flb_free(resource_log->scope_logs); + if (resource_log->resource != NULL) { + otlp_kvarray_destroy(resource_log->resource->attributes, + resource_log->resource->n_attributes); + flb_free(resource_log->resource); + } + if (resource_log->schema_url != NULL && + resource_log->schema_url != protobuf_c_empty_string) { + flb_free(resource_log->schema_url); + } + flb_free(resource_log); + } + + flb_free(export_logs->resource_logs); +} + +flb_sds_t flb_opentelemetry_metrics_to_otlp_proto(struct cmt *context, + int *result) +{ + cfl_sds_t encoded; + + if (context == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + encoded = cmt_encode_opentelemetry_create(context); + if (encoded == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + set_result(result, FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); + return (flb_sds_t) encoded; +} + +flb_sds_t flb_opentelemetry_metrics_msgpack_to_otlp_proto(const void *data, + size_t size, + int *result) +{ + int ret; + size_t offset; + size_t payload_length; + cfl_sds_t output; + cfl_sds_t encoded; + struct cmt *context; + + if (data == NULL || size == 0) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + output = NULL; + offset = 0; + + while ((ret = cmt_decode_msgpack_create(&context, + (char *) data, + size, + &offset)) == CMT_DECODE_MSGPACK_SUCCESS) { + encoded = cmt_encode_opentelemetry_create(context); + if (encoded == NULL) { + cmt_destroy(context); + if (output != NULL) { + cfl_sds_destroy(output); + } + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + payload_length = cfl_sds_len(encoded); + + if (output == NULL) { + output = cfl_sds_create_len(encoded, payload_length); + } + else { + output = cfl_sds_cat(output, encoded, payload_length); + } + + cmt_encode_opentelemetry_destroy(encoded); + cmt_destroy(context); + + if (output == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + if (ret != CMT_DECODE_MSGPACK_INSUFFICIENT_DATA && + ret != CMT_DECODE_MSGPACK_SUCCESS) { + if (output != NULL) { + cfl_sds_destroy(output); + } + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + if (output == NULL) { + output = cfl_sds_create_size(0); + if (output == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + set_result(result, FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); + return (flb_sds_t) output; +} + +flb_sds_t flb_opentelemetry_traces_to_otlp_proto(struct ctrace *context, + int *result) +{ + cfl_sds_t encoded; + + if (context == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + encoded = ctr_encode_opentelemetry_create(context); + if (encoded == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + set_result(result, FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); + return (flb_sds_t) encoded; +} + +flb_sds_t flb_opentelemetry_logs_to_otlp_proto(const void *event_chunk_data, + size_t event_chunk_size, + struct flb_opentelemetry_otlp_json_options *options, + int *result) +{ + int ret; + int require_otel_metadata; + int logs_body_key_attributes; + int record_type; + int64_t resource_id; + int64_t scope_id; + size_t logs_body_key_count; + flb_sds_t output; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + msgpack_object *group_metadata; + msgpack_object *group_body; + msgpack_object *resource_object; + msgpack_object *scope_object; + struct otlp_proto_logs_scope_state *current_scope; + struct otlp_proto_logs_resource_state *current_resource; + struct otlp_proto_logs_resource_state *resource_states; + size_t resource_state_count; + const char **logs_body_keys; + const char *logs_body_key; + static const char *default_logs_body_keys[] = {"log", "message"}; + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest export_logs; + + require_otel_metadata = FLB_FALSE; + logs_body_key = "log"; + logs_body_keys = default_logs_body_keys; + logs_body_key_count = 2; + logs_body_key_attributes = FLB_FALSE; + + if (options != NULL) { + require_otel_metadata = options->logs_require_otel_metadata; + logs_body_key_attributes = options->logs_body_key_attributes; + + if (options->logs_body_keys != NULL && + options->logs_body_key_count > 0) { + logs_body_keys = options->logs_body_keys; + logs_body_key_count = options->logs_body_key_count; + } + else if (options->logs_body_key != NULL) { + logs_body_key = options->logs_body_key; + logs_body_keys = &logs_body_key; + logs_body_key_count = 1; + } + } + + opentelemetry__proto__collector__logs__v1__export_logs_service_request__init( + &export_logs); + + current_scope = NULL; + current_resource = NULL; + resource_states = NULL; + resource_state_count = 0; + + ret = flb_log_event_decoder_init(&decoder, + (char *) event_chunk_data, + event_chunk_size); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_ARGUMENT, EINVAL); + return NULL; + } + + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + + while ((ret = flb_log_event_decoder_next(&decoder, &event)) == + FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &record_type); + if (ret != 0) { + flb_log_event_decoder_destroy(&decoder); + destroy_export_logs(&export_logs); + destroy_logs_resource_states(resource_states, resource_state_count); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_LOG_EVENT, EINVAL); + return NULL; + } + + if (record_type == FLB_LOG_EVENT_GROUP_START) { + group_metadata = event.group_metadata != NULL ? event.group_metadata : event.metadata; + group_body = event.body; + + if (group_metadata == NULL || + group_metadata->type != MSGPACK_OBJECT_MAP || + msgpack_map_entry_is_string(&group_metadata->via.map, + FLB_OTEL_LOGS_SCHEMA_KEY, + FLB_OTEL_LOGS_SCHEMA_OTLP) != FLB_TRUE || + msgpack_map_get_int64(&group_metadata->via.map, "resource_id", &resource_id) != 0 || + msgpack_map_get_int64(&group_metadata->via.map, "scope_id", &scope_id) != 0) { + if (require_otel_metadata == FLB_TRUE) { + flb_log_event_decoder_destroy(&decoder); + destroy_export_logs(&export_logs); + destroy_logs_resource_states(resource_states, resource_state_count); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_LOG_EVENT, EINVAL); + return NULL; + } + + current_resource = NULL; + current_scope = NULL; + continue; + } + + resource_object = NULL; + scope_object = NULL; + + if (group_body != NULL && group_body->type == MSGPACK_OBJECT_MAP) { + resource_object = msgpack_map_get_object(&group_body->via.map, "resource"); + scope_object = msgpack_map_get_object(&group_body->via.map, "scope"); + } + + current_resource = find_logs_resource_state(resource_states, + resource_state_count, + resource_id); + if (current_resource == NULL) { + current_resource = append_logs_resource_state(&export_logs, + &resource_states, + &resource_state_count, + resource_id, + resource_object, + group_body); + if (current_resource == NULL) { + flb_log_event_decoder_destroy(&decoder); + destroy_export_logs(&export_logs); + destroy_logs_resource_states(resource_states, resource_state_count); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + current_scope = find_logs_scope_state(current_resource, scope_id); + if (current_scope == NULL) { + current_scope = append_logs_scope_state(current_resource, + scope_id, + scope_object); + if (current_scope == NULL) { + flb_log_event_decoder_destroy(&decoder); + destroy_export_logs(&export_logs); + destroy_logs_resource_states(resource_states, resource_state_count); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + continue; + } + else if (record_type == FLB_LOG_EVENT_GROUP_END) { + current_resource = NULL; + current_scope = NULL; + continue; + } + + if (current_scope == NULL) { + if (require_otel_metadata == FLB_TRUE) { + flb_log_event_decoder_destroy(&decoder); + destroy_export_logs(&export_logs); + destroy_logs_resource_states(resource_states, resource_state_count); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_LOG_EVENT, EINVAL); + return NULL; + } + + if (ensure_default_logs_scope_state(&export_logs, + &resource_states, + &resource_state_count, + ¤t_resource, + ¤t_scope) != 0) { + flb_log_event_decoder_destroy(&decoder); + destroy_export_logs(&export_logs); + destroy_logs_resource_states(resource_states, resource_state_count); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + } + + Opentelemetry__Proto__Logs__V1__LogRecord **tmp; + + tmp = flb_realloc(current_scope->scope_log->log_records, + sizeof(Opentelemetry__Proto__Logs__V1__LogRecord *) * + (current_scope->scope_log->n_log_records + 1)); + if (tmp == NULL) { + flb_log_event_decoder_destroy(&decoder); + destroy_export_logs(&export_logs); + destroy_logs_resource_states(resource_states, resource_state_count); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + current_scope->scope_log->log_records = tmp; + + current_scope->scope_log->log_records[ + current_scope->scope_log->n_log_records] = + flb_calloc(1, sizeof(Opentelemetry__Proto__Logs__V1__LogRecord)); + if (current_scope->scope_log->log_records[ + current_scope->scope_log->n_log_records] == NULL) { + flb_log_event_decoder_destroy(&decoder); + destroy_export_logs(&export_logs); + destroy_logs_resource_states(resource_states, resource_state_count); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + opentelemetry__proto__logs__v1__log_record__init( + current_scope->scope_log->log_records[ + current_scope->scope_log->n_log_records]); + + if (log_record_to_proto( + current_scope->scope_log->log_records[ + current_scope->scope_log->n_log_records], + &event, + logs_body_keys, + logs_body_key_count, + logs_body_key_attributes) != 0) { + flb_log_event_decoder_destroy(&decoder); + destroy_export_logs(&export_logs); + destroy_logs_resource_states(resource_states, resource_state_count); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + current_scope->scope_log->n_log_records++; + } + + flb_log_event_decoder_destroy(&decoder); + destroy_logs_resource_states(resource_states, resource_state_count); + + if (ret != FLB_EVENT_DECODER_SUCCESS && + ret != FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA) { + destroy_export_logs(&export_logs); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_LOG_EVENT, EINVAL); + return NULL; + } + + output = flb_sds_create_size( + opentelemetry__proto__collector__logs__v1__export_logs_service_request__get_packed_size( + &export_logs)); + if (output == NULL) { + destroy_export_logs(&export_logs); + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + + cfl_sds_set_len((cfl_sds_t) output, + opentelemetry__proto__collector__logs__v1__export_logs_service_request__pack( + &export_logs, (uint8_t *) output)); + + destroy_export_logs(&export_logs); + set_result(result, FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); + + return output; +} From 1a12b8f866bdea0a692ff61f68b0dbb677dd556b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 03:00:05 -0600 Subject: [PATCH 02/29] pack: add otlp_json and otlp_proto formats Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_pack.h | 6 ++++-- src/flb_pack.c | 27 ++++++++++++--------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/include/fluent-bit/flb_pack.h b/include/fluent-bit/flb_pack.h index ecce24ab1f2..aba8f401434 100644 --- a/include/fluent-bit/flb_pack.h +++ b/include/fluent-bit/flb_pack.h @@ -27,8 +27,6 @@ #include #include -#include - /* JSON types */ #define FLB_PACK_JSON_UNDEFINED JSMN_UNDEFINED #define FLB_PACK_JSON_OBJECT JSMN_OBJECT @@ -60,6 +58,8 @@ #define FLB_PACK_JSON_FORMAT_JSON 1 #define FLB_PACK_JSON_FORMAT_STREAM 2 #define FLB_PACK_JSON_FORMAT_LINES 3 +#define FLB_PACK_JSON_FORMAT_OTLP 4 +#define FLB_PACK_JSON_FORMAT_OTLP_PRETTY 5 struct flb_pack_state { int multiple; /* support multiple jsons? */ @@ -81,11 +81,13 @@ int flb_pack_json(const char *js, size_t len, char **buffer, size_t *size, int *root_type, size_t *consumed); int flb_pack_json_recs(const char *js, size_t len, char **buffer, size_t *size, int *root_type, int *out_records, size_t *consumed); +#ifdef FLB_HAVE_YYJSON int flb_pack_json_yyjson(const char *js, size_t len, char **buffer, size_t *size, int *root_type, size_t *consumed); int flb_pack_json_recs_yyjson(const char *js, size_t len, char **buffer, size_t *size, int *root_type, int *out_records, size_t *consumed); +#endif int flb_pack_state_init(struct flb_pack_state *s); void flb_pack_state_reset(struct flb_pack_state *s); diff --git a/src/flb_pack.c b/src/flb_pack.c index 6c40e3b82f2..28594b4f8d6 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -45,7 +45,9 @@ #include #include #include +#ifdef FLB_HAVE_YYJSON #include +#endif #define try_to_write_str flb_utils_write_str @@ -311,6 +313,7 @@ static inline int pack_string_token(struct flb_pack_state *state, } /* Convert a yyjson value to msgpack */ +#ifdef FLB_HAVE_YYJSON static void yyjson_val_to_msgpack(yyjson_val *val, msgpack_packer *pck) { size_t idx, max; @@ -503,6 +506,7 @@ static int pack_json_to_msgpack_yyjson(const char *js, size_t len, char **buffer flb_free(insitu_buf); return 0; } +#endif /* Receive a tokenized JSON message and convert it to MsgPack */ static char *tokens_to_msgpack(struct flb_pack_state *state, @@ -668,18 +672,8 @@ int flb_pack_json(const char *js, size_t len, char **buffer, size_t *size, { int records; -#ifdef FLB_HAVE_SIMD - /* - * When SIMD support is compiled in, route the default JSON pack API through - * the extensible frontend so callers inherit the YYJSON backend selection. - * Explicit backend-specific entry points remain available for forced JSMN - * or YYJSON behavior. - */ return flb_pack_json_recs_ext(js, len, buffer, size, root_type, &records, consumed, NULL); -#endif - - return flb_pack_json_legacy(js, len, buffer, size, root_type, consumed); } /* @@ -689,16 +683,12 @@ int flb_pack_json(const char *js, size_t len, char **buffer, size_t *size, int flb_pack_json_recs(const char *js, size_t len, char **buffer, size_t *size, int *root_type, int *out_records, size_t *consumed) { -#ifdef FLB_HAVE_SIMD return flb_pack_json_recs_ext(js, len, buffer, size, root_type, out_records, consumed, NULL); -#endif - - return flb_pack_json_recs_legacy(js, len, buffer, size, root_type, - out_records, consumed); } /* Pack a JSON message using yyjson */ +#ifdef FLB_HAVE_YYJSON int flb_pack_json_yyjson(const char *js, size_t len, char **buffer, size_t *size, int *root_type, size_t *consumed) { @@ -712,6 +702,7 @@ int flb_pack_json_recs_yyjson(const char *js, size_t len, char **buffer, size_t { return pack_json_to_msgpack_yyjson(js, len, buffer, size, root_type, out_records, consumed); } +#endif /* Initialize a JSON packer state */ int flb_pack_state_init(struct flb_pack_state *s) @@ -1257,6 +1248,12 @@ int flb_pack_to_json_format_type(const char *str) else if (strcasecmp(str, "json_lines") == 0) { return FLB_PACK_JSON_FORMAT_LINES; } + else if (strcasecmp(str, "otlp_json") == 0) { + return FLB_PACK_JSON_FORMAT_OTLP; + } + else if (strcasecmp(str, "otlp_json_pretty") == 0) { + return FLB_PACK_JSON_FORMAT_OTLP_PRETTY; + } return -1; } From 4d1fb127ae68feb7cefe0e18d1eaea0cc98c0317 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 03:02:48 -0600 Subject: [PATCH 03/29] pack_json: use pre-processor conditionals for JSON backend Signed-off-by: Eduardo Silva --- src/flb_pack_json.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/flb_pack_json.c b/src/flb_pack_json.c index f4085415122..3c3eeb1ec79 100644 --- a/src/flb_pack_json.c +++ b/src/flb_pack_json.c @@ -38,7 +38,11 @@ static int flb_pack_json_ext_internal(const char *json, size_t len, struct flb_pack_state *state = NULL; if (!opts) { +#ifdef FLB_HAVE_YYJSON backend = FLB_PACK_JSON_BACKEND_YYJSON; +#else + backend = FLB_PACK_JSON_BACKEND_JSMN; +#endif } else { backend = opts->backend; @@ -77,6 +81,7 @@ static int flb_pack_json_ext_internal(const char *json, size_t len, out_root_type, NULL); } else if (backend == FLB_PACK_JSON_BACKEND_YYJSON) { +#ifdef FLB_HAVE_YYJSON if (require_records) { return flb_pack_json_recs_yyjson(json, len, out_buf, out_size, out_root_type, out_records, @@ -85,6 +90,9 @@ static int flb_pack_json_ext_internal(const char *json, size_t len, return flb_pack_json_yyjson(json, len, out_buf, out_size, out_root_type, NULL); +#else + return -1; +#endif } /* unknown backend */ From ceddbcb63be2c2707b21afd90bad8317174b2b4c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 03:28:20 -0600 Subject: [PATCH 04/29] json: new JSON API writer with SIMD and fallback backends Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_json.h | 106 ++ src/flb_json.c | 1699 +++++++++++++++++++++++++++++++++ 2 files changed, 1805 insertions(+) create mode 100644 include/fluent-bit/flb_json.h create mode 100644 src/flb_json.c diff --git a/include/fluent-bit/flb_json.h b/include/fluent-bit/flb_json.h new file mode 100644 index 00000000000..094dfeabe0a --- /dev/null +++ b/include/fluent-bit/flb_json.h @@ -0,0 +1,106 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_JSON_H +#define FLB_JSON_H + +#include + +struct flb_json_doc; +struct flb_json_val; +struct flb_json_mut_doc; +struct flb_json_mut_val; + +struct flb_json_doc *flb_json_read(const char *input, size_t length); +void flb_json_doc_destroy(struct flb_json_doc *document); +struct flb_json_val *flb_json_doc_get_root(struct flb_json_doc *document); +char *flb_json_write(struct flb_json_doc *document, size_t *length); +char *flb_json_write_pretty(struct flb_json_doc *document, size_t *length); +char *flb_json_prettify(const char *input, size_t input_length, size_t *length); +struct flb_json_val *flb_json_obj_get(struct flb_json_val *value, + const char *key); +size_t flb_json_arr_size(struct flb_json_val *value); +struct flb_json_val *flb_json_arr_get(struct flb_json_val *value, size_t index); + +struct flb_json_mut_doc *flb_json_mut_doc_create(void); +void flb_json_mut_doc_destroy(struct flb_json_mut_doc *document); +void flb_json_mut_doc_set_root(struct flb_json_mut_doc *document, + struct flb_json_mut_val *root); +char *flb_json_mut_write(struct flb_json_mut_doc *document, size_t *length); +char *flb_json_mut_write_pretty(struct flb_json_mut_doc *document, size_t *length); + +struct flb_json_mut_val *flb_json_mut_obj(struct flb_json_mut_doc *document); +struct flb_json_mut_val *flb_json_mut_arr(struct flb_json_mut_doc *document); +struct flb_json_mut_val *flb_json_mut_strncpy(struct flb_json_mut_doc *document, + const char *value, + size_t length); +struct flb_json_mut_val *flb_json_val_mut_copy(struct flb_json_mut_doc *target, + struct flb_json_val *source); + +int flb_json_mut_arr_add_real(struct flb_json_mut_doc *document, + struct flb_json_mut_val *array, + double value); +int flb_json_mut_arr_add_strncpy(struct flb_json_mut_doc *document, + struct flb_json_mut_val *array, + const char *value, + size_t length); +int flb_json_mut_arr_add_val(struct flb_json_mut_val *array, + struct flb_json_mut_val *value); +size_t flb_json_mut_arr_size(struct flb_json_mut_val *array); + +int flb_json_mut_obj_add_bool(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + int value); +int flb_json_mut_obj_add_int(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + long long value); +int flb_json_mut_obj_add_real(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + double value); +int flb_json_mut_obj_add_str(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + const char *value); +int flb_json_mut_obj_add_strcpy(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + const char *value); +int flb_json_mut_obj_add_strn(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + const char *value, + size_t length); +int flb_json_mut_obj_add_strncpy(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + const char *value, + size_t length); +int flb_json_mut_obj_add_uint(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + unsigned long long value); +int flb_json_mut_obj_add_val(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + struct flb_json_mut_val *value); + +#endif diff --git a/src/flb_json.c b/src/flb_json.c new file mode 100644 index 00000000000..dd8c8553a7a --- /dev/null +++ b/src/flb_json.c @@ -0,0 +1,1699 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include +#include +#include +#include +#ifdef FLB_HAVE_YYJSON +#include +#endif + +enum flb_json_mut_type { + FLB_JSON_MUT_OBJECT, + FLB_JSON_MUT_ARRAY, + FLB_JSON_MUT_STRING, + FLB_JSON_MUT_BOOL, + FLB_JSON_MUT_INT, + FLB_JSON_MUT_UINT, + FLB_JSON_MUT_REAL, + FLB_JSON_MUT_NULL +}; + +struct flb_json_doc; +struct flb_json_mut_doc; + +struct flb_json_val { + struct flb_json_doc *owner; + msgpack_object *object; + struct flb_json_val *next; +}; + +struct flb_json_doc { + char *buffer; + size_t buffer_size; + msgpack_unpacked unpacked; + struct flb_json_val root; + struct flb_json_val *wrappers; +}; + +struct flb_json_mut_kv { + flb_sds_t key; + struct flb_json_mut_val *value; + struct flb_json_mut_kv *next; + struct flb_json_mut_kv *alloc_next; +}; + +struct flb_json_mut_entry { + struct flb_json_mut_val *value; + struct flb_json_mut_entry *next; + struct flb_json_mut_entry *alloc_next; +}; + +struct flb_json_mut_val { + enum flb_json_mut_type type; + struct flb_json_mut_doc *owner; + + union { + struct { + struct flb_json_mut_kv *head; + struct flb_json_mut_kv *tail; + } object; + + struct { + struct flb_json_mut_entry *head; + struct flb_json_mut_entry *tail; + size_t count; + } array; + + flb_sds_t string; + int boolean; + long long sint; + unsigned long long uint; + double real; + } data; + + struct flb_json_mut_val *next; +}; + +struct flb_json_mut_doc { + struct flb_json_mut_val *root; + struct flb_json_mut_val *values; + struct flb_json_mut_kv *kvs; + struct flb_json_mut_entry *entries; +}; + +#ifdef FLB_HAVE_YYJSON +static void *flb_json_yyjson_malloc(void *ctx, size_t size) +{ + (void) ctx; + + return flb_malloc(size); +} + +static void *flb_json_yyjson_realloc(void *ctx, void *ptr, + size_t old_size, size_t size) +{ + (void) ctx; + (void) old_size; + + return flb_realloc(ptr, size); +} + +static void flb_json_yyjson_free(void *ctx, void *ptr) +{ + (void) ctx; + + flb_free(ptr); +} + +static void flb_json_yyjson_init_alc(yyjson_alc *allocator) +{ + allocator->malloc = flb_json_yyjson_malloc; + allocator->realloc = flb_json_yyjson_realloc; + allocator->free = flb_json_yyjson_free; + allocator->ctx = NULL; +} +#endif + +static struct flb_json_val *parsed_value_wrap(struct flb_json_doc *doc, + msgpack_object *object) +{ + struct flb_json_val *wrapper; + + wrapper = flb_calloc(1, sizeof(struct flb_json_val)); + if (wrapper == NULL) { + flb_errno(); + return NULL; + } + + wrapper->owner = doc; + wrapper->object = object; + wrapper->next = doc->wrappers; + doc->wrappers = wrapper; + + return wrapper; +} + +static int json_append_indent(flb_sds_t *buffer, size_t depth, size_t spaces) +{ + size_t index; + + for (index = 0; index < depth * spaces; index++) { + *buffer = flb_sds_cat(*buffer, " ", 1); + if (*buffer == NULL) { + return -1; + } + } + + return 0; +} + +static int json_append_escaped_string(flb_sds_t *buffer, + const char *value, + size_t length) +{ + static const char hex[] = "0123456789abcdef"; + size_t index; + char escaped[6]; + unsigned char c; + + *buffer = flb_sds_cat(*buffer, "\"", 1); + if (*buffer == NULL) { + return -1; + } + + for (index = 0; index < length; index++) { + c = (unsigned char) value[index]; + + if (c == '"' || c == '\\') { + escaped[0] = '\\'; + escaped[1] = (char) c; + + *buffer = flb_sds_cat(*buffer, escaped, 2); + } + else if (c == '\b') { + *buffer = flb_sds_cat(*buffer, "\\b", 2); + } + else if (c == '\f') { + *buffer = flb_sds_cat(*buffer, "\\f", 2); + } + else if (c == '\n') { + *buffer = flb_sds_cat(*buffer, "\\n", 2); + } + else if (c == '\r') { + *buffer = flb_sds_cat(*buffer, "\\r", 2); + } + else if (c == '\t') { + *buffer = flb_sds_cat(*buffer, "\\t", 2); + } + else if (c < 0x20) { + escaped[0] = '\\'; + escaped[1] = 'u'; + escaped[2] = '0'; + escaped[3] = '0'; + escaped[4] = hex[(c >> 4) & 0x0f]; + escaped[5] = hex[c & 0x0f]; + + *buffer = flb_sds_cat(*buffer, escaped, sizeof(escaped)); + } + else { + *buffer = flb_sds_cat(*buffer, (const char *) &value[index], 1); + } + + if (*buffer == NULL) { + return -1; + } + } + + *buffer = flb_sds_cat(*buffer, "\"", 1); + if (*buffer == NULL) { + return -1; + } + + return 0; +} + +static int render_msgpack_object(flb_sds_t *buffer, + msgpack_object *object, + int pretty, + size_t depth); + +static int render_msgpack_map(flb_sds_t *buffer, + msgpack_object_map *map, + int pretty, + size_t depth) +{ + size_t index; + msgpack_object *key; + msgpack_object *value; + + *buffer = flb_sds_cat(*buffer, "{", 1); + if (*buffer == NULL) { + return -1; + } + + if (map->size == 0) { + *buffer = flb_sds_cat(*buffer, "}", 1); + return (*buffer == NULL) ? -1 : 0; + } + + for (index = 0; index < map->size; index++) { + key = &map->ptr[index].key; + value = &map->ptr[index].val; + + if (pretty) { + *buffer = flb_sds_cat(*buffer, "\n", 1); + if (*buffer == NULL || json_append_indent(buffer, depth + 1, 2) != 0) { + return -1; + } + } + if (key->type != MSGPACK_OBJECT_STR || + json_append_escaped_string(buffer, + key->via.str.ptr, + key->via.str.size) != 0) { + return -1; + } + + *buffer = flb_sds_cat(*buffer, pretty ? ": " : ":", pretty ? 2 : 1); + if (*buffer == NULL) { + return -1; + } + + if (render_msgpack_object(buffer, value, pretty, depth + 1) != 0) { + return -1; + } + + if (index + 1 < map->size) { + *buffer = flb_sds_cat(*buffer, ",", 1); + if (*buffer == NULL) { + return -1; + } + } + } + + if (pretty) { + *buffer = flb_sds_cat(*buffer, "\n", 1); + if (*buffer == NULL || json_append_indent(buffer, depth, 2) != 0) { + return -1; + } + } + + *buffer = flb_sds_cat(*buffer, "}", 1); + return (*buffer == NULL) ? -1 : 0; +} + +static int render_msgpack_array(flb_sds_t *buffer, + msgpack_object_array *array, + int pretty, + size_t depth) +{ + size_t index; + + *buffer = flb_sds_cat(*buffer, "[", 1); + if (*buffer == NULL) { + return -1; + } + + if (array->size == 0) { + *buffer = flb_sds_cat(*buffer, "]", 1); + return (*buffer == NULL) ? -1 : 0; + } + + for (index = 0; index < array->size; index++) { + if (pretty) { + *buffer = flb_sds_cat(*buffer, "\n", 1); + if (*buffer == NULL || json_append_indent(buffer, depth + 1, 2) != 0) { + return -1; + } + } + if (render_msgpack_object(buffer, &array->ptr[index], pretty, depth + 1) != 0) { + return -1; + } + + if (index + 1 < array->size) { + *buffer = flb_sds_cat(*buffer, ",", 1); + if (*buffer == NULL) { + return -1; + } + } + } + + if (pretty) { + *buffer = flb_sds_cat(*buffer, "\n", 1); + if (*buffer == NULL || json_append_indent(buffer, depth, 2) != 0) { + return -1; + } + } + + *buffer = flb_sds_cat(*buffer, "]", 1); + return (*buffer == NULL) ? -1 : 0; +} + +static int render_msgpack_object(flb_sds_t *buffer, + msgpack_object *object, + int pretty, + size_t depth) +{ + char tmp[64]; + int length; + + switch (object->type) { + case MSGPACK_OBJECT_NIL: + *buffer = flb_sds_cat(*buffer, "null", 4); + return (*buffer == NULL) ? -1 : 0; + case MSGPACK_OBJECT_BOOLEAN: + *buffer = flb_sds_cat(*buffer, + object->via.boolean ? "true" : "false", + object->via.boolean ? 4 : 5); + return (*buffer == NULL) ? -1 : 0; + case MSGPACK_OBJECT_POSITIVE_INTEGER: + length = snprintf(tmp, sizeof(tmp), "%" PRIu64, object->via.u64); + break; + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + length = snprintf(tmp, sizeof(tmp), "%" PRId64, object->via.i64); + break; + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + length = snprintf(tmp, sizeof(tmp), "%.17g", object->via.f64); + break; + case MSGPACK_OBJECT_STR: + return json_append_escaped_string(buffer, + object->via.str.ptr, + object->via.str.size); + case MSGPACK_OBJECT_ARRAY: + return render_msgpack_array(buffer, &object->via.array, pretty, depth); + case MSGPACK_OBJECT_MAP: + return render_msgpack_map(buffer, &object->via.map, pretty, depth); + default: + return -1; + } + + if (length <= 0 || (size_t) length >= sizeof(tmp)) { + return -1; + } + + *buffer = flb_sds_cat(*buffer, tmp, length); + return (*buffer == NULL) ? -1 : 0; +} + +static char *render_msgpack_document(struct flb_json_doc *document, + size_t *length, + int pretty) +{ + flb_sds_t buffer; + char *output; + size_t size; + + buffer = flb_sds_create_size(512); + if (buffer == NULL) { + flb_errno(); + return NULL; + } + + if (render_msgpack_object(&buffer, document->root.object, pretty, 0) != 0) { + flb_sds_destroy(buffer); + return NULL; + } + + size = flb_sds_len(buffer); + output = flb_malloc(size + 1); + if (output == NULL) { + flb_errno(); + flb_sds_destroy(buffer); + return NULL; + } + + memcpy(output, buffer, size); + output[size] = '\0'; + flb_sds_destroy(buffer); + + if (length != NULL) { + *length = size; + } + + return output; +} + +#ifdef FLB_HAVE_YYJSON +static yyjson_mut_val *msgpack_to_yyjson_mut(yyjson_mut_doc *document, + msgpack_object *object) +{ + size_t index; + yyjson_mut_val *result; + yyjson_mut_val *value; + msgpack_object *key; + + switch (object->type) { + case MSGPACK_OBJECT_NIL: + return yyjson_mut_null(document); + case MSGPACK_OBJECT_BOOLEAN: + return yyjson_mut_bool(document, object->via.boolean); + case MSGPACK_OBJECT_POSITIVE_INTEGER: + return yyjson_mut_uint(document, object->via.u64); + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + return yyjson_mut_sint(document, object->via.i64); + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + return yyjson_mut_real(document, object->via.f64); + case MSGPACK_OBJECT_STR: + return yyjson_mut_strncpy(document, + object->via.str.ptr, + object->via.str.size); + case MSGPACK_OBJECT_ARRAY: + result = yyjson_mut_arr(document); + if (result == NULL) { + return NULL; + } + + for (index = 0; index < object->via.array.size; index++) { + value = msgpack_to_yyjson_mut(document, &object->via.array.ptr[index]); + if (value == NULL || !yyjson_mut_arr_add_val(result, value)) { + return NULL; + } + } + + return result; + case MSGPACK_OBJECT_MAP: + result = yyjson_mut_obj(document); + if (result == NULL) { + return NULL; + } + + for (index = 0; index < object->via.map.size; index++) { + key = &object->via.map.ptr[index].key; + value = msgpack_to_yyjson_mut(document, &object->via.map.ptr[index].val); + + if (key->type != MSGPACK_OBJECT_STR || value == NULL) { + return NULL; + } + + if (!yyjson_mut_obj_add(result, + yyjson_mut_strncpy(document, + key->via.str.ptr, + key->via.str.size), + value)) { + return NULL; + } + } + + return result; + default: + return NULL; + } +} + +static yyjson_mut_val *mutable_to_yyjson_mut(yyjson_mut_doc *document, + struct flb_json_mut_val *value) +{ + struct flb_json_mut_kv *kv_entry; + struct flb_json_mut_entry *array_entry; + yyjson_mut_val *result; + yyjson_mut_val *item; + + switch (value->type) { + case FLB_JSON_MUT_OBJECT: + result = yyjson_mut_obj(document); + if (result == NULL) { + return NULL; + } + + for (kv_entry = value->data.object.head; + kv_entry != NULL; + kv_entry = kv_entry->next) { + item = mutable_to_yyjson_mut(document, kv_entry->value); + if (item == NULL) { + return NULL; + } + + if (!yyjson_mut_obj_add(result, + yyjson_mut_strcpy(document, kv_entry->key), + item)) { + return NULL; + } + } + + return result; + case FLB_JSON_MUT_ARRAY: + result = yyjson_mut_arr(document); + if (result == NULL) { + return NULL; + } + + for (array_entry = value->data.array.head; + array_entry != NULL; + array_entry = array_entry->next) { + item = mutable_to_yyjson_mut(document, array_entry->value); + if (item == NULL || !yyjson_mut_arr_add_val(result, item)) { + return NULL; + } + } + + return result; + case FLB_JSON_MUT_STRING: + return yyjson_mut_strcpy(document, value->data.string); + case FLB_JSON_MUT_BOOL: + return yyjson_mut_bool(document, value->data.boolean); + case FLB_JSON_MUT_INT: + return yyjson_mut_sint(document, value->data.sint); + case FLB_JSON_MUT_UINT: + return yyjson_mut_uint(document, value->data.uint); + case FLB_JSON_MUT_REAL: + return yyjson_mut_real(document, value->data.real); + case FLB_JSON_MUT_NULL: + return yyjson_mut_null(document); + default: + return NULL; + } +} + +static char *render_msgpack_document_yyjson(struct flb_json_doc *document, + size_t *length, + int pretty) +{ + yyjson_alc allocator; + yyjson_mut_doc *yy_document; + yyjson_mut_val *root; + yyjson_write_flag flags; + char *output; + + flb_json_yyjson_init_alc(&allocator); + + yy_document = yyjson_mut_doc_new(&allocator); + if (yy_document == NULL) { + return NULL; + } + + root = msgpack_to_yyjson_mut(yy_document, document->root.object); + if (root == NULL) { + yyjson_mut_doc_free(yy_document); + return NULL; + } + + yyjson_mut_doc_set_root(yy_document, root); + + flags = 0; + if (pretty) { + flags |= YYJSON_WRITE_PRETTY_TWO_SPACES; + } + + output = yyjson_mut_write_opts(yy_document, flags, &allocator, length, NULL); + yyjson_mut_doc_free(yy_document); + + return output; +} + +static char *render_mutable_document_yyjson(struct flb_json_mut_doc *document, + size_t *length, + int pretty) +{ + yyjson_alc allocator; + yyjson_mut_doc *yy_document; + yyjson_mut_val *root; + yyjson_write_flag flags; + char *output; + + if (document == NULL || document->root == NULL) { + return NULL; + } + + flb_json_yyjson_init_alc(&allocator); + + yy_document = yyjson_mut_doc_new(&allocator); + if (yy_document == NULL) { + return NULL; + } + + root = mutable_to_yyjson_mut(yy_document, document->root); + if (root == NULL) { + yyjson_mut_doc_free(yy_document); + return NULL; + } + + yyjson_mut_doc_set_root(yy_document, root); + + flags = 0; + if (pretty) { + flags |= YYJSON_WRITE_PRETTY_TWO_SPACES; + } + + output = yyjson_mut_write_opts(yy_document, flags, &allocator, length, NULL); + yyjson_mut_doc_free(yy_document); + + return output; +} +#endif + +static struct flb_json_mut_val *mut_value_create(struct flb_json_mut_doc *document, + enum flb_json_mut_type type) +{ + struct flb_json_mut_val *value; + + value = flb_calloc(1, sizeof(struct flb_json_mut_val)); + if (value == NULL) { + flb_errno(); + return NULL; + } + + value->type = type; + value->owner = document; + value->next = document->values; + document->values = value; + + return value; +} + +static struct flb_json_mut_kv *mut_kv_create(struct flb_json_mut_doc *document, + const char *key, + size_t key_length, + struct flb_json_mut_val *value) +{ + struct flb_json_mut_kv *entry; + + entry = flb_calloc(1, sizeof(struct flb_json_mut_kv)); + if (entry == NULL) { + flb_errno(); + return NULL; + } + + entry->key = flb_sds_create_len(key, key_length); + if (entry->key == NULL) { + flb_free(entry); + return NULL; + } + + entry->value = value; + entry->alloc_next = document->kvs; + document->kvs = entry; + + return entry; +} + +static struct flb_json_mut_entry *mut_array_entry_create(struct flb_json_mut_doc *document, + struct flb_json_mut_val *value) +{ + struct flb_json_mut_entry *entry; + + entry = flb_calloc(1, sizeof(struct flb_json_mut_entry)); + if (entry == NULL) { + flb_errno(); + return NULL; + } + + entry->value = value; + entry->alloc_next = document->entries; + document->entries = entry; + + return entry; +} + +static struct flb_json_mut_val *copy_msgpack_to_mutable(struct flb_json_mut_doc *document, + msgpack_object *object); + +static struct flb_json_mut_val *copy_msgpack_array_to_mutable(struct flb_json_mut_doc *document, + msgpack_object_array *array) +{ + size_t index; + struct flb_json_mut_val *result; + struct flb_json_mut_val *value; + + result = mut_value_create(document, FLB_JSON_MUT_ARRAY); + if (result == NULL) { + return NULL; + } + + for (index = 0; index < array->size; index++) { + value = copy_msgpack_to_mutable(document, &array->ptr[index]); + if (value == NULL || !flb_json_mut_arr_add_val(result, value)) { + return NULL; + } + } + + return result; +} + +static struct flb_json_mut_val *copy_msgpack_map_to_mutable(struct flb_json_mut_doc *document, + msgpack_object_map *map) +{ + size_t index; + msgpack_object *key; + struct flb_json_mut_val *result; + struct flb_json_mut_val *value; + + result = mut_value_create(document, FLB_JSON_MUT_OBJECT); + if (result == NULL) { + return NULL; + } + + for (index = 0; index < map->size; index++) { + key = &map->ptr[index].key; + value = copy_msgpack_to_mutable(document, &map->ptr[index].val); + + if (key->type != MSGPACK_OBJECT_STR || value == NULL || + !flb_json_mut_obj_add_val(document, + result, + key->via.str.ptr, + value)) { + return NULL; + } + } + + return result; +} + +static struct flb_json_mut_val *copy_msgpack_to_mutable(struct flb_json_mut_doc *document, + msgpack_object *object) +{ + struct flb_json_mut_val *value; + + switch (object->type) { + case MSGPACK_OBJECT_MAP: + return copy_msgpack_map_to_mutable(document, &object->via.map); + case MSGPACK_OBJECT_ARRAY: + return copy_msgpack_array_to_mutable(document, &object->via.array); + case MSGPACK_OBJECT_STR: + value = mut_value_create(document, FLB_JSON_MUT_STRING); + if (value == NULL) { + return NULL; + } + + value->data.string = flb_sds_create_len(object->via.str.ptr, + object->via.str.size); + return (value->data.string != NULL) ? value : NULL; + case MSGPACK_OBJECT_BOOLEAN: + value = mut_value_create(document, FLB_JSON_MUT_BOOL); + if (value != NULL) { + value->data.boolean = object->via.boolean; + } + return value; + case MSGPACK_OBJECT_POSITIVE_INTEGER: + value = mut_value_create(document, FLB_JSON_MUT_UINT); + if (value != NULL) { + value->data.uint = object->via.u64; + } + return value; + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + value = mut_value_create(document, FLB_JSON_MUT_INT); + if (value != NULL) { + value->data.sint = object->via.i64; + } + return value; + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + value = mut_value_create(document, FLB_JSON_MUT_REAL); + if (value != NULL) { + value->data.real = object->via.f64; + } + return value; + case MSGPACK_OBJECT_NIL: + return mut_value_create(document, FLB_JSON_MUT_NULL); + default: + return NULL; + } +} + +static int render_mutable_value(flb_sds_t *buffer, + struct flb_json_mut_val *value, + int pretty, + size_t depth); + +static int render_mutable_object(flb_sds_t *buffer, + struct flb_json_mut_val *value, + int pretty, + size_t depth) +{ + int first; + struct flb_json_mut_kv *entry; + + *buffer = flb_sds_cat(*buffer, "{", 1); + if (*buffer == NULL) { + return -1; + } + + if (value->data.object.head == NULL) { + *buffer = flb_sds_cat(*buffer, "}", 1); + return (*buffer == NULL) ? -1 : 0; + } + + first = FLB_TRUE; + + for (entry = value->data.object.head; entry != NULL; entry = entry->next) { + if (!first) { + *buffer = flb_sds_cat(*buffer, ",", 1); + if (*buffer == NULL) { + return -1; + } + } + + if (pretty) { + *buffer = flb_sds_cat(*buffer, "\n", 1); + if (*buffer == NULL || json_append_indent(buffer, depth + 1, 2) != 0) { + return -1; + } + } + + if (json_append_escaped_string(buffer, entry->key, flb_sds_len(entry->key)) != 0) { + return -1; + } + + *buffer = flb_sds_cat(*buffer, pretty ? ": " : ":", pretty ? 2 : 1); + if (*buffer == NULL || + render_mutable_value(buffer, entry->value, pretty, depth + 1) != 0) { + return -1; + } + + first = FLB_FALSE; + } + + if (pretty) { + *buffer = flb_sds_cat(*buffer, "\n", 1); + if (*buffer == NULL || json_append_indent(buffer, depth, 2) != 0) { + return -1; + } + } + + *buffer = flb_sds_cat(*buffer, "}", 1); + return (*buffer == NULL) ? -1 : 0; +} + +static int render_mutable_array(flb_sds_t *buffer, + struct flb_json_mut_val *value, + int pretty, + size_t depth) +{ + int first; + struct flb_json_mut_entry *entry; + + *buffer = flb_sds_cat(*buffer, "[", 1); + if (*buffer == NULL) { + return -1; + } + + if (value->data.array.head == NULL) { + *buffer = flb_sds_cat(*buffer, "]", 1); + return (*buffer == NULL) ? -1 : 0; + } + + first = FLB_TRUE; + + for (entry = value->data.array.head; entry != NULL; entry = entry->next) { + if (!first) { + *buffer = flb_sds_cat(*buffer, ",", 1); + if (*buffer == NULL) { + return -1; + } + } + + if (pretty) { + *buffer = flb_sds_cat(*buffer, "\n", 1); + if (*buffer == NULL || json_append_indent(buffer, depth + 1, 2) != 0) { + return -1; + } + } + + if (render_mutable_value(buffer, entry->value, pretty, depth + 1) != 0) { + return -1; + } + + first = FLB_FALSE; + } + + if (pretty) { + *buffer = flb_sds_cat(*buffer, "\n", 1); + if (*buffer == NULL || json_append_indent(buffer, depth, 2) != 0) { + return -1; + } + } + + *buffer = flb_sds_cat(*buffer, "]", 1); + return (*buffer == NULL) ? -1 : 0; +} + +static int render_mutable_value(flb_sds_t *buffer, + struct flb_json_mut_val *value, + int pretty, + size_t depth) +{ + char tmp[64]; + int length; + + switch (value->type) { + case FLB_JSON_MUT_OBJECT: + return render_mutable_object(buffer, value, pretty, depth); + case FLB_JSON_MUT_ARRAY: + return render_mutable_array(buffer, value, pretty, depth); + case FLB_JSON_MUT_STRING: + return json_append_escaped_string(buffer, + value->data.string, + flb_sds_len(value->data.string)); + case FLB_JSON_MUT_BOOL: + *buffer = flb_sds_cat(*buffer, + value->data.boolean ? "true" : "false", + value->data.boolean ? 4 : 5); + return (*buffer == NULL) ? -1 : 0; + case FLB_JSON_MUT_INT: + length = snprintf(tmp, sizeof(tmp), "%lld", value->data.sint); + break; + case FLB_JSON_MUT_UINT: + length = snprintf(tmp, sizeof(tmp), "%llu", value->data.uint); + break; + case FLB_JSON_MUT_REAL: + length = snprintf(tmp, sizeof(tmp), "%.17g", value->data.real); + break; + case FLB_JSON_MUT_NULL: + *buffer = flb_sds_cat(*buffer, "null", 4); + return (*buffer == NULL) ? -1 : 0; + default: + return -1; + } + + if (length <= 0 || (size_t) length >= sizeof(tmp)) { + return -1; + } + + *buffer = flb_sds_cat(*buffer, tmp, length); + return (*buffer == NULL) ? -1 : 0; +} + +static char *render_mutable_document(struct flb_json_mut_doc *document, + size_t *length, + int pretty) +{ + flb_sds_t buffer; + char *output; + size_t size; + + if (document == NULL || document->root == NULL) { + return NULL; + } + + buffer = flb_sds_create_size(512); + if (buffer == NULL) { + flb_errno(); + return NULL; + } + + if (render_mutable_value(&buffer, document->root, pretty, 0) != 0) { + flb_sds_destroy(buffer); + return NULL; + } + + size = flb_sds_len(buffer); + output = flb_malloc(size + 1); + if (output == NULL) { + flb_errno(); + flb_sds_destroy(buffer); + return NULL; + } + + memcpy(output, buffer, size); + output[size] = '\0'; + flb_sds_destroy(buffer); + + if (length != NULL) { + *length = size; + } + + return output; +} + +struct flb_json_doc *flb_json_read(const char *input, size_t length) +{ + int root_type; + char *buffer; + size_t buffer_size; + size_t offset; + struct flb_json_doc *document; + + buffer = NULL; + buffer_size = 0; + offset = 0; + + if (flb_pack_json(input, length, &buffer, &buffer_size, &root_type, NULL) != 0) { + return NULL; + } + + document = flb_calloc(1, sizeof(struct flb_json_doc)); + if (document == NULL) { + flb_errno(); + flb_free(buffer); + return NULL; + } + + document->buffer = buffer; + document->buffer_size = buffer_size; + msgpack_unpacked_init(&document->unpacked); + + if (msgpack_unpack_next(&document->unpacked, buffer, buffer_size, &offset) != + MSGPACK_UNPACK_SUCCESS) { + msgpack_unpacked_destroy(&document->unpacked); + flb_free(buffer); + flb_free(document); + return NULL; + } + + document->root.owner = document; + document->root.object = &document->unpacked.data; + + return document; +} + +void flb_json_doc_destroy(struct flb_json_doc *document) +{ + struct flb_json_val *wrapper; + struct flb_json_val *tmp; + + if (document == NULL) { + return; + } + + wrapper = document->wrappers; + while (wrapper != NULL) { + tmp = wrapper->next; + flb_free(wrapper); + wrapper = tmp; + } + + msgpack_unpacked_destroy(&document->unpacked); + flb_free(document->buffer); + flb_free(document); +} + +struct flb_json_val *flb_json_doc_get_root(struct flb_json_doc *document) +{ + return (document != NULL) ? &document->root : NULL; +} + +char *flb_json_write(struct flb_json_doc *document, size_t *length) +{ +#ifdef FLB_HAVE_YYJSON + char *output; +#endif + + if (document == NULL || document->root.object == NULL) { + return NULL; + } + +#ifdef FLB_HAVE_YYJSON + output = render_msgpack_document_yyjson(document, length, FLB_FALSE); + if (output != NULL) { + return output; + } +#endif + + return render_msgpack_document(document, length, FLB_FALSE); +} + +char *flb_json_write_pretty(struct flb_json_doc *document, size_t *length) +{ +#ifdef FLB_HAVE_YYJSON + char *output; +#endif + + if (document == NULL || document->root.object == NULL) { + return NULL; + } + +#ifdef FLB_HAVE_YYJSON + output = render_msgpack_document_yyjson(document, length, FLB_TRUE); + if (output != NULL) { + return output; + } +#endif + + return render_msgpack_document(document, length, FLB_TRUE); +} + +char *flb_json_prettify(const char *input, size_t input_length, size_t *length) +{ +#ifdef FLB_HAVE_YYJSON + yyjson_alc allocator; + yyjson_doc *document; + yyjson_read_err read_error; + yyjson_write_flag flags; + char *output; + + if (input == NULL) { + return NULL; + } + + flb_json_yyjson_init_alc(&allocator); + + document = yyjson_read_opts((char *) input, input_length, 0, + &allocator, &read_error); + if (document != NULL) { + flags = YYJSON_WRITE_PRETTY_TWO_SPACES; + output = yyjson_write_opts(document, flags, &allocator, length, NULL); + yyjson_doc_free(document); + + if (output != NULL) { + return output; + } + } +#endif + int escape_next; + int in_string; + char current; + char next; + char last_token; + flb_sds_t buffer; + size_t depth; + size_t index; + size_t lookahead; + + if (input == NULL) { + return NULL; + } + + buffer = flb_sds_create_size(input_length + 32); + if (buffer == NULL) { + return NULL; + } + + depth = 0; + in_string = FLB_FALSE; + escape_next = FLB_FALSE; + last_token = '\0'; + + for (index = 0; index < input_length; index++) { + current = input[index]; + + if (in_string == FLB_TRUE) { + buffer = flb_sds_cat(buffer, ¤t, 1); + if (buffer == NULL) { + return NULL; + } + + if (escape_next == FLB_TRUE) { + escape_next = FLB_FALSE; + } + else if (current == '\\') { + escape_next = FLB_TRUE; + } + else if (current == '"') { + in_string = FLB_FALSE; + } + + continue; + } + + if (current == ' ' || current == '\n' || + current == '\r' || current == '\t') { + continue; + } + + if (current == '"') { + in_string = FLB_TRUE; + buffer = flb_sds_cat(buffer, ¤t, 1); + if (buffer == NULL) { + return NULL; + } + continue; + } + + if (current == '{' || current == '[') { + buffer = flb_sds_cat(buffer, ¤t, 1); + if (buffer == NULL) { + return NULL; + } + + next = '\0'; + for (lookahead = index + 1; lookahead < input_length; lookahead++) { + next = input[lookahead]; + if (next != ' ' && next != '\n' && + next != '\r' && next != '\t') { + break; + } + } + + depth++; + + if (!((current == '{' && next == '}') || + (current == '[' && next == ']'))) { + buffer = flb_sds_cat(buffer, "\n", 1); + if (buffer == NULL || json_append_indent(&buffer, depth, 2) != 0) { + return NULL; + } + } + + last_token = current; + continue; + } + + if (current == '}' || current == ']') { + if (depth > 0) { + depth--; + } + + if (last_token != '{' && last_token != '[') { + buffer = flb_sds_cat(buffer, "\n", 1); + if (buffer == NULL || json_append_indent(&buffer, depth, 2) != 0) { + return NULL; + } + } + + buffer = flb_sds_cat(buffer, ¤t, 1); + if (buffer == NULL) { + return NULL; + } + + last_token = current; + continue; + } + + if (current == ',') { + buffer = flb_sds_cat(buffer, ",\n", 2); + if (buffer == NULL || json_append_indent(&buffer, depth, 2) != 0) { + return NULL; + } + + last_token = current; + continue; + } + + if (current == ':') { + buffer = flb_sds_cat(buffer, ": ", 2); + if (buffer == NULL) { + return NULL; + } + + last_token = current; + continue; + } + + buffer = flb_sds_cat(buffer, ¤t, 1); + if (buffer == NULL) { + return NULL; + } + + last_token = current; + } + + if (length != NULL) { + *length = flb_sds_len(buffer); + } + + return buffer; +} + +struct flb_json_val *flb_json_obj_get(struct flb_json_val *value, const char *key) +{ + size_t index; + msgpack_object *entry_key; + + if (value == NULL || key == NULL || value->object == NULL || + value->object->type != MSGPACK_OBJECT_MAP) { + return NULL; + } + + for (index = 0; index < value->object->via.map.size; index++) { + entry_key = &value->object->via.map.ptr[index].key; + + if (entry_key->type == MSGPACK_OBJECT_STR && + entry_key->via.str.size == strlen(key) && + strncmp(entry_key->via.str.ptr, key, entry_key->via.str.size) == 0) { + return parsed_value_wrap(value->owner, + &value->object->via.map.ptr[index].val); + } + } + + return NULL; +} + +size_t flb_json_arr_size(struct flb_json_val *value) +{ + if (value == NULL || value->object == NULL || + value->object->type != MSGPACK_OBJECT_ARRAY) { + return 0; + } + + return value->object->via.array.size; +} + +struct flb_json_val *flb_json_arr_get(struct flb_json_val *value, size_t index) +{ + if (value == NULL || value->object == NULL || + value->object->type != MSGPACK_OBJECT_ARRAY || + index >= value->object->via.array.size) { + return NULL; + } + + return parsed_value_wrap(value->owner, &value->object->via.array.ptr[index]); +} + +struct flb_json_mut_doc *flb_json_mut_doc_create(void) +{ + struct flb_json_mut_doc *document; + + document = flb_calloc(1, sizeof(struct flb_json_mut_doc)); + if (document == NULL) { + flb_errno(); + } + + return document; +} + +void flb_json_mut_doc_destroy(struct flb_json_mut_doc *document) +{ + struct flb_json_mut_val *value; + struct flb_json_mut_val *next_value; + struct flb_json_mut_kv *kv; + struct flb_json_mut_kv *next_kv; + struct flb_json_mut_entry *entry; + struct flb_json_mut_entry *next_entry; + + if (document == NULL) { + return; + } + + value = document->values; + while (value != NULL) { + next_value = value->next; + + if (value->type == FLB_JSON_MUT_STRING && value->data.string != NULL) { + flb_sds_destroy(value->data.string); + } + + flb_free(value); + value = next_value; + } + + kv = document->kvs; + while (kv != NULL) { + next_kv = kv->alloc_next; + flb_sds_destroy(kv->key); + flb_free(kv); + kv = next_kv; + } + + entry = document->entries; + while (entry != NULL) { + next_entry = entry->alloc_next; + flb_free(entry); + entry = next_entry; + } + + flb_free(document); +} + +void flb_json_mut_doc_set_root(struct flb_json_mut_doc *document, + struct flb_json_mut_val *root) +{ + if (document != NULL) { + document->root = root; + } +} + +char *flb_json_mut_write(struct flb_json_mut_doc *document, size_t *length) +{ +#ifdef FLB_HAVE_YYJSON + char *output; + + output = render_mutable_document_yyjson(document, length, FLB_FALSE); + if (output != NULL) { + return output; + } +#endif + + return render_mutable_document(document, length, FLB_FALSE); +} + +char *flb_json_mut_write_pretty(struct flb_json_mut_doc *document, size_t *length) +{ +#ifdef FLB_HAVE_YYJSON + char *output; + + output = render_mutable_document_yyjson(document, length, FLB_TRUE); + if (output != NULL) { + return output; + } +#endif + + return render_mutable_document(document, length, FLB_TRUE); +} + +struct flb_json_mut_val *flb_json_mut_obj(struct flb_json_mut_doc *document) +{ + return (document != NULL) ? mut_value_create(document, FLB_JSON_MUT_OBJECT) : NULL; +} + +struct flb_json_mut_val *flb_json_mut_arr(struct flb_json_mut_doc *document) +{ + return (document != NULL) ? mut_value_create(document, FLB_JSON_MUT_ARRAY) : NULL; +} + +struct flb_json_mut_val *flb_json_mut_strncpy(struct flb_json_mut_doc *document, + const char *value, + size_t length) +{ + struct flb_json_mut_val *result; + + if (document == NULL || value == NULL) { + return NULL; + } + + result = mut_value_create(document, FLB_JSON_MUT_STRING); + if (result == NULL) { + return NULL; + } + + result->data.string = flb_sds_create_len(value, length); + if (result->data.string == NULL) { + return NULL; + } + + return result; +} + +struct flb_json_mut_val *flb_json_val_mut_copy(struct flb_json_mut_doc *target, + struct flb_json_val *source) +{ + if (target == NULL || source == NULL || source->object == NULL) { + return NULL; + } + + return copy_msgpack_to_mutable(target, source->object); +} + +int flb_json_mut_arr_add_real(struct flb_json_mut_doc *document, + struct flb_json_mut_val *array, + double value) +{ + struct flb_json_mut_val *entry; + + if (document == NULL || array == NULL || + array->type != FLB_JSON_MUT_ARRAY) { + return 0; + } + + entry = mut_value_create(document, FLB_JSON_MUT_REAL); + if (entry == NULL) { + return 0; + } + + entry->data.real = value; + + return flb_json_mut_arr_add_val(array, entry); +} + +int flb_json_mut_arr_add_strncpy(struct flb_json_mut_doc *document, + struct flb_json_mut_val *array, + const char *value, + size_t length) +{ + struct flb_json_mut_val *entry; + + entry = flb_json_mut_strncpy(document, value, length); + if (entry == NULL) { + return 0; + } + + return flb_json_mut_arr_add_val(array, entry); +} + +int flb_json_mut_arr_add_val(struct flb_json_mut_val *array, + struct flb_json_mut_val *value) +{ + struct flb_json_mut_entry *entry; + + if (array == NULL || value == NULL || array->type != FLB_JSON_MUT_ARRAY) { + return 0; + } + + if (array->owner == NULL) { + return 0; + } + + entry = mut_array_entry_create(array->owner, value); + if (entry == NULL) { + return 0; + } + + if (array->data.array.tail != NULL) { + array->data.array.tail->next = entry; + } + else { + array->data.array.head = entry; + } + + array->data.array.tail = entry; + array->data.array.count++; + + return 1; +} + +size_t flb_json_mut_arr_size(struct flb_json_mut_val *array) +{ + if (array == NULL || array->type != FLB_JSON_MUT_ARRAY) { + return 0; + } + + return array->data.array.count; +} + +int flb_json_mut_obj_add_bool(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + int value) +{ + struct flb_json_mut_val *entry; + + if (document == NULL || object == NULL || key == NULL) { + return 0; + } + + entry = mut_value_create(document, FLB_JSON_MUT_BOOL); + if (entry == NULL) { + return 0; + } + + entry->data.boolean = value; + + return flb_json_mut_obj_add_val(document, object, key, entry); +} + +int flb_json_mut_obj_add_int(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + long long value) +{ + struct flb_json_mut_val *entry; + + entry = mut_value_create(document, FLB_JSON_MUT_INT); + if (entry == NULL) { + return 0; + } + + entry->data.sint = value; + + return flb_json_mut_obj_add_val(document, object, key, entry); +} + +int flb_json_mut_obj_add_real(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + double value) +{ + struct flb_json_mut_val *entry; + + entry = mut_value_create(document, FLB_JSON_MUT_REAL); + if (entry == NULL) { + return 0; + } + + entry->data.real = value; + + return flb_json_mut_obj_add_val(document, object, key, entry); +} + +int flb_json_mut_obj_add_str(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + const char *value) +{ + return flb_json_mut_obj_add_strncpy(document, object, + key, value, strlen(value)); +} + +int flb_json_mut_obj_add_strcpy(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + const char *value) +{ + return flb_json_mut_obj_add_str(document, object, key, value); +} + +int flb_json_mut_obj_add_strn(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + const char *value, + size_t length) +{ + return flb_json_mut_obj_add_strncpy(document, object, key, value, length); +} + +int flb_json_mut_obj_add_strncpy(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + const char *value, + size_t length) +{ + struct flb_json_mut_val *entry; + + entry = flb_json_mut_strncpy(document, value, length); + if (entry == NULL) { + return 0; + } + + return flb_json_mut_obj_add_val(document, object, key, entry); +} + +int flb_json_mut_obj_add_uint(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + unsigned long long value) +{ + struct flb_json_mut_val *entry; + + entry = mut_value_create(document, FLB_JSON_MUT_UINT); + if (entry == NULL) { + return 0; + } + + entry->data.uint = value; + + return flb_json_mut_obj_add_val(document, object, key, entry); +} + +int flb_json_mut_obj_add_val(struct flb_json_mut_doc *document, + struct flb_json_mut_val *object, + const char *key, + struct flb_json_mut_val *value) +{ + size_t key_length; + struct flb_json_mut_kv *entry; + + if (document == NULL || object == NULL || key == NULL || value == NULL || + object->type != FLB_JSON_MUT_OBJECT) { + return 0; + } + + key_length = strlen(key); + entry = mut_kv_create(document, key, key_length, value); + if (entry == NULL) { + return 0; + } + + if (object->data.object.tail != NULL) { + object->data.object.tail->next = entry; + } + else { + object->data.object.head = entry; + } + + object->data.object.tail = entry; + + return 1; +} From 9b89564cfc339ca25acb6bed5edd6caf83ae3a8f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:09:52 -0600 Subject: [PATCH 05/29] lib: cmetrics: upgrade to v2.1.0 Signed-off-by: Eduardo Silva --- lib/cmetrics/CMakeLists.txt | 4 +- lib/cmetrics/src/cmt_cat.c | 2 - lib/cmetrics/src/cmt_decode_msgpack.c | 57 ++++++++++++++++--- lib/cmetrics/src/cmt_encode_msgpack.c | 11 +++- lib/cmetrics/tests/encoding.c | 67 ++++++++++++++++++++++ lib/cmetrics/tests/opentelemetry.c | 80 +++++++++++++++++++++++++++ 6 files changed, 208 insertions(+), 13 deletions(-) diff --git a/lib/cmetrics/CMakeLists.txt b/lib/cmetrics/CMakeLists.txt index c3913c1cd81..1ea5c19ac8d 100644 --- a/lib/cmetrics/CMakeLists.txt +++ b/lib/cmetrics/CMakeLists.txt @@ -5,8 +5,8 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version set(CMT_VERSION_MAJOR 2) -set(CMT_VERSION_MINOR 0) -set(CMT_VERSION_PATCH 5) +set(CMT_VERSION_MINOR 1) +set(CMT_VERSION_PATCH 0) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers diff --git a/lib/cmetrics/src/cmt_cat.c b/lib/cmetrics/src/cmt_cat.c index bd01fc01f81..9ec70176cdf 100644 --- a/lib/cmetrics/src/cmt_cat.c +++ b/lib/cmetrics/src/cmt_cat.c @@ -945,8 +945,6 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr return -1; } - eh->aggregation_type = exp_histogram->aggregation_type; - if (filtered_map != NULL) { ret = cmt_cat_copy_map(&eh->opts, eh->map, filtered_map); } diff --git a/lib/cmetrics/src/cmt_decode_msgpack.c b/lib/cmetrics/src/cmt_decode_msgpack.c index a78573ec7cf..c3a0098789f 100644 --- a/lib/cmetrics/src/cmt_decode_msgpack.c +++ b/lib/cmetrics/src/cmt_decode_msgpack.c @@ -181,41 +181,77 @@ static int create_metric_instance(struct cmt_map *map) static int unpack_opts_ns(mpack_reader_t *reader, size_t index, void *context) { + struct cmt_map *map; struct cmt_opts *opts; - opts = (struct cmt_opts *) context; + map = (struct cmt_map *) context; + opts = map->opts; return cmt_mpack_consume_string_tag(reader, &opts->ns); } static int unpack_opts_ss(mpack_reader_t *reader, size_t index, void *context) { + struct cmt_map *map; struct cmt_opts *opts; - opts = (struct cmt_opts *) context; + map = (struct cmt_map *) context; + opts = map->opts; return cmt_mpack_consume_string_tag(reader, &opts->subsystem); } static int unpack_opts_name(mpack_reader_t *reader, size_t index, void *context) { + struct cmt_map *map; struct cmt_opts *opts; - opts = (struct cmt_opts *) context; + map = (struct cmt_map *) context; + opts = map->opts; return cmt_mpack_consume_string_tag(reader, &opts->name); } static int unpack_opts_desc(mpack_reader_t *reader, size_t index, void *context) { + struct cmt_map *map; struct cmt_opts *opts; - opts = (struct cmt_opts *) context; + map = (struct cmt_map *) context; + opts = map->opts; return cmt_mpack_consume_string_tag(reader, &opts->description); } -static int unpack_opts(mpack_reader_t *reader, struct cmt_opts *opts) +static int unpack_opts_unit(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_map *map; + cfl_sds_t value; + int result; + + map = (struct cmt_map *) context; + value = NULL; + + result = cmt_mpack_consume_string_tag(reader, &value); + if (result != CMT_DECODE_MSGPACK_SUCCESS) { + return result; + } + + if (value != NULL && cfl_sds_len(value) > 0) { + map->unit = value; + } + else { + if (value != NULL) { + cfl_sds_destroy(value); + } + + map->unit = NULL; + } + + return CMT_DECODE_MSGPACK_SUCCESS; +} + +static int unpack_opts(mpack_reader_t *reader, struct cmt_map *map) { int result; struct cmt_mpack_map_entry_callback_t callbacks[] = { @@ -223,17 +259,22 @@ static int unpack_opts(mpack_reader_t *reader, struct cmt_opts *opts) {"ss", unpack_opts_ss}, {"name", unpack_opts_name}, {"desc", unpack_opts_desc}, + {"unit", unpack_opts_unit}, {NULL, NULL} }; + struct cmt_opts *opts; if (NULL == reader || - NULL == opts ) { + NULL == map || + NULL == map->opts) { return CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; } + opts = map->opts; memset(opts, 0, sizeof(struct cmt_opts)); + map->unit = NULL; - result = cmt_mpack_unpack_map(reader, callbacks, (void *) opts); + result = cmt_mpack_unpack_map(reader, callbacks, (void *) map); if (CMT_DECODE_MSGPACK_SUCCESS == result) { /* Ensure required string fields are not NULL */ @@ -1313,7 +1354,7 @@ static int unpack_meta_opts(mpack_reader_t *reader, size_t index, void *context) decode_context = (struct cmt_msgpack_decode_context *) context; - return unpack_opts(reader, decode_context->map->opts); + return unpack_opts(reader, decode_context->map); } static int unpack_meta_label(mpack_reader_t *reader, size_t index, void *context) diff --git a/lib/cmetrics/src/cmt_encode_msgpack.c b/lib/cmetrics/src/cmt_encode_msgpack.c index 87fddf45324..e29ce7cc79d 100644 --- a/lib/cmetrics/src/cmt_encode_msgpack.c +++ b/lib/cmetrics/src/cmt_encode_msgpack.c @@ -94,7 +94,7 @@ static void pack_header(mpack_writer_t *writer, struct cmt *cmt, struct cmt_map /* 'opts' */ mpack_write_cstr(writer, "opts"); - mpack_start_map(writer, 4); + mpack_start_map(writer, 5); /* opts['ns'] */ mpack_write_cstr(writer, "ns"); @@ -112,6 +112,15 @@ static void pack_header(mpack_writer_t *writer, struct cmt *cmt, struct cmt_map mpack_write_cstr(writer, "desc"); mpack_write_cstr(writer, opts->description); + /* opts['unit'] */ + mpack_write_cstr(writer, "unit"); + if (map->unit != NULL) { + mpack_write_cstr(writer, map->unit); + } + else { + mpack_write_cstr(writer, ""); + } + mpack_finish_map(writer); /* 'opts' */ /* 'labels' (label keys) */ diff --git a/lib/cmetrics/tests/encoding.c b/lib/cmetrics/tests/encoding.c index 85905dfecaa..7bec9348d35 100644 --- a/lib/cmetrics/tests/encoding.c +++ b/lib/cmetrics/tests/encoding.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -210,6 +211,71 @@ void test_cmt_to_msgpack() cmt_encode_msgpack_destroy(mp2_buf); } +void test_cmt_msgpack_metric_unit_roundtrip() +{ + int ret; + size_t offset; + char *mp_buf; + size_t mp_size; + struct cmt *cmt1; + struct cmt *cmt2; + struct cmt_counter *counter; + + cmt_initialize(); + + offset = 0; + mp_buf = NULL; + mp_size = 0; + cmt1 = generate_simple_encoder_test_data(); + TEST_CHECK(cmt1 != NULL); + if (cmt1 == NULL) { + return; + } + + counter = cfl_list_entry_first(&cmt1->counters, struct cmt_counter, _head); + TEST_CHECK(counter != NULL); + if (counter == NULL) { + cmt_destroy(cmt1); + return; + } + + counter->map->unit = cfl_sds_create("seconds"); + TEST_CHECK(counter->map->unit != NULL); + if (counter->map->unit == NULL) { + cmt_destroy(cmt1); + return; + } + + ret = cmt_encode_msgpack_create(cmt1, &mp_buf, &mp_size); + TEST_CHECK(ret == 0); + if (ret != 0) { + cmt_destroy(cmt1); + return; + } + + cmt2 = NULL; + ret = cmt_decode_msgpack_create(&cmt2, mp_buf, mp_size, &offset); + TEST_CHECK(ret == 0); + TEST_CHECK(cmt2 != NULL); + if (ret == 0 && cmt2 != NULL) { + counter = cfl_list_entry_first(&cmt2->counters, struct cmt_counter, _head); + TEST_CHECK(counter != NULL); + if (counter != NULL) { + TEST_CHECK(counter->map != NULL); + if (counter->map != NULL) { + TEST_CHECK(counter->map->unit != NULL); + if (counter->map->unit != NULL) { + TEST_CHECK(strcmp(counter->map->unit, "seconds") == 0); + } + } + } + } + + cmt_destroy(cmt1); + cmt_decode_msgpack_destroy(cmt2); + cmt_encode_msgpack_destroy(mp_buf); +} + /* * Encode a context, corrupt the last metric in the msgpack packet * and invoke the decoder to verify if there are any leaks. @@ -1171,6 +1237,7 @@ TEST_LIST = { {"cmt_msgpack_stability", test_cmt_to_msgpack_stability}, {"cmt_msgpack_integrity", test_cmt_to_msgpack_integrity}, {"cmt_msgpack_labels", test_cmt_to_msgpack_labels}, + {"cmt_msgpack_metric_unit_roundtrip", test_cmt_msgpack_metric_unit_roundtrip}, {"cmt_msgpack", test_cmt_to_msgpack}, {"opentelemetry", test_opentelemetry}, {"cloudwatch_emf", test_cloudwatch_emf}, diff --git a/lib/cmetrics/tests/opentelemetry.c b/lib/cmetrics/tests/opentelemetry.c index 4770f6a26b2..479728e26fa 100644 --- a/lib/cmetrics/tests/opentelemetry.c +++ b/lib/cmetrics/tests/opentelemetry.c @@ -79,6 +79,11 @@ static struct cmt *generate_api_test_data() cmt_gauge_set(gauge, ts, 2.0, 0, NULL); cmt_gauge_inc(gauge, ts, 0, NULL); cmt_gauge_sub(gauge, ts, 5.0, 0, NULL); + gauge->map->unit = cfl_sds_create("bytes"); + if (gauge->map->unit == NULL) { + cmt_destroy(cmt); + return NULL; + } untyped = cmt_untyped_create(cmt, "kubernetes", "network", "load_untyped", "Network load untyped", 0, NULL); if (untyped == NULL) { @@ -253,6 +258,47 @@ static int are_texts_equivalent_ignoring_line_order(const char *left, const char return CMT_TRUE; } +static Opentelemetry__Proto__Metrics__V1__Metric *find_otlp_metric_by_name( + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *service_request, + char *name) +{ + size_t resource_index; + size_t scope_index; + size_t metric_index; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics *resource_metrics; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics *scope_metrics; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + if (service_request == NULL || name == NULL) { + return NULL; + } + + for (resource_index = 0; + resource_index < service_request->n_resource_metrics; + resource_index++) { + resource_metrics = service_request->resource_metrics[resource_index]; + + for (scope_index = 0; + scope_index < resource_metrics->n_scope_metrics; + scope_index++) { + scope_metrics = resource_metrics->scope_metrics[scope_index]; + + for (metric_index = 0; + metric_index < scope_metrics->n_metrics; + metric_index++) { + metric = scope_metrics->metrics[metric_index]; + + if (metric->name != NULL && + strcmp(metric->name, name) == 0) { + return metric; + } + } + } + } + + return NULL; +} + static cfl_sds_t generate_exponential_histogram_otlp_payload() { Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest request; @@ -703,7 +749,10 @@ void test_opentelemetry_api_full_roundtrip_with_msgpack() struct cmt *api_context; struct cmt *decoded_context_1; struct cmt *decoded_context_2; + struct cmt_gauge *roundtrip_gauge; struct cmt *msgpack_context; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *service_request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; cmt_initialize(); @@ -767,6 +816,18 @@ void test_opentelemetry_api_full_roundtrip_with_msgpack() return; } + roundtrip_gauge = cfl_list_entry_first(&msgpack_context->gauges, struct cmt_gauge, _head); + TEST_CHECK(roundtrip_gauge != NULL); + if (roundtrip_gauge != NULL) { + TEST_CHECK(roundtrip_gauge->map != NULL); + if (roundtrip_gauge->map != NULL) { + TEST_CHECK(roundtrip_gauge->map->unit != NULL); + if (roundtrip_gauge->map->unit != NULL) { + TEST_CHECK(strcmp(roundtrip_gauge->map->unit, "bytes") == 0); + } + } + } + otlp_payload_2 = cmt_encode_opentelemetry_create(msgpack_context); TEST_CHECK(otlp_payload_2 != NULL); if (otlp_payload_2 == NULL) { @@ -779,6 +840,25 @@ void test_opentelemetry_api_full_roundtrip_with_msgpack() return; } + service_request = + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(otlp_payload_2), (uint8_t *) otlp_payload_2); + TEST_CHECK(service_request != NULL); + if (service_request != NULL) { + metric = find_otlp_metric_by_name(service_request, + "kubernetes_network_load_gauge"); + TEST_CHECK(metric != NULL); + if (metric != NULL) { + TEST_CHECK(metric->unit != NULL); + if (metric->unit != NULL) { + TEST_CHECK(strcmp(metric->unit, "bytes") == 0); + } + } + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked( + service_request, NULL); + } + offset = 0; ret = cmt_decode_opentelemetry_create(&decoded_list_2, otlp_payload_2, cfl_sds_len(otlp_payload_2), &offset); TEST_CHECK(ret == CMT_DECODE_OPENTELEMETRY_SUCCESS); From 00b2ef185179b029a291e7fad138f07ee7236267 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:14:28 -0600 Subject: [PATCH 06/29] lib: ctraces: upgrade to v0.7.1 Signed-off-by: Eduardo Silva --- lib/ctraces/.github/workflows/packages.yaml | 8 ++++---- lib/ctraces/CMakeLists.txt | 2 +- lib/ctraces/include/ctraces/ctr_encode_opentelemetry.h | 6 +++--- lib/ctraces/src/ctr_id.c | 1 + 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/ctraces/.github/workflows/packages.yaml b/lib/ctraces/.github/workflows/packages.yaml index c423590747a..e0aa413fd45 100644 --- a/lib/ctraces/.github/workflows/packages.yaml +++ b/lib/ctraces/.github/workflows/packages.yaml @@ -39,7 +39,7 @@ jobs: echo ${{ matrix.format }} | awk '{print toupper($0)}' | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: ${{ matrix.format }}-arm64 path: | @@ -63,7 +63,7 @@ jobs: echo ${{ matrix.format }} | awk '{print toupper($0)}' | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: ${{ matrix.format }}-amd64 path: | @@ -90,7 +90,7 @@ jobs: echo ${{ matrix.config.format }} | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: ${{ matrix.config.format }}-${{matrix.config.arch}} path: | @@ -107,7 +107,7 @@ jobs: contents: write steps: - name: Download all artefacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v8 with: path: artifacts/ diff --git a/lib/ctraces/CMakeLists.txt b/lib/ctraces/CMakeLists.txt index 226e70dd674..bb2c54fa6d8 100644 --- a/lib/ctraces/CMakeLists.txt +++ b/lib/ctraces/CMakeLists.txt @@ -27,7 +27,7 @@ endif() # CTraces Version set(CTR_VERSION_MAJOR 0) set(CTR_VERSION_MINOR 7) -set(CTR_VERSION_PATCH 0) +set(CTR_VERSION_PATCH 1) set(CTR_VERSION_STR "${CTR_VERSION_MAJOR}.${CTR_VERSION_MINOR}.${CTR_VERSION_PATCH}") # Define __FILENAME__ consistently across Operating Systems diff --git a/lib/ctraces/include/ctraces/ctr_encode_opentelemetry.h b/lib/ctraces/include/ctraces/ctr_encode_opentelemetry.h index ed6093658c2..b4607610235 100644 --- a/lib/ctraces/include/ctraces/ctr_encode_opentelemetry.h +++ b/lib/ctraces/include/ctraces/ctr_encode_opentelemetry.h @@ -17,12 +17,12 @@ * limitations under the License. */ -#ifndef CMT_ENCODE_OPENTELEMETRY_H -#define CMT_ENCODE_OPENTELEMETRY_H +#ifndef CTR_ENCODE_OPENTELEMETRY_H +#define CTR_ENCODE_OPENTELEMETRY_H #include cfl_sds_t ctr_encode_opentelemetry_create(struct ctrace *ctr); void ctr_encode_opentelemetry_destroy(cfl_sds_t text); -#endif \ No newline at end of file +#endif diff --git a/lib/ctraces/src/ctr_id.c b/lib/ctraces/src/ctr_id.c index 0c2299fcd89..3be5eb010ab 100644 --- a/lib/ctraces/src/ctr_id.c +++ b/lib/ctraces/src/ctr_id.c @@ -148,6 +148,7 @@ cfl_sds_t ctr_id_to_lower_base16(struct ctrace_id *cid) } out[i * 2] = 0; + cfl_sds_set_len(out, len * 2); return out; } From 0cf39bd83a980f36b634b8262c606108a06217a5 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:15:13 -0600 Subject: [PATCH 07/29] out_stdout: add format otlp json Signed-off-by: Eduardo Silva --- plugins/out_stdout/stdout.c | 100 ++++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/plugins/out_stdout/stdout.c b/plugins/out_stdout/stdout.c index 082663a7b66..5642ae84d16 100644 --- a/plugins/out_stdout/stdout.c +++ b/plugins/out_stdout/stdout.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include @@ -169,6 +171,85 @@ static void print_traces_text(struct flb_output_instance *ins, } } +static int print_otlp_json(struct flb_stdout *ctx, + struct flb_event_chunk *event_chunk) +{ + static const char *default_logs_body_keys[] = {"log", "message"}; + int result; + int pretty; + flb_sds_t json; + flb_sds_t formatted; + struct flb_opentelemetry_otlp_json_options options; + size_t length; + char *buffer; + + json = NULL; + formatted = NULL; + pretty = (ctx->out_format == FLB_PACK_JSON_FORMAT_OTLP_PRETTY); + + if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { + memset(&options, 0, sizeof(options)); + options.logs_require_otel_metadata = FLB_FALSE; + options.logs_body_keys = default_logs_body_keys; + options.logs_body_key_count = 2; + options.logs_body_key_attributes = FLB_FALSE; + + json = flb_opentelemetry_logs_to_otlp_json(event_chunk->data, + event_chunk->size, + &options, + &result); + } +#ifdef FLB_HAVE_METRICS + else if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { + json = flb_opentelemetry_metrics_msgpack_to_otlp_json(event_chunk->data, + event_chunk->size, + &result); + } +#endif + else if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { + json = flb_opentelemetry_traces_msgpack_to_otlp_json(event_chunk->data, + event_chunk->size, + &result); + } + else { + return -1; + } + + if (json == NULL) { + flb_plg_error(ctx->ins, "could not convert event chunk to OTLP JSON: %d", + result); + return -1; + } + + if (pretty) { + buffer = flb_json_prettify(json, flb_sds_len(json), &length); + if (buffer == NULL) { + flb_sds_destroy(json); + flb_plg_error(ctx->ins, "could not render pretty OTLP JSON payload"); + return -1; + } + + formatted = flb_sds_create_len(buffer, length); + flb_sds_destroy(buffer); + + if (formatted == NULL) { + flb_sds_destroy(json); + flb_plg_error(ctx->ins, "could not allocate pretty OTLP JSON payload"); + return -1; + } + + flb_sds_destroy(json); + json = formatted; + } + + fwrite(json, 1, flb_sds_len(json), stdout); + fputc('\n', stdout); + fflush(stdout); + flb_sds_destroy(json); + + return 0; +} + static void print_profiles_text(struct flb_output_instance *ins, const void *data, size_t bytes) { @@ -231,7 +312,9 @@ static void cb_stdout_flush(struct flb_event_chunk *event_chunk, #ifdef FLB_HAVE_METRICS /* Check if the event type is metrics, handle the payload differently */ - if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { + if (event_chunk->type == FLB_EVENT_TYPE_METRICS && + ctx->out_format != FLB_PACK_JSON_FORMAT_OTLP && + ctx->out_format != FLB_PACK_JSON_FORMAT_OTLP_PRETTY) { print_metrics_text(ctx->ins, (char *) event_chunk->data, event_chunk->size); @@ -239,7 +322,9 @@ static void cb_stdout_flush(struct flb_event_chunk *event_chunk, } #endif - if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { + if (event_chunk->type == FLB_EVENT_TYPE_TRACES && + ctx->out_format != FLB_PACK_JSON_FORMAT_OTLP && + ctx->out_format != FLB_PACK_JSON_FORMAT_OTLP_PRETTY) { print_traces_text(ctx->ins, (char *) event_chunk->data, event_chunk->size); @@ -253,6 +338,15 @@ static void cb_stdout_flush(struct flb_event_chunk *event_chunk, FLB_OUTPUT_RETURN(FLB_OK); } + if (ctx->out_format == FLB_PACK_JSON_FORMAT_OTLP || + ctx->out_format == FLB_PACK_JSON_FORMAT_OTLP_PRETTY) { + if (print_otlp_json(ctx, event_chunk) != 0) { + FLB_OUTPUT_RETURN(FLB_ERROR); + } + + FLB_OUTPUT_RETURN(FLB_OK); + } + /* Assuming data is a log entry...*/ if (ctx->out_format != FLB_PACK_JSON_FORMAT_NONE) { json = flb_pack_msgpack_to_json_format(event_chunk->data, @@ -343,7 +437,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "format", NULL, 0, FLB_FALSE, 0, - "Specifies the data format to be printed. Supported formats are msgpack json, json_lines and json_stream." + "Specifies the data format to be printed. Supported formats are msgpack, json, json_lines, json_stream, otlp_json and otlp_json_pretty." }, { FLB_CONFIG_MAP_STR, "json_date_format", NULL, From f6800affc638ce80f332dd9fd03e491936688cdb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:15:28 -0600 Subject: [PATCH 08/29] out_kafka: add support for OTLP json Signed-off-by: Eduardo Silva --- plugins/out_kafka/kafka.c | 224 ++++++++++++++++++++++++++++++- plugins/out_kafka/kafka_config.c | 6 + plugins/out_kafka/kafka_config.h | 2 + 3 files changed, 230 insertions(+), 2 deletions(-) diff --git a/plugins/out_kafka/kafka.c b/plugins/out_kafka/kafka.c index f4e8965bc86..9fbd9fa177d 100644 --- a/plugins/out_kafka/kafka.c +++ b/plugins/out_kafka/kafka.c @@ -22,9 +22,13 @@ #include #include #include +#include #include #include +#include +#include + #include "kafka_config.h" #include "kafka_topic.h" @@ -487,6 +491,206 @@ int produce_message(struct flb_time *tm, msgpack_object *map, return FLB_OK; } +static int produce_raw_payload(const void *payload, size_t payload_size, + struct flb_out_kafka *ctx) +{ + int ret; + int queue_full_retries; + char *message_key; + size_t message_key_len; + struct flb_kafka_topic *topic; + + if (payload == NULL || payload_size == 0) { + return FLB_OK; + } + + queue_full_retries = 0; + message_key = ctx->message_key; + message_key_len = ctx->message_key_len; + topic = flb_kafka_topic_default(ctx); + + if (topic == NULL) { + flb_plg_error(ctx->ins, "no default topic found"); + return FLB_ERROR; + } + +retry: + if (ctx->queue_full_retries > 0 && + queue_full_retries >= ctx->queue_full_retries) { + ctx->blocked = FLB_FALSE; + return FLB_RETRY; + } + + ret = rd_kafka_produce(topic->tp, + RD_KAFKA_PARTITION_UA, + RD_KAFKA_MSG_F_COPY, + (void *) payload, + payload_size, + message_key, + message_key_len, + ctx); + if (ret == -1) { + if (rd_kafka_last_error() == RD_KAFKA_RESP_ERR__QUEUE_FULL) { + ctx->blocked = FLB_TRUE; + flb_time_sleep(1000); + rd_kafka_poll(ctx->kafka.rk, 0); + queue_full_retries++; + goto retry; + } + + flb_plg_error(ctx->ins, + "failed to produce OTLP payload to topic %s: %s", + rd_kafka_topic_name(topic->tp), + rd_kafka_err2str(rd_kafka_last_error())); + return FLB_ERROR; + } + + ctx->blocked = FLB_FALSE; + rd_kafka_poll(ctx->kafka.rk, 0); + + return FLB_OK; +} + +static int produce_otlp_json(struct flb_out_kafka *ctx, + struct flb_event_chunk *event_chunk) +{ + int result; + flb_sds_t payload; + struct flb_opentelemetry_otlp_json_options options; + static const char *default_logs_body_keys[] = {"log", "message"}; + + payload = NULL; + + if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { + memset(&options, 0, sizeof(options)); + options.logs_require_otel_metadata = FLB_FALSE; + options.logs_body_keys = default_logs_body_keys; + options.logs_body_key_count = 2; + options.logs_body_key_attributes = FLB_FALSE; + + payload = flb_opentelemetry_logs_to_otlp_json(event_chunk->data, + event_chunk->size, + &options, + &result); + } +#ifdef FLB_HAVE_METRICS + else if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { + payload = flb_opentelemetry_metrics_msgpack_to_otlp_json( + event_chunk->data, + event_chunk->size, + &result); + } +#endif + else if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { + payload = flb_opentelemetry_traces_msgpack_to_otlp_json( + event_chunk->data, + event_chunk->size, + &result); + } + else { + return FLB_ERROR; + } + + if (payload == NULL) { + flb_plg_error(ctx->ins, + "could not convert event chunk to OTLP JSON: %d", + result); + return FLB_ERROR; + } + + result = produce_raw_payload(payload, flb_sds_len(payload), ctx); + flb_sds_destroy(payload); + + return result; +} + +static int produce_otlp_proto(struct flb_out_kafka *ctx, + struct flb_event_chunk *event_chunk) +{ + int ret; + int result; + size_t off; + struct ctrace *ctr; + flb_sds_t payload; + struct flb_opentelemetry_otlp_json_options options; + static const char *default_logs_body_keys[] = {"log", "message"}; + + if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { + memset(&options, 0, sizeof(options)); + options.logs_require_otel_metadata = FLB_FALSE; + options.logs_body_keys = default_logs_body_keys; + options.logs_body_key_count = 2; + options.logs_body_key_attributes = FLB_FALSE; + + payload = flb_opentelemetry_logs_to_otlp_proto(event_chunk->data, + event_chunk->size, + &options, + &result); + if (payload == NULL) { + flb_plg_error(ctx->ins, + "could not convert event chunk to OTLP protobuf: %d", + result); + return FLB_ERROR; + } + + result = produce_raw_payload(payload, flb_sds_len(payload), ctx); + flb_sds_destroy(payload); + return result; + } +#ifdef FLB_HAVE_METRICS + else if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { + payload = flb_opentelemetry_metrics_msgpack_to_otlp_proto(event_chunk->data, + event_chunk->size, + &result); + if (payload == NULL) { + flb_plg_error(ctx->ins, + "could not convert metrics chunk to OTLP protobuf: %d", + result); + return FLB_ERROR; + } + + result = produce_raw_payload(payload, cfl_sds_len((cfl_sds_t) payload), ctx); + cmt_encode_opentelemetry_destroy((cfl_sds_t) payload); + + return result; + } +#endif + else if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { + off = 0; + + while ((ret = ctr_decode_msgpack_create(&ctr, + (char *) event_chunk->data, + event_chunk->size, + &off)) == CTR_DECODE_MSGPACK_SUCCESS) { + payload = flb_opentelemetry_traces_to_otlp_proto(ctr, &result); + + if (payload == NULL) { + ctr_destroy(ctr); + flb_plg_error(ctx->ins, + "could not convert trace context to OTLP protobuf: %d", + result); + return FLB_ERROR; + } + + result = produce_raw_payload(payload, flb_sds_len(payload), ctx); + ctr_encode_opentelemetry_destroy((cfl_sds_t) payload); + ctr_destroy(ctr); + if (result != FLB_OK) { + return result; + } + } + + if (ret != CTR_MPACK_INSUFFICIENT_DATA) { + flb_plg_error(ctx->ins, "could not decode traces msgpack: %d", ret); + return FLB_ERROR; + } + + return FLB_OK; + } + + return FLB_ERROR; +} + static void cb_kafka_flush(struct flb_event_chunk *event_chunk, struct flb_output_flush *out_flush, struct flb_input_instance *i_ins, @@ -508,6 +712,21 @@ static void cb_kafka_flush(struct flb_event_chunk *event_chunk, FLB_OUTPUT_RETURN(FLB_RETRY); } + if (ctx->format == FLB_KAFKA_FMT_OTLP_JSON) { + FLB_OUTPUT_RETURN(produce_otlp_json(ctx, event_chunk)); + } + + if (ctx->format == FLB_KAFKA_FMT_OTLP_PROTO) { + FLB_OUTPUT_RETURN(produce_otlp_proto(ctx, event_chunk)); + } + + if (event_chunk->type != FLB_EVENT_TYPE_LOGS) { + flb_plg_error(ctx->ins, + "format '%s' only supports logs; use 'otlp_json' or 'otlp_proto' for metrics and traces", + ctx->format_str != NULL ? ctx->format_str : "json"); + FLB_OUTPUT_RETURN(FLB_ERROR); + } + ret = flb_log_event_decoder_init(&log_decoder, (char *) event_chunk->data, event_chunk->size); @@ -580,7 +799,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "format", (char *)NULL, 0, FLB_TRUE, offsetof(struct flb_out_kafka, format_str), - "Set the record output format." + "Set the record output format. Supported values include json, msgpack, gelf, raw, otlp_json and otlp_proto." }, { FLB_CONFIG_MAP_STR, "message_key", (char *)NULL, @@ -702,5 +921,6 @@ struct flb_output_plugin out_kafka_plugin = { .cb_flush = cb_kafka_flush, .cb_exit = cb_kafka_exit, .config_map = config_map, - .flags = 0 + .flags = 0, + .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES }; diff --git a/plugins/out_kafka/kafka_config.c b/plugins/out_kafka/kafka_config.c index 8b6f910dc88..64fd1c5f851 100644 --- a/plugins/out_kafka/kafka_config.c +++ b/plugins/out_kafka/kafka_config.c @@ -131,6 +131,12 @@ struct flb_out_kafka *flb_out_kafka_create(struct flb_output_instance *ins, else if (strcasecmp(ctx->format_str, "raw") == 0) { ctx->format = FLB_KAFKA_FMT_RAW; } + else if (strcasecmp(ctx->format_str, "otlp_json") == 0) { + ctx->format = FLB_KAFKA_FMT_OTLP_JSON; + } + else if (strcasecmp(ctx->format_str, "otlp_proto") == 0) { + ctx->format = FLB_KAFKA_FMT_OTLP_PROTO; + } } else { ctx->format = FLB_KAFKA_FMT_JSON; diff --git a/plugins/out_kafka/kafka_config.h b/plugins/out_kafka/kafka_config.h index 71bfad88966..1f33d4c794d 100644 --- a/plugins/out_kafka/kafka_config.h +++ b/plugins/out_kafka/kafka_config.h @@ -36,6 +36,8 @@ #define FLB_KAFKA_FMT_AVRO 3 #endif #define FLB_KAFKA_FMT_RAW 4 +#define FLB_KAFKA_FMT_OTLP_JSON 5 +#define FLB_KAFKA_FMT_OTLP_PROTO 6 #define FLB_KAFKA_TS_KEY "@timestamp" #define FLB_KAFKA_QUEUE_FULL_RETRIES "10" From a1ae9ea8fc870926e5166f1301068df299226278 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:16:48 -0600 Subject: [PATCH 09/29] tests: internal: opentelemetry: add format options Signed-off-by: Eduardo Silva --- tests/internal/opentelemetry.c | 489 ++++++++++++++++++++++++++++++++- 1 file changed, 487 insertions(+), 2 deletions(-) diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 4a6722fa41c..975e58d099b 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -31,14 +31,20 @@ #include #include #include +#include +#include #include #include #include #include #include #include +#include +#include +#include #include #include +#include #include "flb_tests_internal.h" @@ -250,6 +256,25 @@ static char *get_group_metadata(void *chunk, size_t size) return result; } +static int json_strings_equal_allowing_otlp_timestamp(const char *expected, + const char *actual) +{ + if (expected == NULL || actual == NULL) { + return expected == actual; + } + + if (strcmp(expected, actual) == 0) { + return FLB_TRUE; + } + + if (strcmp(expected, "{\"otlp\":{}}") == 0 && + strncmp(actual, "{\"otlp\":{\"timestamp\":", 21) == 0) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + static char *get_group_body(void *chunk, size_t size) { struct test_output *output; @@ -319,7 +344,8 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * ret = flb_otel_utils_find_map_entry_by_key(&group_obj->via.map, "metadata", 0, FLB_TRUE); if (ret >= 0) { expected_meta = flb_msgpack_to_json_str(256, &group_obj->via.map.ptr[ret].val, FLB_TRUE); - if (strcmp(expected_meta, actual->groups[i].metadata) != 0) { + if (!json_strings_equal_allowing_otlp_timestamp(expected_meta, + actual->groups[i].metadata)) { printf("Group %zu metadata mismatch:\nExpected: %s\nGot: %s\n", i, expected_meta, actual->groups[i].metadata); flb_free(expected_meta); @@ -370,7 +396,8 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * ret = flb_otel_utils_find_map_entry_by_key(&record_obj->via.map, "metadata", 0, FLB_TRUE); if (ret >= 0) { expected_meta = flb_msgpack_to_json_str(256, &record_obj->via.map.ptr[ret].val, FLB_TRUE); - if (strcmp(expected_meta, actual->groups[i].records[j].metadata) != 0) { + if (!json_strings_equal_allowing_otlp_timestamp(expected_meta, + actual->groups[i].records[j].metadata)) { printf("Group %zu record %zu metadata mismatch:\nExpected: %s\nGot: %s\n", i, j, expected_meta, actual->groups[i].records[j].metadata); flb_free(expected_meta); @@ -550,6 +577,30 @@ void test_json_payload_get_wrapped_value() #define OTEL_TRACES_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/traces.json" #define OTEL_METRICS_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/metrics.json" +static flb_sds_t test_normalize_json(const char *input) +{ + char *buffer; + size_t length; + flb_sds_t normalized; + struct flb_json_doc *doc; + + doc = flb_json_read(input, strlen(input)); + if (doc == NULL) { + return NULL; + } + + buffer = flb_json_write(doc, &length); + flb_json_doc_destroy(doc); + if (buffer == NULL) { + return NULL; + } + + normalized = flb_sds_create_len(buffer, length); + free(buffer); + + return normalized; +} + static void destroy_metrics_context_list(struct cfl_list *context_list) { struct cfl_list *iterator; @@ -1919,6 +1970,426 @@ void test_opentelemetry_metrics_cases() flb_free(cases_json); } +void test_opentelemetry_logs_otlp_json_roundtrip() +{ + int ret; + int result; + char *expected; + flb_sds_t actual; + flb_sds_t normalized_expected; + struct flb_log_event_encoder encoder; + struct flb_opentelemetry_otlp_json_options options; + + expected = + "{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\"," + "\"value\":{\"stringValue\":\"svc\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"scope-a\"," + "\"version\":\"1.0.0\",\"attributes\":[{\"key\":\"scope.key\",\"value\":{" + "\"stringValue\":\"scope.value\"}}]},\"logRecords\":[{\"timeUnixNano\":\"1640995200000000000\"," + "\"observedTimeUnixNano\":\"1640995201000000000\",\"severityNumber\":9," + "\"severityText\":\"INFO\",\"attributes\":[{\"key\":\"http.status_code\"," + "\"value\":{\"intValue\":\"200\"}},{\"key\":\"retryable\",\"value\":{\"boolValue\":true}}]," + "\"traceId\":\"00112233445566778899aabbccddeeff\",\"spanId\":\"0011223344556677\"," + "\"body\":{\"stringValue\":\"hello otlp\"}}]}]}]}"; + + ret = flb_log_event_encoder_init(&encoder, + FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + + ret = flb_opentelemetry_logs_json_to_msgpack(&encoder, + expected, + strlen(expected), + "log", + &result); + TEST_CHECK(ret == 0); + TEST_CHECK(result == 0); + + memset(&options, 0, sizeof(options)); + options.logs_require_otel_metadata = FLB_TRUE; + options.logs_body_key = "log"; + + actual = flb_opentelemetry_logs_to_otlp_json(encoder.output_buffer, + encoder.output_length, + &options, + &result); + TEST_CHECK(actual != NULL); + TEST_CHECK(result == FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS); + if (actual == NULL) { + flb_log_event_encoder_destroy(&encoder); + return; + } + + normalized_expected = test_normalize_json(expected); + TEST_CHECK(normalized_expected != NULL); + if (normalized_expected == NULL) { + flb_sds_destroy(actual); + flb_log_event_encoder_destroy(&encoder); + return; + } + TEST_CHECK(strcmp(normalized_expected, actual) == 0); + + flb_sds_destroy(normalized_expected); + flb_sds_destroy(actual); + flb_log_event_encoder_destroy(&encoder); +} + +void test_opentelemetry_logs_otlp_json_from_plain_logs() +{ + int ret; + int result; + flb_sds_t actual; + flb_sds_t normalized_expected; + struct flb_time timestamp; + struct flb_log_event_encoder encoder; + struct flb_opentelemetry_otlp_json_options options; + char *expected; + + expected = + "{\"resourceLogs\":[{\"resource\":{},\"scopeLogs\":[{\"scope\":{}," + "\"logRecords\":[{\"timeUnixNano\":\"1640995200000000000\"," + "\"body\":{\"stringValue\":\"hello from dummy\"}}]}]}]}"; + + timestamp.tm.tv_sec = 1640995200; + timestamp.tm.tv_nsec = 0; + + ret = flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return; + } + + ret = flb_log_event_encoder_begin_record(&encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_set_timestamp(&encoder, ×tamp); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_append_body_values( + &encoder, + FLB_LOG_EVENT_CSTRING_VALUE("message"), + FLB_LOG_EVENT_CSTRING_VALUE("hello from dummy")); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_commit_record(&encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(&encoder); + return; + } + + memset(&options, 0, sizeof(options)); + options.logs_require_otel_metadata = FLB_FALSE; + + actual = flb_opentelemetry_logs_to_otlp_json(encoder.output_buffer, + encoder.output_length, + &options, + &result); + TEST_CHECK(actual != NULL); + TEST_CHECK(result == FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS); + if (actual == NULL) { + flb_log_event_encoder_destroy(&encoder); + return; + } + + normalized_expected = test_normalize_json(expected); + TEST_CHECK(normalized_expected != NULL); + if (normalized_expected == NULL) { + flb_sds_destroy(actual); + flb_log_event_encoder_destroy(&encoder); + return; + } + + TEST_CHECK(strcmp(normalized_expected, actual) == 0); + + flb_sds_destroy(normalized_expected); + flb_sds_destroy(actual); + flb_log_event_encoder_destroy(&encoder); +} + +void test_opentelemetry_metrics_otlp_json_roundtrip() +{ + int ret; + int result; + char *msgpack_buffer; + char *expected; + size_t msgpack_size; + flb_sds_t actual; + flb_sds_t normalized_expected; + struct cfl_list contexts; + struct cmt *context; + + expected = + "{\"resourceMetrics\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\"," + "\"value\":{\"stringValue\":\"svc\"}}]},\"scopeMetrics\":[{\"scope\":{\"name\":\"scope-a\"," + "\"version\":\"1.0.0\"},\"metrics\":[{\"name\":\"requests_total\",\"description\":\"count\"," + "\"unit\":\"1\",\"sum\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"method\"," + "\"value\":{\"stringValue\":\"GET\"}}],\"timeUnixNano\":\"1704067201000000000\"," + "\"startTimeUnixNano\":\"1704067200000000000\",\"asInt\":\"42\"}]," + "\"aggregationTemporality\":2,\"isMonotonic\":true}}]}]}]}"; + + cfl_list_init(&contexts); + + ret = flb_opentelemetry_metrics_json_to_cmt(&contexts, + expected, + strlen(expected)); + TEST_CHECK(ret == 0); + + context = cfl_list_entry_first(&contexts, struct cmt, _head); + ret = cmt_encode_msgpack_create(context, &msgpack_buffer, &msgpack_size); + TEST_CHECK(ret == 0); + + actual = flb_opentelemetry_metrics_msgpack_to_otlp_json(msgpack_buffer, + msgpack_size, + &result); + TEST_CHECK(actual != NULL); + TEST_CHECK(result == FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS); + if (actual == NULL) { + flb_free(msgpack_buffer); + destroy_metrics_context_list(&contexts); + return; + } + + normalized_expected = test_normalize_json(expected); + TEST_CHECK(normalized_expected != NULL); + if (normalized_expected == NULL) { + flb_free(msgpack_buffer); + flb_sds_destroy(actual); + destroy_metrics_context_list(&contexts); + return; + } + TEST_CHECK(strcmp(normalized_expected, actual) == 0); + + flb_free(msgpack_buffer); + flb_sds_destroy(normalized_expected); + flb_sds_destroy(actual); + destroy_metrics_context_list(&contexts); +} + +void test_opentelemetry_traces_otlp_json_roundtrip() +{ + int ret; + int result; + char *msgpack_buffer; + char *expected; + size_t msgpack_size; + flb_sds_t actual; + flb_sds_t normalized_expected; + struct ctrace *trace_context; + + expected = + "{\"resourceSpans\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\"," + "\"value\":{\"stringValue\":\"svc\"}}]},\"scopeSpans\":[{\"scope\":{\"name\":\"scope-a\"," + "\"version\":\"1.0.0\"},\"spans\":[{\"traceId\":\"00112233445566778899aabbccddeeff\"," + "\"spanId\":\"0011223344556677\",\"name\":\"op-a\",\"kind\":2," + "\"startTimeUnixNano\":\"1704067200000000000\",\"endTimeUnixNano\":\"1704067201000000000\"," + "\"attributes\":[{\"key\":\"http.method\",\"value\":{\"stringValue\":\"GET\"}}]," + "\"status\":{\"code\":\"OK\"}}]}]}]}"; + + trace_context = flb_opentelemetry_json_traces_to_ctrace(expected, + strlen(expected), + &result); + TEST_CHECK(trace_context != NULL); + TEST_CHECK(result == 0); + + ret = ctr_encode_msgpack_create(trace_context, &msgpack_buffer, &msgpack_size); + TEST_CHECK(ret == 0); + + actual = flb_opentelemetry_traces_msgpack_to_otlp_json(msgpack_buffer, + msgpack_size, + &result); + TEST_CHECK(actual != NULL); + TEST_CHECK(result == FLB_OPENTELEMETRY_OTLP_JSON_SUCCESS); + if (actual == NULL) { + flb_free(msgpack_buffer); + ctr_destroy(trace_context); + return; + } + + normalized_expected = test_normalize_json(expected); + TEST_CHECK(normalized_expected != NULL); + if (normalized_expected == NULL) { + flb_free(msgpack_buffer); + flb_sds_destroy(actual); + ctr_destroy(trace_context); + return; + } + TEST_CHECK(strcmp(normalized_expected, actual) == 0); + + flb_free(msgpack_buffer); + flb_sds_destroy(normalized_expected); + flb_sds_destroy(actual); + ctr_destroy(trace_context); +} + +void test_opentelemetry_logs_otlp_proto_from_plain_logs() +{ + int ret; + int result; + flb_sds_t actual; + struct flb_time timestamp; + struct flb_log_event_encoder encoder; + struct flb_opentelemetry_otlp_json_options options; + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *decoded; + + timestamp.tm.tv_sec = 1640995200; + timestamp.tm.tv_nsec = 0; + + ret = flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return; + } + + ret = flb_log_event_encoder_begin_record(&encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + ret = flb_log_event_encoder_set_timestamp(&encoder, ×tamp); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + ret = flb_log_event_encoder_append_body_values( + &encoder, + FLB_LOG_EVENT_CSTRING_VALUE("message"), + FLB_LOG_EVENT_CSTRING_VALUE("hello from dummy")); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + ret = flb_log_event_encoder_commit_record(&encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + + memset(&options, 0, sizeof(options)); + options.logs_require_otel_metadata = FLB_FALSE; + + actual = flb_opentelemetry_logs_to_otlp_proto(encoder.output_buffer, + encoder.output_length, + &options, + &result); + TEST_CHECK(actual != NULL); + TEST_CHECK(result == FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); + if (actual == NULL) { + flb_log_event_encoder_destroy(&encoder); + return; + } + + decoded = + opentelemetry__proto__collector__logs__v1__export_logs_service_request__unpack( + NULL, flb_sds_len(actual), (uint8_t *) actual); + TEST_CHECK(decoded != NULL); + if (decoded != NULL) { + TEST_CHECK(decoded->n_resource_logs == 1); + TEST_CHECK(decoded->resource_logs[0]->n_scope_logs == 1); + TEST_CHECK(decoded->resource_logs[0]->scope_logs[0]->n_log_records == 1); + TEST_CHECK(decoded->resource_logs[0]->scope_logs[0]->log_records[0]->body != NULL); + TEST_CHECK(decoded->resource_logs[0]->scope_logs[0]->log_records[0]->body->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE); + TEST_CHECK(strcmp(decoded->resource_logs[0]->scope_logs[0]->log_records[0]->body->string_value, + "hello from dummy") == 0); + opentelemetry__proto__collector__logs__v1__export_logs_service_request__free_unpacked(decoded, + NULL); + } + + flb_sds_destroy(actual); + flb_log_event_encoder_destroy(&encoder); +} + +void test_opentelemetry_metrics_otlp_proto_roundtrip() +{ + int ret; + int result; + flb_sds_t actual; + char *expected; + struct cfl_list contexts; + struct cmt *context; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *decoded; + + expected = + "{\"resourceMetrics\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\"," + "\"value\":{\"stringValue\":\"svc\"}}]},\"scopeMetrics\":[{\"scope\":{\"name\":\"scope-a\"," + "\"version\":\"1.0.0\"},\"metrics\":[{\"name\":\"requests_total\",\"description\":\"count\"," + "\"unit\":\"1\",\"sum\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"method\"," + "\"value\":{\"stringValue\":\"GET\"}}],\"timeUnixNano\":\"1704067201000000000\"," + "\"startTimeUnixNano\":\"1704067200000000000\",\"asInt\":\"42\"}]," + "\"aggregationTemporality\":2,\"isMonotonic\":true}}]}]}]}"; + + cfl_list_init(&contexts); + ret = flb_opentelemetry_metrics_json_to_cmt(&contexts, + expected, + strlen(expected)); + TEST_CHECK(ret == 0); + if (ret != 0) { + return; + } + + context = cfl_list_entry_first(&contexts, struct cmt, _head); + + actual = flb_opentelemetry_metrics_to_otlp_proto(context, &result); + TEST_CHECK(actual != NULL); + TEST_CHECK(result == FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); + if (actual != NULL) { + decoded = + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, flb_sds_len(actual), (uint8_t *) actual); + TEST_CHECK(decoded != NULL); + if (decoded != NULL) { + TEST_CHECK(decoded->n_resource_metrics > 0); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(decoded, + NULL); + } + cmt_encode_opentelemetry_destroy((cfl_sds_t) actual); + } + + destroy_metrics_context_list(&contexts); +} + +void test_opentelemetry_traces_otlp_proto_roundtrip() +{ + int result; + flb_sds_t actual; + struct ctrace *trace_context; + char *expected; + Opentelemetry__Proto__Collector__Trace__V1__ExportTraceServiceRequest *decoded; + + expected = + "{\"resourceSpans\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\"," + "\"value\":{\"stringValue\":\"svc\"}}]},\"scopeSpans\":[{\"scope\":{\"name\":\"scope-a\"," + "\"version\":\"1.0.0\"},\"spans\":[{\"traceId\":\"00112233445566778899aabbccddeeff\"," + "\"spanId\":\"0011223344556677\",\"name\":\"op-a\",\"kind\":2," + "\"startTimeUnixNano\":\"1704067200000000000\",\"endTimeUnixNano\":\"1704067201000000000\"" + "}]}]}]}"; + + trace_context = flb_opentelemetry_json_traces_to_ctrace(expected, + strlen(expected), + &result); + TEST_CHECK(trace_context != NULL); + if (trace_context == NULL) { + return; + } + + actual = flb_opentelemetry_traces_to_otlp_proto(trace_context, &result); + TEST_CHECK(actual != NULL); + TEST_CHECK(result == FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); + if (actual != NULL) { + decoded = + opentelemetry__proto__collector__trace__v1__export_trace_service_request__unpack( + NULL, flb_sds_len(actual), (uint8_t *) actual); + TEST_CHECK(decoded != NULL); + if (decoded != NULL) { + TEST_CHECK(decoded->n_resource_spans > 0); + opentelemetry__proto__collector__trace__v1__export_trace_service_request__free_unpacked(decoded, + NULL); + } + ctr_encode_opentelemetry_destroy((cfl_sds_t) actual); + } + + ctr_destroy(trace_context); +} + /* Test list */ TEST_LIST = { { "hex_to_id", test_hex_to_id }, @@ -1927,8 +2398,22 @@ TEST_LIST = { { "find_map_entry_by_key", test_find_map_entry_by_key }, { "json_payload_get_wrapped_value", test_json_payload_get_wrapped_value }, { "opentelemetry_cases", test_opentelemetry_cases }, + { "opentelemetry_logs_otlp_json_roundtrip", + test_opentelemetry_logs_otlp_json_roundtrip }, + { "opentelemetry_logs_otlp_json_from_plain_logs", + test_opentelemetry_logs_otlp_json_from_plain_logs }, + { "opentelemetry_logs_otlp_proto_from_plain_logs", + test_opentelemetry_logs_otlp_proto_from_plain_logs }, { "opentelemetry_traces_cases", test_opentelemetry_traces_cases }, + { "opentelemetry_traces_otlp_json_roundtrip", + test_opentelemetry_traces_otlp_json_roundtrip }, + { "opentelemetry_traces_otlp_proto_roundtrip", + test_opentelemetry_traces_otlp_proto_roundtrip }, { "trace_span_binary_sizes", test_trace_span_binary_sizes }, { "opentelemetry_metrics_cases", test_opentelemetry_metrics_cases }, + { "opentelemetry_metrics_otlp_json_roundtrip", + test_opentelemetry_metrics_otlp_json_roundtrip }, + { "opentelemetry_metrics_otlp_proto_roundtrip", + test_opentelemetry_metrics_otlp_proto_roundtrip }, { 0 } }; From fe26ec841694c00d4fd2b07adea2dba953d2847c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:39:23 -0600 Subject: [PATCH 10/29] build: wire yyjson backend support Signed-off-by: Eduardo Silva --- CMakeLists.txt | 23 ++++++++++++++--------- cmake/headers.cmake | 5 ++++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9268b6bd573..133fd0321d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -723,15 +723,20 @@ endif() # ring buffer library add_subdirectory(${FLB_PATH_LIB_RING_BUFFER} EXCLUDE_FROM_ALL) -# yyson -add_subdirectory(${FLB_PATH_LIB_YYJSON} EXCLUDE_FROM_ALL) -if (TARGET yyjson AND NOT FLB_COVERAGE AND - CMAKE_C_COMPILER_ID MATCHES "GNU|Clang|AppleClang|Intel") - # yyjson's O0 reader can exceed the default coroutine stack budget. - # Keep debug symbols, but force enough optimization to shrink the frame. - target_compile_options(yyjson PRIVATE - $<$,$,$>>:-Og> - ) +# yyjson +option(FLB_YYJSON "Enable yyjson backend" ON) +if(FLB_YYJSON) + add_subdirectory(${FLB_PATH_LIB_YYJSON} EXCLUDE_FROM_ALL) + FLB_DEFINITION(FLB_HAVE_YYJSON) + + if (TARGET yyjson AND NOT FLB_COVERAGE AND + CMAKE_C_COMPILER_ID MATCHES "GNU|Clang|AppleClang|Intel") + # yyjson's O0 reader can exceed the default coroutine stack budget. + # Keep debug symbols, but force enough optimization to shrink the frame. + target_compile_options(yyjson PRIVATE + $<$,$,$>>:-Og> + ) + endif() endif() # Avro diff --git a/cmake/headers.cmake b/cmake/headers.cmake index 5abe376f53f..663bb180601 100755 --- a/cmake/headers.cmake +++ b/cmake/headers.cmake @@ -35,7 +35,6 @@ include_directories( ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CTRACES}/include ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_CPROFILES}/include ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_RING_BUFFER}/lwrb/src/include - ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_YYJSON}/src ${FLB_PATH_ROOT_BINARY_DIR}/${FLB_PATH_LIB_JANSSON}/include ${FLB_PATH_ROOT_BINARY_DIR}/lib/cmetrics ${FLB_PATH_ROOT_BINARY_DIR}/lib/cprofiles/include @@ -45,6 +44,10 @@ include_directories( ${FLB_PATH_ROOT_BINARY_DIR}/lib/monkey/include/monkey/ ) +if(FLB_YYJSON) + include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_YYJSON}/src) +endif() + if(FLB_UTF8_ENCODER) include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_TUTF8E}/include) endif() From 0ea28bdd2d1eaec16fcaa0a5b19b116e2165b9b0 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:39:31 -0600 Subject: [PATCH 11/29] oauth2_jwt: use generic JSON pack API Signed-off-by: Eduardo Silva --- src/flb_oauth2_jwt.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/flb_oauth2_jwt.c b/src/flb_oauth2_jwt.c index b37376c182e..57557c7c1e1 100644 --- a/src/flb_oauth2_jwt.c +++ b/src/flb_oauth2_jwt.c @@ -395,8 +395,8 @@ static int oauth2_jwt_parse_header(const char *json, size_t json_len, const char *val_str; /* Convert JSON to msgpack */ - ret = flb_pack_json_yyjson(json, json_len, &mp_buf, &mp_size, - &root_type, NULL); + ret = flb_pack_json(json, json_len, &mp_buf, &mp_size, + &root_type, NULL); if (ret != 0 || root_type != JSMN_OBJECT) { if (mp_buf) { flb_free(mp_buf); @@ -477,8 +477,8 @@ static int oauth2_jwt_parse_payload(const char *json, size_t json_len, msgpack_object *first; /* Convert JSON to msgpack */ - ret = flb_pack_json_yyjson(json, json_len, &mp_buf, &mp_size, - &root_type, NULL); + ret = flb_pack_json(json, json_len, &mp_buf, &mp_size, + &root_type, NULL); if (ret != 0 || root_type != JSMN_OBJECT) { if (mp_buf) { flb_free(mp_buf); @@ -849,9 +849,9 @@ static int oauth2_jwks_parse_json(flb_sds_t jwks_json, struct flb_oauth2_jwks_ca struct flb_oauth2_jwks_key *jwks_key; /* Convert JSON to msgpack */ - ret = flb_pack_json_yyjson(jwks_json, flb_sds_len(jwks_json), - &mp_buf, &mp_size, - &root_type, NULL); + ret = flb_pack_json(jwks_json, flb_sds_len(jwks_json), + &mp_buf, &mp_size, + &root_type, NULL); if (ret != 0 || root_type != JSMN_OBJECT) { if (mp_buf) { flb_free(mp_buf); @@ -1167,8 +1167,8 @@ static int oauth2_jwt_check_audience(const char *json, size_t json_len, } /* Convert JSON to msgpack */ - ret = flb_pack_json_yyjson(json, json_len, &mp_buf, &mp_size, - &root_type, NULL); + ret = flb_pack_json(json, json_len, &mp_buf, &mp_size, + &root_type, NULL); if (ret != 0 || root_type != JSMN_OBJECT) { if (mp_buf) { flb_free(mp_buf); From 4db7393e70ea636a54b8d0c488fba4401fc1530f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:39:39 -0600 Subject: [PATCH 12/29] parser_json: use generic JSON pack API Signed-off-by: Eduardo Silva --- src/flb_parser_json.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/flb_parser_json.c b/src/flb_parser_json.c index 203793a7e9e..e6b2dfc3198 100644 --- a/src/flb_parser_json.c +++ b/src/flb_parser_json.c @@ -57,15 +57,12 @@ int flb_parser_json_do(struct flb_parser *parser, struct flb_tm tm = {0}; struct flb_time *t; size_t consumed; - struct flb_pack_opts pack_opts = {0}; consumed = 0; /* Convert incoming in_buf JSON message to message pack format */ - pack_opts.backend = FLB_PACK_JSON_BACKEND_YYJSON; - ret = flb_pack_json_recs_ext(in_buf, in_size, &mp_buf, &mp_size, - &root_type, &records, &consumed, - &pack_opts); + ret = flb_pack_json_recs(in_buf, in_size, &mp_buf, &mp_size, + &root_type, &records, &consumed); if (ret != 0) { return -1; } From 346b8b7efe486577943ac874f640c2631bf948f1 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:39:45 -0600 Subject: [PATCH 13/29] tests: mp: use generic JSON pack API Signed-off-by: Eduardo Silva --- tests/internal/mp.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/internal/mp.c b/tests/internal/mp.c index 3beea43a8d7..3dc7be4b302 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -106,7 +106,7 @@ void test_accessor_keys_remove() /* Convert to msgpack */ len = strlen(json); - ret = flb_pack_json_yyjson(json, len, &buf, &size, &type, NULL); + ret = flb_pack_json(json, len, &buf, &size, &type, NULL); TEST_CHECK(ret == 0); if (ret == -1) { exit(EXIT_FAILURE); @@ -178,7 +178,7 @@ void test_keys_remove_subkey_key() /* Convert to msgpack */ len = strlen(json); - ret = flb_pack_json_yyjson(json, len, &buf, &size, &type, NULL); + ret = flb_pack_json(json, len, &buf, &size, &type, NULL); TEST_CHECK(ret == 0); if (ret == -1) { exit(EXIT_FAILURE); @@ -267,7 +267,7 @@ void test_remove_sibling_subkeys() /* Convert to msgpack */ len = strlen(json); - ret = flb_pack_json_yyjson(json, len, &buf, &size, &type, NULL); + ret = flb_pack_json(json, len, &buf, &size, &type, NULL); TEST_CHECK(ret == 0); if (ret == -1) { exit(EXIT_FAILURE); @@ -367,7 +367,7 @@ void remove_subkey_keys(char *list[], int list_size, int index_start) /* Convert to msgpack */ len = strlen(json); - ret = flb_pack_json_yyjson(json, len, &buf, &size, &type, NULL); + ret = flb_pack_json(json, len, &buf, &size, &type, NULL); TEST_CHECK(ret == 0); if (ret == -1) { exit(EXIT_FAILURE); @@ -483,7 +483,7 @@ void test_object_to_cfl_to_msgpack() /* Convert to msgpack */ len = strlen(json); - ret = flb_pack_json_yyjson(json, len, &buf, &size, &type, NULL); + ret = flb_pack_json(json, len, &buf, &size, &type, NULL); TEST_CHECK(ret == 0); if (ret == -1) { exit(EXIT_FAILURE); From 18db0af1d0fbd872fdf05a0920a9fbb6e710b4f2 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:47:51 -0600 Subject: [PATCH 14/29] out_stdout: fix buffer release Signed-off-by: Eduardo Silva --- plugins/out_stdout/stdout.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/out_stdout/stdout.c b/plugins/out_stdout/stdout.c index 5642ae84d16..12838cb7899 100644 --- a/plugins/out_stdout/stdout.c +++ b/plugins/out_stdout/stdout.c @@ -230,7 +230,7 @@ static int print_otlp_json(struct flb_stdout *ctx, } formatted = flb_sds_create_len(buffer, length); - flb_sds_destroy(buffer); + flb_free(buffer); if (formatted == NULL) { flb_sds_destroy(json); From 1e947038ca03019eb87c046a1df36f16869755b3 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:55:24 -0600 Subject: [PATCH 15/29] opentelemetry: merge metrics contexts before protobuf encoding Signed-off-by: Eduardo Silva --- .../flb_opentelemetry_otlp_proto.c | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/src/opentelemetry/flb_opentelemetry_otlp_proto.c b/src/opentelemetry/flb_opentelemetry_otlp_proto.c index 06fd5efbb27..46130ec9bcc 100644 --- a/src/opentelemetry/flb_opentelemetry_otlp_proto.c +++ b/src/opentelemetry/flb_opentelemetry_otlp_proto.c @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -1317,71 +1318,73 @@ flb_sds_t flb_opentelemetry_metrics_msgpack_to_otlp_proto(const void *data, int *result) { int ret; + int decoded_count; size_t offset; - size_t payload_length; - cfl_sds_t output; cfl_sds_t encoded; + cfl_sds_t output; struct cmt *context; + struct cmt *merged_context; if (data == NULL || size == 0) { set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_ARGUMENT, EINVAL); return NULL; } + merged_context = cmt_create(); + if (merged_context == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; + } + output = NULL; offset = 0; + decoded_count = 0; while ((ret = cmt_decode_msgpack_create(&context, (char *) data, size, &offset)) == CMT_DECODE_MSGPACK_SUCCESS) { - encoded = cmt_encode_opentelemetry_create(context); - if (encoded == NULL) { - cmt_destroy(context); - if (output != NULL) { - cfl_sds_destroy(output); - } - set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); - return NULL; - } - - payload_length = cfl_sds_len(encoded); - - if (output == NULL) { - output = cfl_sds_create_len(encoded, payload_length); - } - else { - output = cfl_sds_cat(output, encoded, payload_length); - } - - cmt_encode_opentelemetry_destroy(encoded); + ret = cmt_cat(merged_context, context); cmt_destroy(context); - if (output == NULL) { + if (ret != 0) { + cmt_destroy(merged_context); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); return NULL; } + + decoded_count++; } if (ret != CMT_DECODE_MSGPACK_INSUFFICIENT_DATA && ret != CMT_DECODE_MSGPACK_SUCCESS) { - if (output != NULL) { - cfl_sds_destroy(output); - } + cmt_destroy(merged_context); set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_INVALID_ARGUMENT, EINVAL); return NULL; } - if (output == NULL) { + if (decoded_count == 0) { + cmt_destroy(merged_context); output = cfl_sds_create_size(0); if (output == NULL) { set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); return NULL; } + + set_result(result, FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); + return (flb_sds_t) output; + } + + encoded = cmt_encode_opentelemetry_create(merged_context); + cmt_destroy(merged_context); + + if (encoded == NULL) { + set_error(result, FLB_OPENTELEMETRY_OTLP_PROTO_NOT_SUPPORTED, ENOMEM); + return NULL; } set_result(result, FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); - return (flb_sds_t) output; + return (flb_sds_t) encoded; } flb_sds_t flb_opentelemetry_traces_to_otlp_proto(struct ctrace *context, From dd8d41871824cd4307869615da10198327262770 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 04:55:31 -0600 Subject: [PATCH 16/29] tests: add metrics OTLP protobuf multi-context regression Signed-off-by: Eduardo Silva --- tests/internal/opentelemetry.c | 98 ++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 975e58d099b..40a8861f6ae 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -2347,6 +2347,102 @@ void test_opentelemetry_metrics_otlp_proto_roundtrip() destroy_metrics_context_list(&contexts); } +void test_opentelemetry_metrics_msgpack_otlp_proto_merges_contexts() +{ + int ret; + int result; + size_t combined_size; + char *combined_buffer; + char *msgpack_buffer_a; + char *msgpack_buffer_b; + size_t msgpack_size_a; + size_t msgpack_size_b; + flb_sds_t actual; + struct cfl_list contexts; + struct cfl_list *head; + struct cmt *context; + const char *json = + "{\"resourceMetrics\":[" + "{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"svc-a\"}}]}," + "\"scopeMetrics\":[{\"scope\":{\"name\":\"scope-a\",\"version\":\"1.0.0\"}," + "\"metrics\":[{\"name\":\"requests_total\",\"description\":\"count\",\"unit\":\"1\"," + "\"sum\":{\"dataPoints\":[{\"attributes\":[{\"key\":\"method\",\"value\":{\"stringValue\":\"GET\"}}]," + "\"timeUnixNano\":\"1704067201000000000\",\"startTimeUnixNano\":\"1704067200000000000\"," + "\"asInt\":\"42\"}],\"aggregationTemporality\":2,\"isMonotonic\":true}}]}]}," + "{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"svc-b\"}}]}," + "\"scopeMetrics\":[{\"scope\":{\"name\":\"scope-b\",\"version\":\"1.0.0\"}," + "\"metrics\":[{\"name\":\"latency_ms\",\"description\":\"latency\",\"unit\":\"ms\"," + "\"gauge\":{\"dataPoints\":[{\"timeUnixNano\":\"1704067202000000000\",\"asDouble\":12.5}]}}]}]}]}"; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *decoded; + + cfl_list_init(&contexts); + + ret = flb_opentelemetry_metrics_json_to_cmt(&contexts, json, strlen(json)); + TEST_CHECK(ret == 0); + if (ret != 0) { + return; + } + + head = contexts.next; + context = cfl_list_entry(head, struct cmt, _head); + ret = cmt_encode_msgpack_create(context, &msgpack_buffer_a, &msgpack_size_a); + TEST_CHECK(ret == 0); + if (ret != 0) { + destroy_metrics_context_list(&contexts); + return; + } + + head = head->next; + context = cfl_list_entry(head, struct cmt, _head); + ret = cmt_encode_msgpack_create(context, &msgpack_buffer_b, &msgpack_size_b); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_free(msgpack_buffer_a); + destroy_metrics_context_list(&contexts); + return; + } + + combined_size = msgpack_size_a + msgpack_size_b; + combined_buffer = flb_malloc(combined_size); + TEST_CHECK(combined_buffer != NULL); + if (combined_buffer == NULL) { + flb_free(msgpack_buffer_a); + flb_free(msgpack_buffer_b); + destroy_metrics_context_list(&contexts); + return; + } + + memcpy(combined_buffer, msgpack_buffer_a, msgpack_size_a); + memcpy(combined_buffer + msgpack_size_a, msgpack_buffer_b, msgpack_size_b); + + actual = flb_opentelemetry_metrics_msgpack_to_otlp_proto(combined_buffer, + combined_size, + &result); + TEST_CHECK(actual != NULL); + TEST_CHECK(result == FLB_OPENTELEMETRY_OTLP_PROTO_SUCCESS); + if (actual != NULL) { + decoded = + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, flb_sds_len(actual), (uint8_t *) actual); + TEST_CHECK(decoded != NULL); + if (decoded != NULL) { + TEST_CHECK(decoded->n_resource_metrics > 0); + TEST_CHECK( + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(decoded) == + flb_sds_len(actual)); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(decoded, + NULL); + } + + cmt_encode_opentelemetry_destroy((cfl_sds_t) actual); + } + + flb_free(combined_buffer); + flb_free(msgpack_buffer_a); + flb_free(msgpack_buffer_b); + destroy_metrics_context_list(&contexts); +} + void test_opentelemetry_traces_otlp_proto_roundtrip() { int result; @@ -2415,5 +2511,7 @@ TEST_LIST = { test_opentelemetry_metrics_otlp_json_roundtrip }, { "opentelemetry_metrics_otlp_proto_roundtrip", test_opentelemetry_metrics_otlp_proto_roundtrip }, + { "opentelemetry_metrics_msgpack_otlp_proto_merges_contexts", + test_opentelemetry_metrics_msgpack_otlp_proto_merges_contexts }, { 0 } }; From 3eb1237a992c1e1ff2c67ac368f39af8187227d4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 06:40:33 -0600 Subject: [PATCH 17/29] lib: cmetrics: upgrade to v2.1.1 Signed-off-by: Eduardo Silva --- lib/cmetrics/CMakeLists.txt | 2 +- lib/cmetrics/src/cmt_cat.c | 2 ++ lib/cmetrics/tests/cat.c | 58 +++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/lib/cmetrics/CMakeLists.txt b/lib/cmetrics/CMakeLists.txt index 1ea5c19ac8d..a3385a8ba45 100644 --- a/lib/cmetrics/CMakeLists.txt +++ b/lib/cmetrics/CMakeLists.txt @@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version set(CMT_VERSION_MAJOR 2) set(CMT_VERSION_MINOR 1) -set(CMT_VERSION_PATCH 0) +set(CMT_VERSION_PATCH 1) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers diff --git a/lib/cmetrics/src/cmt_cat.c b/lib/cmetrics/src/cmt_cat.c index 9ec70176cdf..bd01fc01f81 100644 --- a/lib/cmetrics/src/cmt_cat.c +++ b/lib/cmetrics/src/cmt_cat.c @@ -945,6 +945,8 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr return -1; } + eh->aggregation_type = exp_histogram->aggregation_type; + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&eh->opts, eh->map, filtered_map); } diff --git a/lib/cmetrics/tests/cat.c b/lib/cmetrics/tests/cat.c index 3ef59fca3e1..dce4385cee7 100644 --- a/lib/cmetrics/tests/cat.c +++ b/lib/cmetrics/tests/cat.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -611,6 +612,62 @@ void test_histogram_populated_to_empty() cmt_destroy(cmt2); } +void test_exp_histogram_preserves_aggregation_type() +{ + int ret; + uint64_t ts; + uint64_t positive_buckets[] = {1, 2}; + struct cmt *src; + struct cmt *dst; + struct cmt_exp_histogram *src_histogram; + struct cmt_exp_histogram *dst_histogram; + + src = cmt_create(); + TEST_CHECK(src != NULL); + + dst = cmt_create(); + TEST_CHECK(dst != NULL); + + src_histogram = cmt_exp_histogram_create(src, + "test", "exp", "aggregation_type", + "Exponential histogram aggregation type test", + 1, (char *[]) {"label"}); + TEST_CHECK(src_histogram != NULL); + + ts = cfl_time_now(); + + ret = cmt_exp_histogram_set_default(src_histogram, + ts, + 1, + 0, + 0.0, + 0, + 2, + positive_buckets, + 0, + 0, + NULL, + CMT_TRUE, + 3.0, + 3, + 1, + (char *[]) {"value"}); + TEST_CHECK(ret == 0); + + src_histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + + ret = cmt_cat(dst, src); + TEST_CHECK(ret == 0); + + dst_histogram = cfl_list_entry_first(&dst->exp_histograms, + struct cmt_exp_histogram, _head); + TEST_CHECK(dst_histogram != NULL); + TEST_CHECK(dst_histogram->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); + + cmt_destroy(src); + cmt_destroy(dst); +} + TEST_LIST = { {"cat", test_cat}, {"duplicate_metrics", test_duplicate_metrics}, @@ -618,5 +675,6 @@ TEST_LIST = { {"histogram_mismatched_buckets", test_histogram_mismatched_buckets}, {"histogram_empty_to_populated", test_histogram_empty_to_populated}, {"histogram_populated_to_empty", test_histogram_populated_to_empty}, + {"exp_histogram_preserves_aggregation_type", test_exp_histogram_preserves_aggregation_type}, { 0 } }; From 7ac632edd9afb03acba9c8c97ff35927a032be73 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 06:46:48 -0600 Subject: [PATCH 18/29] opentelemetry: fix C89 loop declarations Signed-off-by: Eduardo Silva --- src/opentelemetry/flb_opentelemetry_otlp_proto.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/opentelemetry/flb_opentelemetry_otlp_proto.c b/src/opentelemetry/flb_opentelemetry_otlp_proto.c index 46130ec9bcc..7cde2d4f054 100644 --- a/src/opentelemetry/flb_opentelemetry_otlp_proto.c +++ b/src/opentelemetry/flb_opentelemetry_otlp_proto.c @@ -1047,6 +1047,8 @@ static int append_binary_id_field(ProtobufCBinaryData *field, msgpack_object *value, size_t expected_size) { + size_t i; + if (value == NULL) { return 0; } @@ -1075,7 +1077,7 @@ static int append_binary_id_field(ProtobufCBinaryData *field, return -1; } - for (size_t i = 0; i < expected_size; i++) { + for (i = 0; i < expected_size; i++) { int high; int low; char *str = (char *) value->via.str.ptr; @@ -1231,6 +1233,7 @@ static void destroy_export_logs( { size_t index; size_t inner; + size_t record_index; Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; @@ -1250,7 +1253,7 @@ static void destroy_export_logs( continue; } - for (size_t record_index = 0; + for (record_index = 0; record_index < scope_log->n_log_records; record_index++) { destroy_log_record(scope_log->log_records[record_index]); From f3c89ce35c31a558ee0541c095065afc9f158417 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 07:52:35 -0600 Subject: [PATCH 19/29] input_metric: rotate metric chunks after append Signed-off-by: Eduardo Silva --- src/flb_input_metric.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/flb_input_metric.c b/src/flb_input_metric.c index 1f55b848905..31a3f6478b2 100644 --- a/src/flb_input_metric.c +++ b/src/flb_input_metric.c @@ -22,6 +22,7 @@ #include #include #include +#include #include static int input_metrics_append(struct flb_input_instance *ins, @@ -89,10 +90,30 @@ static int input_metrics_append(struct flb_input_instance *ins, return -1; } - /* Append packed metrics */ - ret = flb_input_chunk_append_raw(ins, FLB_INPUT_METRICS, 0, + /* + * Metrics chunks still need a positive logical event count when they + * enter the chunk/task pipeline. Passing 0 propagates as total_records=0, + * and later stages can treat the chunk as empty even though it carries a + * valid encoded metrics payload. Use 1 as a non-zero sentinel count. + */ + ret = flb_input_chunk_append_raw(ins, FLB_INPUT_METRICS, 1, tag, tag_len, mt_buf, mt_size); + if (ret == 0 && tag != NULL) { + void *chunk_ref; + + /* + * Keep metric chunks short-lived per append. Reusing the same tag-bound + * metric chunk can delay task creation for rapidly updated series, + * which makes runtime consumers miss freshly generated metrics. + */ + chunk_ref = flb_hash_table_get_ptr(ins->ht_metric_chunks, tag, tag_len); + if (chunk_ref != NULL) { + flb_hash_table_del_ptr(ins->ht_metric_chunks, + tag, tag_len, chunk_ref); + } + } + cmt_encode_msgpack_destroy(mt_buf); return ret; From bef39467d8c065827f9806148aec689f2c17b164 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 07:53:01 -0600 Subject: [PATCH 20/29] tests: runtime: log_to_metrics: makes checks binary-safe Signed-off-by: Eduardo Silva --- tests/runtime/filter_log_to_metrics.c | 204 ++++++++++++++++++-------- 1 file changed, 144 insertions(+), 60 deletions(-) diff --git a/tests/runtime/filter_log_to_metrics.c b/tests/runtime/filter_log_to_metrics.c index a68007b0885..3594b0806f0 100644 --- a/tests/runtime/filter_log_to_metrics.c +++ b/tests/runtime/filter_log_to_metrics.c @@ -129,18 +129,69 @@ TEST_LIST = { pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; int data_size = 0; -bool new_data = false; char output[32768]; +static int buffer_contains(const char *buffer, size_t buffer_size, + const char *needle) +{ + size_t offset; + size_t needle_length; + + needle_length = strlen(needle); + + if (needle_length == 0) { + return FLB_TRUE; + } + + if (buffer_size < needle_length) { + return FLB_FALSE; + } + + for (offset = 0; offset + needle_length <= buffer_size; offset++) { + if (memcmp(buffer + offset, needle, needle_length) == 0) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + +static void snapshot_output(char *out_result, size_t *out_size) +{ + pthread_mutex_lock(&result_mutex); + memcpy(out_result, output, data_size); + out_result[data_size] = '\0'; + *out_size = (size_t) data_size; + pthread_mutex_unlock(&result_mutex); +} + +static void reset_output_state(void) +{ + pthread_mutex_lock(&result_mutex); + output[0] = '\0'; + data_size = 0; + pthread_mutex_unlock(&result_mutex); +} int callback_test(void* data, size_t size, void* cb_data) { + size_t available; + size_t copy_size; + if (size > 0) { - new_data = true; flb_debug("[test_filter_log_to_metrics] received message: %s", (char*)data); pthread_mutex_lock(&result_mutex); - strncat(output, data, size); - data_size = size; + available = sizeof(output) - (size_t) data_size - 1; + copy_size = available; + if (copy_size > size) { + copy_size = size; + } + + if (copy_size > 0) { + memcpy(output + data_size, data, copy_size); + data_size += copy_size; + output[data_size] = '\0'; + } pthread_mutex_unlock(&result_mutex); } flb_free(data); @@ -160,17 +211,12 @@ void wait_with_timeout(uint32_t timeout_ms, char *out_result) struct flb_time end_time; struct flb_time diff_time; uint64_t elapsed_time_flb = 0; - flb_time_get(&start_time); - while (true) { - if(new_data){ - pthread_mutex_lock(&result_mutex); - new_data = false; - strcat(out_result, output); - pthread_mutex_unlock(&result_mutex); + (void) out_result; - } + flb_time_get(&start_time); + while (true) { flb_time_msleep(100); flb_time_get(&end_time); flb_time_diff(&end_time, &start_time, &diff_time); @@ -191,17 +237,20 @@ void flb_test_log_to_metrics_counter_k8s(void) int in_ffd; int filter_ffd; int out_ffd; - char *result = NULL; + int result; struct flb_lib_out_cb cb_data; char *input = JSON_MSG1; char finalString[32768] = ""; + size_t finalSize = 0; const char *expected = "\"value\":5.0,\"labels\":[\"k8s-dummy\"," "\"testpod\",\"mycontainer\",\"abc123\"," "\"def456\",\"red\",\"right\"]"; - const char *expected2 = "{\"ns\":\"log_metric\",\"ss\":\"counter\"," - "\"name\":\"test\",\"desc\":\"Counts messages\"}"; + const char *expected2 = "\"opts\":{\"ns\":\"log_metric\",\"ss\":\"counter\"," + "\"name\":\"test\",\"desc\":\"Counts messages\"," + "\"unit\":\"\"}"; + reset_output_state(); ctx = flb_create(); flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", "error", NULL); @@ -241,13 +290,16 @@ void flb_test_log_to_metrics_counter_k8s(void) flb_lib_push(ctx, in_ffd, input, strlen(input)); } wait_with_timeout(2000, finalString); - result = strstr(finalString, expected); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + snapshot_output(finalString, &finalSize); + result = buffer_contains(finalString, finalSize, expected); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected, (unsigned long) finalSize, (int) finalSize, finalString); } - result = strstr(finalString, expected2); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + result = buffer_contains(finalString, finalSize, expected2); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected2, (unsigned long) finalSize, (int) finalSize, finalString); } filter_test_destroy(ctx); @@ -262,14 +314,17 @@ void flb_test_log_to_metrics_counter(void) int in_ffd; int filter_ffd; int out_ffd; - char *result = NULL; + int result; struct flb_lib_out_cb cb_data; char *input = JSON_MSG1; char finalString[32768] = ""; + size_t finalSize = 0; const char *expected = "\"value\":5.0,\"labels\":[\"red\",\"right\"]"; - const char *expected2 = "{\"ns\":\"myns\",\"ss\":\"subsystem\"," - "\"name\":\"test\",\"desc\":\"Counts messages\"}"; + const char *expected2 = "\"opts\":{\"ns\":\"myns\",\"ss\":\"subsystem\"," + "\"name\":\"test\",\"desc\":\"Counts messages\"," + "\"unit\":\"\"}"; + reset_output_state(); ctx = flb_create(); flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", "error", NULL); @@ -309,13 +364,16 @@ void flb_test_log_to_metrics_counter(void) flb_lib_push(ctx, in_ffd, input, strlen(input)); } wait_with_timeout(2000, finalString); - result = strstr(finalString, expected); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + snapshot_output(finalString, &finalSize); + result = buffer_contains(finalString, finalSize, expected); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected, (unsigned long) finalSize, (int) finalSize, finalString); } - result = strstr(finalString, expected2); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + result = buffer_contains(finalString, finalSize, expected2); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected2, (unsigned long) finalSize, (int) finalSize, finalString); } filter_test_destroy(ctx); @@ -329,11 +387,12 @@ void flb_test_log_to_metrics_counter_k8s_two_tuples(void) int in_ffd; int filter_ffd; int out_ffd; - char *result = NULL; + int result; struct flb_lib_out_cb cb_data; char *input1 = JSON_MSG1; char *input2 = JSON_MSG2; char finalString[32768] = ""; + size_t finalSize = 0; const char *expected1 = "\"value\":5.0,\"labels\":[\"k8s-dummy\"," "\"testpod\",\"mycontainer\",\"abc123\"," "\"def456\",\"red\",\"right\"]"; @@ -342,6 +401,7 @@ void flb_test_log_to_metrics_counter_k8s_two_tuples(void) "\"def456\",\"red\",\"left\"]"; + reset_output_state(); ctx = flb_create(); flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", "error", NULL); @@ -384,14 +444,17 @@ void flb_test_log_to_metrics_counter_k8s_two_tuples(void) flb_lib_push(ctx, in_ffd, input2, strlen(input2)); } wait_with_timeout(2000, finalString); - result = strstr(finalString, expected1); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected1, finalString); + snapshot_output(finalString, &finalSize); + result = buffer_contains(finalString, finalSize, expected1); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected1, (unsigned long) finalSize, (int) finalSize, finalString); } - result = strstr(finalString, expected2); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected2, finalString); + result = buffer_contains(finalString, finalSize, expected2); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected2, (unsigned long) finalSize, (int) finalSize, finalString); } filter_test_destroy(ctx); @@ -405,12 +468,14 @@ void flb_test_log_to_metrics_gauge(void) int in_ffd; int filter_ffd; int out_ffd; - char *result = NULL; + int result; struct flb_lib_out_cb cb_data; char *input = JSON_MSG1; char finalString[32768] = ""; + size_t finalSize = 0; const char *expected = "\"value\":20.0,\"labels\":[\"red\",\"right\"]"; + reset_output_state(); ctx = flb_create(); flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", "error", NULL); @@ -450,9 +515,11 @@ void flb_test_log_to_metrics_gauge(void) flb_lib_push(ctx, in_ffd, input, strlen(input)); wait_with_timeout(2000, finalString); - result = strstr(finalString, expected); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + snapshot_output(finalString, &finalSize); + result = buffer_contains(finalString, finalSize, expected); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected, (unsigned long) finalSize, (int) finalSize, finalString); } filter_test_destroy(ctx); @@ -468,14 +535,16 @@ void flb_test_log_to_metrics_histogram(void) int in_ffd; int filter_ffd; int out_ffd; - char *result = NULL; + int result; struct flb_lib_out_cb cb_data; char *input = JSON_MSG1; char finalString[32768] = ""; + size_t finalSize = 0; const char *expected = "\"histogram\":{\"buckets\":" \ "[0,0,0,0,0,0,0,0,0,0,0,5],\"" \ "sum\":100.0,\"count\":5},\"" \ "labels\":[\"red\",\"right\"]"; + reset_output_state(); ctx = flb_create(); flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", "error", NULL); @@ -517,9 +586,11 @@ void flb_test_log_to_metrics_histogram(void) } wait_with_timeout(2000, finalString); - result = strstr(finalString, expected); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + snapshot_output(finalString, &finalSize); + result = buffer_contains(finalString, finalSize, expected); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected, (unsigned long) finalSize, (int) finalSize, finalString); } filter_test_destroy(ctx); @@ -533,14 +604,16 @@ void flb_test_log_to_metrics_reg(void) int in_ffd; int filter_ffd; int out_ffd; - char *result = NULL; + int result; struct flb_lib_out_cb cb_data; char *input1 = JSON_MSG1; char *input2 = JSON_MSG3; char finalString[32768] = ""; + size_t finalSize = 0; const char *expected = "\"value\":3.0,\"labels\":[\"red\",\"left\"]"; + reset_output_state(); ctx = flb_create(); flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", "error", NULL); @@ -583,9 +656,11 @@ void flb_test_log_to_metrics_reg(void) flb_lib_push(ctx, in_ffd, input2, strlen(input2)); } wait_with_timeout(2000, finalString); - result = strstr(finalString, expected); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + snapshot_output(finalString, &finalSize); + result = buffer_contains(finalString, finalSize, expected); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected, (unsigned long) finalSize, (int) finalSize, finalString); } filter_test_destroy(ctx); @@ -600,13 +675,15 @@ void flb_test_log_to_metrics_empty_label_keys_regex(void) int in_ffd; int filter_ffd; int out_ffd; - char *result = NULL; + int result; struct flb_lib_out_cb cb_data; char *input = JSON_MSG3; char finalString[32768] = ""; + size_t finalSize = 0; const char *expected = "\"value\":3.0,"; + reset_output_state(); ctx = flb_create(); flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", "error", NULL); @@ -646,9 +723,11 @@ void flb_test_log_to_metrics_empty_label_keys_regex(void) flb_lib_push(ctx, in_ffd, input, strlen(input)); } wait_with_timeout(2000, finalString); - result = strstr(finalString, expected); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected, finalString); + snapshot_output(finalString, &finalSize); + result = buffer_contains(finalString, finalSize, expected); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected, (unsigned long) finalSize, (int) finalSize, finalString); } filter_test_destroy(ctx); @@ -662,13 +741,15 @@ void flb_test_log_to_metrics_label(void) int in_ffd; int filter_ffd; int out_ffd; - char *result = NULL; + int result; struct flb_lib_out_cb cb_data; char *input = JSON_MSG1; char finalString[32768] = ""; + size_t finalSize = 0; const char *expected_label_name = ",\"labels\":[\"pod_name\"],"; const char *expected_label_value = "\"value\":2.0,\"labels\":[\"testpod\"]"; + reset_output_state(); ctx = flb_create(); flb_service_set(ctx, "Flush", "0.200000000", "Grace", "1", "Log_Level", "error", NULL); @@ -706,13 +787,16 @@ void flb_test_log_to_metrics_label(void) flb_lib_push(ctx, in_ffd, input, strlen(input)); } wait_with_timeout(2000, finalString); - result = strstr(finalString, expected_label_name); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected_label_name, finalString); + snapshot_output(finalString, &finalSize); + result = buffer_contains(finalString, finalSize, expected_label_name); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected_label_name, (unsigned long) finalSize, (int) finalSize, finalString); } - result = strstr(finalString, expected_label_value); - if (!TEST_CHECK(result != NULL)) { - TEST_MSG("expected substring:\n%s\ngot:\n%s\n", expected_label_value, finalString); + result = buffer_contains(finalString, finalSize, expected_label_value); + if (!TEST_CHECK(result == FLB_TRUE)) { + TEST_MSG("expected substring:\n%s\ngot (%lu bytes):\n%.*s\n", + expected_label_value, (unsigned long) finalSize, (int) finalSize, finalString); } filter_test_destroy(ctx); } From 7f4b0719f24450ef2c9797d86b54287fa7c05f57 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 08:33:17 -0600 Subject: [PATCH 21/29] time: fix nanosecond conversion from uint64 Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_time.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fluent-bit/flb_time.h b/include/fluent-bit/flb_time.h index 5205446a815..99d060c23ae 100644 --- a/include/fluent-bit/flb_time.h +++ b/include/fluent-bit/flb_time.h @@ -87,7 +87,7 @@ static inline void flb_time_copy(struct flb_time *dst, struct flb_time *src) static inline void flb_time_from_uint64(struct flb_time *dst, uint64_t value) { dst->tm.tv_sec = (long) (value / 1000000000L); - dst->tm.tv_nsec = (long) (value - dst->tm.tv_sec); + dst->tm.tv_nsec = (long) (value - ((uint64_t) dst->tm.tv_sec * 1000000000L)); } static inline void flb_time_from_double(struct flb_time *dst, double d) From b8bdc816dc4434298bedea5ac353766bfe26b105 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 10:26:56 -0600 Subject: [PATCH 22/29] out_kafka: fix OTLP trace EOF handling and event types Signed-off-by: Eduardo Silva --- plugins/out_kafka/kafka.c | 5 +++-- plugins/out_kafka/kafka_config.c | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/out_kafka/kafka.c b/plugins/out_kafka/kafka.c index 9fbd9fa177d..3ae6b176b0d 100644 --- a/plugins/out_kafka/kafka.c +++ b/plugins/out_kafka/kafka.c @@ -680,7 +680,8 @@ static int produce_otlp_proto(struct flb_out_kafka *ctx, } } - if (ret != CTR_MPACK_INSUFFICIENT_DATA) { + if (ret != CTR_MPACK_INSUFFICIENT_DATA && + !(ret == CTR_MPACK_ENGINE_ERROR && off >= event_chunk->size)) { flb_plg_error(ctx->ins, "could not decode traces msgpack: %d", ret); return FLB_ERROR; } @@ -922,5 +923,5 @@ struct flb_output_plugin out_kafka_plugin = { .cb_exit = cb_kafka_exit, .config_map = config_map, .flags = 0, - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES + .event_type = FLB_OUTPUT_LOGS }; diff --git a/plugins/out_kafka/kafka_config.c b/plugins/out_kafka/kafka_config.c index 64fd1c5f851..d6ba68c41b9 100644 --- a/plugins/out_kafka/kafka_config.c +++ b/plugins/out_kafka/kafka_config.c @@ -142,6 +142,12 @@ struct flb_out_kafka *flb_out_kafka_create(struct flb_output_instance *ins, ctx->format = FLB_KAFKA_FMT_JSON; } + ins->event_type = FLB_OUTPUT_LOGS; + if (ctx->format == FLB_KAFKA_FMT_OTLP_JSON || + ctx->format == FLB_KAFKA_FMT_OTLP_PROTO) { + ins->event_type |= FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES; + } + /* Config: Message_Key */ if (ctx->message_key) { ctx->message_key_len = strlen(ctx->message_key); From 9db55b9cf0f153ca68d01f68dc1b3a958c462622 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 10:27:03 -0600 Subject: [PATCH 23/29] out_stdout: fix prettified OTLP JSON buffer cleanup Signed-off-by: Eduardo Silva --- plugins/out_stdout/stdout.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/out_stdout/stdout.c b/plugins/out_stdout/stdout.c index 12838cb7899..6981f6a6f04 100644 --- a/plugins/out_stdout/stdout.c +++ b/plugins/out_stdout/stdout.c @@ -230,7 +230,11 @@ static int print_otlp_json(struct flb_stdout *ctx, } formatted = flb_sds_create_len(buffer, length); +#ifdef FLB_HAVE_YYJSON flb_free(buffer); +#else + flb_sds_destroy(buffer); +#endif if (formatted == NULL) { flb_sds_destroy(json); From 0aad83f71e3eda7650d01b3e3539c5f30d9fc8f9 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 10:27:09 -0600 Subject: [PATCH 24/29] json: validate mutable document inputs and ownership Signed-off-by: Eduardo Silva --- src/flb_json.c | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/flb_json.c b/src/flb_json.c index dd8c8553a7a..d3e544a1f80 100644 --- a/src/flb_json.c +++ b/src/flb_json.c @@ -1401,9 +1401,15 @@ void flb_json_mut_doc_destroy(struct flb_json_mut_doc *document) void flb_json_mut_doc_set_root(struct flb_json_mut_doc *document, struct flb_json_mut_val *root) { - if (document != NULL) { - document->root = root; + if (document == NULL) { + return; } + + if (root != NULL && root->owner != document) { + return; + } + + document->root = root; } char *flb_json_mut_write(struct flb_json_mut_doc *document, size_t *length) @@ -1522,7 +1528,7 @@ int flb_json_mut_arr_add_val(struct flb_json_mut_val *array, return 0; } - if (array->owner == NULL) { + if (array->owner == NULL || value->owner != array->owner) { return 0; } @@ -1581,6 +1587,10 @@ int flb_json_mut_obj_add_int(struct flb_json_mut_doc *document, { struct flb_json_mut_val *entry; + if (document == NULL || object == NULL || key == NULL) { + return 0; + } + entry = mut_value_create(document, FLB_JSON_MUT_INT); if (entry == NULL) { return 0; @@ -1598,6 +1608,10 @@ int flb_json_mut_obj_add_real(struct flb_json_mut_doc *document, { struct flb_json_mut_val *entry; + if (document == NULL || object == NULL || key == NULL) { + return 0; + } + entry = mut_value_create(document, FLB_JSON_MUT_REAL); if (entry == NULL) { return 0; @@ -1613,6 +1627,10 @@ int flb_json_mut_obj_add_str(struct flb_json_mut_doc *document, const char *key, const char *value) { + if (document == NULL || object == NULL || key == NULL || value == NULL) { + return 0; + } + return flb_json_mut_obj_add_strncpy(document, object, key, value, strlen(value)); } @@ -1657,6 +1675,10 @@ int flb_json_mut_obj_add_uint(struct flb_json_mut_doc *document, { struct flb_json_mut_val *entry; + if (document == NULL || object == NULL || key == NULL) { + return 0; + } + entry = mut_value_create(document, FLB_JSON_MUT_UINT); if (entry == NULL) { return 0; @@ -1680,6 +1702,10 @@ int flb_json_mut_obj_add_val(struct flb_json_mut_doc *document, return 0; } + if (object->owner != document || value->owner != document) { + return 0; + } + key_length = strlen(key); entry = mut_kv_create(document, key, key_length, value); if (entry == NULL) { From a27a6f5548b989c19a473f4b09d6b04f530bef23 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 10:27:15 -0600 Subject: [PATCH 25/29] opentelemetry: validate OTLP msgpack keys and binary IDs Signed-off-by: Eduardo Silva --- src/opentelemetry/flb_opentelemetry_otlp_proto.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/opentelemetry/flb_opentelemetry_otlp_proto.c b/src/opentelemetry/flb_opentelemetry_otlp_proto.c index 7cde2d4f054..08f98783cb1 100644 --- a/src/opentelemetry/flb_opentelemetry_otlp_proto.c +++ b/src/opentelemetry/flb_opentelemetry_otlp_proto.c @@ -402,6 +402,11 @@ static Opentelemetry__Proto__Common__V1__KeyValue *msgpack_kv_to_otlp_any_value( return NULL; } + if (input_pair->key.type != MSGPACK_OBJECT_STR) { + flb_free(kv); + return NULL; + } + kv->key = flb_strndup(input_pair->key.via.str.ptr, input_pair->key.via.str.size); if (kv->key == NULL) { flb_free(kv); @@ -1054,6 +1059,10 @@ static int append_binary_id_field(ProtobufCBinaryData *field, } if (value->type == MSGPACK_OBJECT_BIN) { + if (value->via.bin.size != expected_size) { + return 0; + } + field->data = flb_malloc(value->via.bin.size); if (field->data == NULL) { return -1; From a106869abc5caf036c2558c1dc62a579e7c0dc15 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 10:27:40 -0600 Subject: [PATCH 26/29] tests: internal: opentelemetry: harden setup and cleanup Signed-off-by: Eduardo Silva --- tests/internal/opentelemetry.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 40a8861f6ae..24d15a559aa 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -596,7 +596,7 @@ static flb_sds_t test_normalize_json(const char *input) } normalized = flb_sds_create_len(buffer, length); - free(buffer); + flb_free(buffer); return normalized; } @@ -2144,6 +2144,10 @@ void test_opentelemetry_metrics_otlp_json_roundtrip() expected, strlen(expected)); TEST_CHECK(ret == 0); + if (ret != 0 || cfl_list_is_empty(&contexts)) { + destroy_metrics_context_list(&contexts); + return; + } context = cfl_list_entry_first(&contexts, struct cmt, _head); ret = cmt_encode_msgpack_create(context, &msgpack_buffer, &msgpack_size); @@ -2201,6 +2205,9 @@ void test_opentelemetry_traces_otlp_json_roundtrip() &result); TEST_CHECK(trace_context != NULL); TEST_CHECK(result == 0); + if (trace_context == NULL || result != 0) { + return; + } ret = ctr_encode_msgpack_create(trace_context, &msgpack_buffer, &msgpack_size); TEST_CHECK(ret == 0); From 8c69b3d2ff47d2830425a707e2ed0f483c931ee9 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 10:28:23 -0600 Subject: [PATCH 27/29] input: add lock for metric chunk table Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_input.h | 1 + src/flb_input.c | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/include/fluent-bit/flb_input.h b/include/fluent-bit/flb_input.h index b8a3e612b66..5ec2eef9546 100644 --- a/include/fluent-bit/flb_input.h +++ b/include/fluent-bit/flb_input.h @@ -446,6 +446,7 @@ struct flb_input_instance { struct flb_hash_table *ht_metric_chunks; struct flb_hash_table *ht_trace_chunks; struct flb_hash_table *ht_profile_chunks; + pthread_mutex_t metrics_chunk_lock; /* TLS settings */ int use_tls; /* bool, try to use TLS for I/O */ diff --git a/src/flb_input.c b/src/flb_input.c index 6538fd7669c..c4d8564c4c2 100644 --- a/src/flb_input.c +++ b/src/flb_input.c @@ -366,6 +366,8 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, return NULL; } + pthread_mutex_init(&instance->metrics_chunk_lock, NULL); + /* format name (with instance id) */ snprintf(instance->name, sizeof(instance->name) - 1, "%s.%i", plugin->name, id); @@ -1101,6 +1103,8 @@ void flb_input_instance_destroy(struct flb_input_instance *ins) ins->ht_profile_chunks = NULL; } + pthread_mutex_destroy(&ins->metrics_chunk_lock); + if (ins->ch_events[0] > 0) { mk_event_closesocket(ins->ch_events[0]); } From 0b4d4649eacb8f93ebd7ba65db5c0a65571a8f1f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 10:28:28 -0600 Subject: [PATCH 28/29] input_chunk: lock metric chunk table updates Signed-off-by: Eduardo Silva --- src/flb_input_chunk.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/flb_input_chunk.c b/src/flb_input_chunk.c index 0955f7affa3..3d82fe1c381 100644 --- a/src/flb_input_chunk.c +++ b/src/flb_input_chunk.c @@ -2081,7 +2081,9 @@ struct flb_input_chunk *flb_input_chunk_create(struct flb_input_instance *in, in flb_hash_table_add(in->ht_log_chunks, tag, tag_len, ic, 0); } else if (event_type == FLB_INPUT_METRICS) { + pthread_mutex_lock(&in->metrics_chunk_lock); flb_hash_table_add(in->ht_metric_chunks, tag, tag_len, ic, 0); + pthread_mutex_unlock(&in->metrics_chunk_lock); } else if (event_type == FLB_INPUT_TRACES) { flb_hash_table_add(in->ht_trace_chunks, tag, tag_len, ic, 0); @@ -2139,8 +2141,10 @@ int flb_input_chunk_destroy_corrupted(struct flb_input_chunk *ic, tag_buf, tag_len, (void *) ic); } else if (ic->event_type == FLB_INPUT_METRICS) { + pthread_mutex_lock(&ic->in->metrics_chunk_lock); flb_hash_table_del_ptr(ic->in->ht_metric_chunks, tag_buf, tag_len, (void *) ic); + pthread_mutex_unlock(&ic->in->metrics_chunk_lock); } else if (ic->event_type == FLB_INPUT_TRACES) { flb_hash_table_del_ptr(ic->in->ht_trace_chunks, @@ -2244,8 +2248,10 @@ int flb_input_chunk_destroy(struct flb_input_chunk *ic, int del) tag_buf, tag_len, (void *) ic); } else if (ic->event_type == FLB_INPUT_METRICS) { + pthread_mutex_lock(&ic->in->metrics_chunk_lock); flb_hash_table_del_ptr(ic->in->ht_metric_chunks, tag_buf, tag_len, (void *) ic); + pthread_mutex_unlock(&ic->in->metrics_chunk_lock); } else if (ic->event_type == FLB_INPUT_TRACES) { flb_hash_table_del_ptr(ic->in->ht_trace_chunks, @@ -2300,8 +2306,10 @@ static struct flb_input_chunk *input_chunk_get(struct flb_input_instance *in, (void *) &ic, &out_size); } else if (event_type == FLB_INPUT_METRICS) { + pthread_mutex_lock(&in->metrics_chunk_lock); id = flb_hash_table_get(in->ht_metric_chunks, tag, tag_len, (void *) &ic, &out_size); + pthread_mutex_unlock(&in->metrics_chunk_lock); } else if (event_type == FLB_INPUT_TRACES) { id = flb_hash_table_get(in->ht_trace_chunks, tag, tag_len, From 00ffa5690f0a401684dee196a791d3143a74bbcb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Mar 2026 10:28:34 -0600 Subject: [PATCH 29/29] input_metric: lock metric chunk table access Signed-off-by: Eduardo Silva --- src/flb_input_metric.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/flb_input_metric.c b/src/flb_input_metric.c index 31a3f6478b2..eef823d10d5 100644 --- a/src/flb_input_metric.c +++ b/src/flb_input_metric.c @@ -23,6 +23,7 @@ #include #include #include +#include #include static int input_metrics_append(struct flb_input_instance *ins, @@ -107,11 +108,13 @@ static int input_metrics_append(struct flb_input_instance *ins, * metric chunk can delay task creation for rapidly updated series, * which makes runtime consumers miss freshly generated metrics. */ + pthread_mutex_lock(&ins->metrics_chunk_lock); chunk_ref = flb_hash_table_get_ptr(ins->ht_metric_chunks, tag, tag_len); if (chunk_ref != NULL) { flb_hash_table_del_ptr(ins->ht_metric_chunks, tag, tag_len, chunk_ref); } + pthread_mutex_unlock(&ins->metrics_chunk_lock); } cmt_encode_msgpack_destroy(mt_buf);