From 359b94bd9576ffb47ed8bd9e9f647a58adeba145 Mon Sep 17 00:00:00 2001 From: Yadong Zhang Date: Sun, 21 Jun 2026 02:49:04 -0700 Subject: [PATCH] http2: pre-allocate body buffer to eliminate realloc overhead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-allocate HTTP/2 body buffer on the first DATA frame to avoid repeated reallocs as 16KB frames arrive. Three strategies in priority order: 1. Content-Length header — exact size (non-gRPC HTTP/2) 2. gRPC 5-byte frame header — parse message length for exact size 3. Fallback to frame size (len) — original behavior cfl_sds_cat calls cfl_sds_increase which grows by exactly len (16KB = nghttp2 frame size), not doubling. For a 4MB gRPC message this means ~250 realloc+memcpy calls. CPU profiling showed _realloc_base consuming 78.8% of fluent-bit CPU at 500 QPS OTLP gRPC ingest. Also validates the pre-allocated size against buffer_max_size upfront to reject oversized messages before allocating. Signed-off-by: Yadong Zhang --- src/http_server/flb_http_server_http2.c | 43 ++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/http_server/flb_http_server_http2.c b/src/http_server/flb_http_server_http2.c index 51a1d50f701..39f8475e2ee 100644 --- a/src/http_server/flb_http_server_http2.c +++ b/src/http_server/flb_http_server_http2.c @@ -743,7 +743,48 @@ static int http2_data_chunk_recv_callback(nghttp2_session *inner_session, } if (stream->request.body == NULL) { - stream->request.body = cfl_sds_create_size(len); + /* + * Pre-allocate body buffer to avoid repeated reallocs as 16KB HTTP/2 + * frames arrive. Three strategies, in priority order: + * 1. Content-Length header — exact size (non-gRPC HTTP/2) + * 2. gRPC 5-byte frame header — parse message length for exact size + * 3. Fallback to frame size (len) — original behavior + */ + size_t initial_size; + + if (stream->request.content_length > 0) { + initial_size = stream->request.content_length; + } + else if (len >= 5 && + stream->request.content_type != NULL && + strncmp(stream->request.content_type, "application/grpc", 16) == 0) { + /* gRPC length-prefixed frame: 1 byte flags + 4 bytes message length */ + uint32_t grpc_msg_len = ((uint32_t) data[1] << 24) | + ((uint32_t) data[2] << 16) | + ((uint32_t) data[3] << 8) | + ((uint32_t) data[4]); + if ((size_t) grpc_msg_len > SIZE_MAX - 5) { + stream->status = HTTP_STREAM_STATUS_ERROR; + return -1; + } + initial_size = (size_t) grpc_msg_len + 5; + } + else { + initial_size = len; + } + + if (initial_size < len) { + stream->status = HTTP_STREAM_STATUS_ERROR; + return -1; + } + + if (http2_request_body_limit_exceeded(stream, initial_size)) { + stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } + + stream->request.body = cfl_sds_create_size(initial_size); if (stream->request.body == NULL) { stream->status = HTTP_STREAM_STATUS_ERROR;