From e9862305da6732a00bc434a632520b7bfe0cc4ca Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 4 Jun 2026 22:38:56 -0400 Subject: [PATCH 01/15] feat(gateways): add Kafka wire protocol listener foundation TCP listener on 9093 with scoped API decode/validation and stub responses for apache/iggy#3421. Includes kafka-message-gen test tool. Co-authored-by: Cursor --- .gitignore | 1 + Cargo.lock | 66 ++ Cargo.toml | 2 + gateways/README.md | 9 + gateways/kafka/Cargo.toml | 49 ++ gateways/kafka/README.md | 35 + gateways/kafka/docs/SCOPE.md | 24 + gateways/kafka/src/error.rs | 41 + gateways/kafka/src/lib.rs | 33 + gateways/kafka/src/main.rs | 45 + gateways/kafka/src/protocol/api.rs | 253 ++++++ gateways/kafka/src/protocol/codec.rs | 264 ++++++ gateways/kafka/src/protocol/header.rs | 191 +++++ gateways/kafka/src/protocol/mod.rs | 22 + gateways/kafka/src/protocol/requests.rs | 450 ++++++++++ gateways/kafka/src/protocol/responses.rs | 274 +++++++ gateways/kafka/src/server.rs | 207 +++++ gateways/kafka/tests/api_handler_tests.rs | 144 ++++ gateways/kafka/tests/codec_tests.rs | 149 ++++ .../kafka/tests/decode_validation_tests.rs | 449 ++++++++++ .../kafka/tests/golden_wire_fixtures_tests.rs | 75 ++ gateways/kafka/tests/header_tests.rs | 139 ++++ .../kafka/tests/server_integration_tests.rs | 110 +++ gateways/kafka/tools/kafka-tool/Cargo.toml | 43 + gateways/kafka/tools/kafka-tool/README.md | 249 ++++++ gateways/kafka/tools/kafka-tool/src/main.rs | 770 ++++++++++++++++++ 26 files changed, 4094 insertions(+) create mode 100644 gateways/README.md create mode 100644 gateways/kafka/Cargo.toml create mode 100644 gateways/kafka/README.md create mode 100644 gateways/kafka/docs/SCOPE.md create mode 100644 gateways/kafka/src/error.rs create mode 100644 gateways/kafka/src/lib.rs create mode 100644 gateways/kafka/src/main.rs create mode 100644 gateways/kafka/src/protocol/api.rs create mode 100644 gateways/kafka/src/protocol/codec.rs create mode 100644 gateways/kafka/src/protocol/header.rs create mode 100644 gateways/kafka/src/protocol/mod.rs create mode 100644 gateways/kafka/src/protocol/requests.rs create mode 100644 gateways/kafka/src/protocol/responses.rs create mode 100644 gateways/kafka/src/server.rs create mode 100644 gateways/kafka/tests/api_handler_tests.rs create mode 100644 gateways/kafka/tests/codec_tests.rs create mode 100644 gateways/kafka/tests/decode_validation_tests.rs create mode 100644 gateways/kafka/tests/golden_wire_fixtures_tests.rs create mode 100644 gateways/kafka/tests/header_tests.rs create mode 100644 gateways/kafka/tests/server_integration_tests.rs create mode 100644 gateways/kafka/tools/kafka-tool/Cargo.toml create mode 100644 gateways/kafka/tools/kafka-tool/README.md create mode 100644 gateways/kafka/tools/kafka-tool/src/main.rs diff --git a/.gitignore b/.gitignore index f0d8b6250b..8c9ec60fc6 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ go.work core/bench/dashboard/frontend/dist LICENSE-binary **/LICENSE-binary +gateways/kafka/tools/kafka-tool/kafka_messages/ diff --git a/Cargo.lock b/Cargo.lock index 5dcfa64639..0d9b4e8efb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7079,6 +7079,17 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "iggy_gateway_kafka" +version = "0.1.0" +dependencies = [ + "bytes", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "ignore" version = "0.4.25" @@ -7566,6 +7577,42 @@ dependencies = [ "rayon", ] +[[package]] +name = "kafka-message-gen" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "clap", + "hex", + "indexmap 2.14.0", + "kafka-protocol", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "kafka-protocol" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66292444a1cd4d430d450d472c30cba839d0724229aba2d79affffcf901516e2" +dependencies = [ + "anyhow", + "bytes", + "crc", + "crc32c", + "flate2", + "indexmap 2.14.0", + "lz4", + "paste", + "snap", + "uuid", + "zstd", +] + [[package]] name = "keccak" version = "0.2.0" @@ -8023,6 +8070,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lz4_flex" version = "0.12.2" diff --git a/Cargo.toml b/Cargo.toml index 2e2212f49a..322d6e488b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ members = [ "core/simulator", "core/tools", "examples/rust", + "gateways/kafka", + "gateways/kafka/tools/kafka-tool", ] exclude = ["foreign/cpp", "foreign/php", "foreign/python"] resolver = "2" diff --git a/gateways/README.md b/gateways/README.md new file mode 100644 index 0000000000..7b0c550f5e --- /dev/null +++ b/gateways/README.md @@ -0,0 +1,9 @@ +# Apache Iggy Gateways + +Protocol gateways that let existing clients talk to Iggy without changing the core server wire surface. + +| Gateway | Issue | Description | +|---------|-------|-------------| +| [kafka](kafka/) | [#3421](https://github.com/apache/iggy/issues/3421) | Kafka wire protocol TCP listener (port 9093) | + +Each gateway is a separate workspace crate under `gateways//`. diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml new file mode 100644 index 0000000000..6bfcb0e34d --- /dev/null +++ b/gateways/kafka/Cargo.toml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "iggy_gateway_kafka" +version = "0.1.0" +description = "Kafka wire protocol gateway foundation for Apache Iggy" +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "kafka", "gateway", "streaming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +readme = "README.md" +publish = false + +[[bin]] +name = "iggy-kafka-gateway" +path = "src/main.rs" + +[dependencies] +bytes = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time", "sync", "signal"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time"] } + +[lints.clippy] +enum_glob_use = "deny" +# Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. +pedantic = "warn" +nursery = "warn" diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md new file mode 100644 index 0000000000..b575a0c803 --- /dev/null +++ b/gateways/kafka/README.md @@ -0,0 +1,35 @@ +# Kafka gateway (`iggy_gateway_kafka`) + +Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421): a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, and returns stub responses. + +## Run + +```bash +cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway +``` + +Default bind: `127.0.0.1:9093`. + +## Test + +```bash +cargo test -p iggy_gateway_kafka +``` + +`decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/`: + +```bash +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 19 +``` + +(Run from workspace root; adjust paths if needed.) + +## Scoped APIs + +See [docs/SCOPE.md](docs/SCOPE.md). + +## Wire fixture tool + +See [tools/kafka-tool/README.md](tools/kafka-tool/README.md). diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md new file mode 100644 index 0000000000..b56a172730 --- /dev/null +++ b/gateways/kafka/docs/SCOPE.md @@ -0,0 +1,24 @@ +# Kafka API scope — issue #3421 foundation + +This gateway iteration implements **wire validation and stub responses only** (no Iggy backend, no real broker semantics). + +## Supported API keys and versions + +| API key | Name | Min version | Max version | Behavior | +|---------|------|-------------|-------------|----------| +| 18 | ApiVersions | 0 | 3 | Advertise supported ranges; flexible encoding at v3+ | +| 3 | Metadata | 0 | 9 | Decode request; stub broker `127.0.0.1:9093` | +| 0 | Produce | 3 | 9 | Decode request; stub response | +| 1 | Fetch | 4 | 12 | Decode request; stub response | +| 2 | ListOffsets | 1 | 6 | Decode request; stub response | +| 19 | CreateTopics | 2 | 5 | Decode request; stub response | + +## Unsupported API keys + +All other API keys receive an error-only response with `UNSUPPORTED_VERSION` (35). + +## Out of scope (later issues) + +- `IggyBridge` / produce-fetch against Iggy streams +- Consumer group APIs (8–14), SASL (17, 36), transactions +- Accurate metadata topology and record batch semantics diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs new file mode 100644 index 0000000000..668fc5c442 --- /dev/null +++ b/gateways/kafka/src/error.rs @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum KafkaProtocolError { + #[error("buffer underflow: needed {needed} bytes, remaining {remaining}")] + BufferUnderflow { needed: usize, remaining: usize }, + #[error("invalid frame length: {0}")] + InvalidFrameLength(i32), + #[error("request exceeds max frame size ({max_bytes} bytes): {actual_bytes} bytes")] + FrameTooLarge { + max_bytes: usize, + actual_bytes: usize, + }, + #[error("invalid utf8 string")] + InvalidUtf8, + #[error("varint overflows 64 bits")] + InvalidVarint, + #[error("unsupported request header version: {0}")] + UnsupportedHeaderVersion(i16), + #[error("io error: {0}")] + Io(#[from] std::io::Error), +} + +pub type Result = std::result::Result; diff --git a/gateways/kafka/src/lib.rs b/gateways/kafka/src/lib.rs new file mode 100644 index 0000000000..f0828508c7 --- /dev/null +++ b/gateways/kafka/src/lib.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka wire protocol gateway foundation for Apache Iggy. + +// Ported wire codec from spike; pedantic clippy cleanup is a follow-up. +#![allow( + clippy::pedantic, + clippy::missing_const_for_fn, + clippy::wildcard_imports, + clippy::match_same_arms, + clippy::needless_pass_by_value +)] + +pub mod error; +pub mod protocol; +pub mod server; + +pub use server::{KafkaServer, ServerConfig, init_tracing}; diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs new file mode 100644 index 0000000000..4a5414b9dd --- /dev/null +++ b/gateways/kafka/src/main.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use tokio::signal; +use tokio::sync::broadcast; + +use iggy_gateway_kafka::server::init_tracing; +use iggy_gateway_kafka::{KafkaServer, ServerConfig}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + init_tracing(); + + let config = ServerConfig::default(); + let server = KafkaServer::new(config); + + let (tx, rx) = broadcast::channel(1); + let mut server_task = tokio::spawn(async move { server.run(rx).await }); + + tokio::select! { + result = &mut server_task => { + return Ok(result??); + } + _ = signal::ctrl_c() => { + let _ = tx.send(()); + } + } + + server_task.await??; + Ok(()) +} diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs new file mode 100644 index 0000000000..8f0367b481 --- /dev/null +++ b/gateways/kafka/src/protocol/api.rs @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use crate::protocol::codec::{Decoder, Encoder}; +use crate::protocol::requests::{ + decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, + decode_produce_request, +}; +use crate::protocol::responses::{ + encode_create_topics_response, encode_fetch_response, encode_list_offsets_response, + encode_produce_response, +}; + +pub const API_KEY_PRODUCE: i16 = 0; +pub const API_KEY_FETCH: i16 = 1; +pub const API_KEY_LIST_OFFSETS: i16 = 2; +pub const API_KEY_METADATA: i16 = 3; +pub const API_KEY_OFFSET_COMMIT: i16 = 8; +pub const API_KEY_OFFSET_FETCH: i16 = 9; +pub const API_KEY_FIND_COORDINATOR: i16 = 10; +pub const API_KEY_JOIN_GROUP: i16 = 11; +pub const API_KEY_HEARTBEAT: i16 = 12; +pub const API_KEY_LEAVE_GROUP: i16 = 13; +pub const API_KEY_SYNC_GROUP: i16 = 14; +pub const API_KEY_DESCRIBE_GROUPS: i16 = 15; +pub const API_KEY_LIST_GROUPS: i16 = 16; +pub const API_KEY_SASL_HANDSHAKE: i16 = 17; +pub const API_KEY_API_VERSIONS: i16 = 18; +pub const API_KEY_CREATE_TOPICS: i16 = 19; +pub const API_KEY_DELETE_TOPICS: i16 = 20; + +pub const ERROR_NONE: i16 = 0; +pub const ERROR_OFFSET_OUT_OF_RANGE: i16 = 1; +pub const ERROR_CORRUPT_MESSAGE: i16 = 2; +pub const ERROR_UNKNOWN_TOPIC_OR_PARTITION: i16 = 3; +pub const ERROR_INVALID_FETCH_SIZE: i16 = 4; +pub const ERROR_LEADER_NOT_AVAILABLE: i16 = 5; +pub const ERROR_NOT_LEADER_OR_FOLLOWER: i16 = 6; +pub const ERROR_REQUEST_TIMED_OUT: i16 = 7; +pub const ERROR_UNKNOWN_SERVER_ERROR: i16 = -1; +pub const ERROR_UNSUPPORTED_VERSION: i16 = 35; +pub const ERROR_TOPIC_ALREADY_EXISTS: i16 = 36; +pub const ERROR_INVALID_PARTITIONS: i16 = 37; +pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; +pub const ERROR_INVALID_REQUEST: i16 = 42; +pub const ERROR_UNSUPPORTED_FOR_MESSAGE_FORMAT: i16 = 43; + +#[derive(Debug, Clone, Copy)] +pub struct ApiVersionRange { + pub api_key: i16, + pub min_version: i16, + pub max_version: i16, +} + +pub fn supported_api_ranges() -> Vec { + vec![ + ApiVersionRange { + api_key: API_KEY_PRODUCE, + min_version: 3, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_FETCH, + min_version: 4, + max_version: 12, + }, + ApiVersionRange { + api_key: API_KEY_LIST_OFFSETS, + min_version: 1, + max_version: 6, + }, + ApiVersionRange { + api_key: API_KEY_METADATA, + min_version: 0, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_API_VERSIONS, + min_version: 0, + max_version: 3, + }, + ApiVersionRange { + api_key: API_KEY_CREATE_TOPICS, + min_version: 2, + max_version: 5, + }, + ] +} + +pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { + match api_key { + API_KEY_API_VERSIONS => { + if is_supported_version(api_key, api_version) { + encode_api_versions_response(api_version, ERROR_NONE) + } else { + encode_api_versions_response(1, ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_METADATA => { + if is_supported_version(api_key, api_version) { + encode_metadata_response(api_version, body, ERROR_NONE) + } else { + encode_metadata_response(0, body, ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_PRODUCE => { + if is_supported_version(api_key, api_version) { + match decode_produce_request(api_version, body) { + Ok(req) => encode_produce_response(api_version, req), + Err(e) => { + tracing::error!("Failed to decode Produce request: {:?}", e); + encode_error_only_response(ERROR_CORRUPT_MESSAGE) + } + } + } else { + encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_FETCH => { + if is_supported_version(api_key, api_version) { + match decode_fetch_request(api_version, body) { + Ok(req) => encode_fetch_response(api_version, req), + Err(e) => { + tracing::error!("Failed to decode Fetch request: {:?}", e); + encode_error_only_response(ERROR_CORRUPT_MESSAGE) + } + } + } else { + encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_LIST_OFFSETS => { + if is_supported_version(api_key, api_version) { + match decode_list_offsets_request(api_version, body) { + Ok(req) => encode_list_offsets_response(api_version, req), + Err(e) => { + tracing::error!("Failed to decode ListOffsets request: {:?}", e); + encode_error_only_response(ERROR_CORRUPT_MESSAGE) + } + } + } else { + encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_CREATE_TOPICS => { + if is_supported_version(api_key, api_version) { + match decode_create_topics_request(api_version, body) { + Ok(req) => encode_create_topics_response(api_version, req), + Err(e) => { + tracing::error!("Failed to decode CreateTopics request: {:?}", e); + encode_error_only_response(ERROR_CORRUPT_MESSAGE) + } + } + } else { + encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + } + } + _ => encode_error_only_response(ERROR_UNSUPPORTED_VERSION), + } +} + +pub fn is_supported_version(api_key: i16, api_version: i16) -> bool { + supported_api_ranges() + .into_iter() + .find(|r| r.api_key == api_key) + .is_some_and(|r| api_version >= r.min_version && api_version <= r.max_version) +} + +fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { + let flexible = api_version >= 3; + let ranges = supported_api_ranges(); + let mut e = Encoder::with_capacity(128); + + e.write_i16(error_code); + + if flexible { + e.write_varint((ranges.len() + 1) as u64); + for r in &ranges { + e.write_i16(r.api_key); + e.write_i16(r.min_version); + e.write_i16(r.max_version); + e.write_empty_tagged_fields(); + } + } else { + e.write_i32(ranges.len() as i32); + for r in &ranges { + e.write_i16(r.api_key); + e.write_i16(r.min_version); + e.write_i16(r.max_version); + } + } + + if api_version >= 1 { + e.write_i32(0); + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +fn encode_metadata_response(_api_version: i16, body: Bytes, top_level_error_code: i16) -> Bytes { + let mut e = Encoder::with_capacity(256); + + e.write_i32(1); + e.write_i32(1); + e.write_nullable_string(Some("127.0.0.1")); + e.write_i32(9093); + + let topics_count = split_metadata_request_topics(body); + e.write_i32(topics_count as i32); + for _ in 0..topics_count { + e.write_i16(if top_level_error_code == ERROR_NONE { + ERROR_UNKNOWN_TOPIC_OR_PARTITION + } else { + top_level_error_code + }); + e.write_nullable_string(Some("unknown-topic")); + e.write_i32(0); + } + + e.write_i32(1); + e.freeze() +} + +fn encode_error_only_response(error_code: i16) -> Bytes { + let mut e = Encoder::with_capacity(2); + e.write_i16(error_code); + e.freeze() +} + +pub fn split_metadata_request_topics(body: Bytes) -> usize { + let mut d = Decoder::new(body); + d.read_i32().unwrap_or_default().max(0) as usize +} diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs new file mode 100644 index 0000000000..1e356b8608 --- /dev/null +++ b/gateways/kafka/src/protocol/codec.rs @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::{Buf, BufMut, Bytes, BytesMut}; + +use crate::error::{KafkaProtocolError, Result}; + +pub struct Decoder { + bytes: Bytes, +} + +impl Decoder { + pub fn new(bytes: Bytes) -> Self { + Self { bytes } + } + + pub fn remaining(&self) -> usize { + self.bytes.remaining() + } + + pub fn read_u8(&mut self) -> Result { + self.ensure(1)?; + Ok(self.bytes.get_u8()) + } + + pub fn read_i8(&mut self) -> Result { + self.ensure(1)?; + Ok(self.bytes.get_i8()) + } + + pub fn read_i16(&mut self) -> Result { + self.ensure(2)?; + Ok(self.bytes.get_i16()) + } + + pub fn read_i32(&mut self) -> Result { + self.ensure(4)?; + Ok(self.bytes.get_i32()) + } + + pub fn read_i64(&mut self) -> Result { + self.ensure(8)?; + Ok(self.bytes.get_i64()) + } + + pub fn read_bool(&mut self) -> Result { + Ok(self.read_i8()? != 0) + } + + /// Unsigned varint (Kafka uses this for compact array lengths and tagged-field counts). + /// Value is encoded with 7 bits per byte, LSB first; the high bit of each byte signals + /// that more bytes follow. + pub fn read_varint(&mut self) -> Result { + let mut result: u64 = 0; + let mut shift = 0u32; + loop { + let byte = self.read_u8()?; + result |= ((byte & 0x7F) as u64) << shift; + if byte & 0x80 == 0 { + return Ok(result); + } + shift += 7; + if shift >= 64 { + return Err(KafkaProtocolError::InvalidVarint); + } + } + } + + /// Legacy nullable string: i16 length prefix (-1 = null). + pub fn read_nullable_string(&mut self) -> Result> { + let len = self.read_i16()?; + if len < 0 { + return Ok(None); + } + let len = len as usize; + self.ensure(len)?; + let chunk = self.bytes.copy_to_bytes(len); + String::from_utf8(chunk.to_vec()) + .map(Some) + .map_err(|_| KafkaProtocolError::InvalidUtf8) + } + + /// Compact nullable string (flexible versions): varint(len+1) prefix, 0 = null. + pub fn read_compact_nullable_string(&mut self) -> Result> { + let len_plus_one = self.read_varint()?; + if len_plus_one == 0 { + return Ok(None); + } + let len = (len_plus_one - 1) as usize; + self.ensure(len)?; + let chunk = self.bytes.copy_to_bytes(len); + String::from_utf8(chunk.to_vec()) + .map(Some) + .map_err(|_| KafkaProtocolError::InvalidUtf8) + } + + /// Legacy nullable bytes: i32 length prefix (-1 = null). + pub fn read_nullable_bytes(&mut self) -> Result> { + let len = self.read_i32()?; + if len < 0 { + return Ok(None); + } + let len = len as usize; + self.ensure(len)?; + Ok(Some(self.bytes.copy_to_bytes(len))) + } + + /// Compact nullable bytes (flexible versions): varint(len+1) prefix, 0 = null. + pub fn read_compact_nullable_bytes(&mut self) -> Result> { + let len_plus_one = self.read_varint()?; + if len_plus_one == 0 { + return Ok(None); + } + let len = (len_plus_one - 1) as usize; + self.ensure(len)?; + Ok(Some(self.bytes.copy_to_bytes(len))) + } + + pub fn read_bytes(&mut self, len: usize) -> Result { + self.ensure(len)?; + Ok(self.bytes.copy_to_bytes(len)) + } + + /// Skip over a tagged-fields section. Each field is: tag (varint) + size (varint) + bytes. + /// A count of 0 is the common case (single byte 0x00). + pub fn read_tagged_fields(&mut self) -> Result<()> { + let count = self.read_varint()? as usize; + for _ in 0..count { + self.read_varint()?; // tag number + let size = self.read_varint()? as usize; + self.ensure(size)?; + self.bytes.advance(size); + } + Ok(()) + } + + fn ensure(&self, needed: usize) -> Result<()> { + let remaining = self.bytes.remaining(); + if remaining < needed { + return Err(KafkaProtocolError::BufferUnderflow { needed, remaining }); + } + Ok(()) + } +} + +pub struct Encoder { + bytes: BytesMut, +} + +impl Encoder { + pub fn with_capacity(capacity: usize) -> Self { + Self { + bytes: BytesMut::with_capacity(capacity), + } + } + + pub fn write_u8(&mut self, v: u8) { + self.bytes.put_u8(v); + } + + pub fn write_i8(&mut self, v: i8) { + self.bytes.put_i8(v); + } + + pub fn write_i16(&mut self, v: i16) { + self.bytes.put_i16(v); + } + + pub fn write_i32(&mut self, v: i32) { + self.bytes.put_i32(v); + } + + pub fn write_i64(&mut self, v: i64) { + self.bytes.put_i64(v); + } + + pub fn write_bool(&mut self, v: bool) { + self.write_i8(if v { 1 } else { 0 }); + } + + /// Unsigned varint, 7 bits per byte, LSB first. + pub fn write_varint(&mut self, mut v: u64) { + loop { + let byte = (v & 0x7F) as u8; + v >>= 7; + if v == 0 { + self.bytes.put_u8(byte); + return; + } + self.bytes.put_u8(byte | 0x80); + } + } + + /// Legacy nullable string: i16 length prefix, -1 for null. + pub fn write_nullable_string(&mut self, v: Option<&str>) { + match v { + None => self.write_i16(-1), + Some(s) => { + self.write_i16(s.len() as i16); + self.bytes.put_slice(s.as_bytes()); + } + } + } + + /// Compact nullable string (flexible versions): varint(len+1), 0 for null. + pub fn write_compact_nullable_string(&mut self, v: Option<&str>) { + match v { + None => self.write_varint(0), + Some(s) => { + self.write_varint((s.len() + 1) as u64); + self.bytes.put_slice(s.as_bytes()); + } + } + } + + /// Legacy nullable bytes: i32 length prefix, -1 for null. + pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) { + match v { + None => self.write_i32(-1), + Some(b) => { + self.write_i32(b.len() as i32); + self.bytes.put_slice(b); + } + } + } + + /// Compact nullable bytes (flexible versions): varint(len+1), 0 for null. + pub fn write_compact_nullable_bytes(&mut self, v: Option<&[u8]>) { + match v { + None => self.write_varint(0), + Some(b) => { + self.write_varint((b.len() + 1) as u64); + self.bytes.put_slice(b); + } + } + } + + pub fn write_bytes(&mut self, b: &[u8]) { + self.bytes.put_slice(b); + } + + /// Write an empty tagged-fields section (single 0x00 byte). + pub fn write_empty_tagged_fields(&mut self) { + self.write_varint(0); + } + + pub fn freeze(self) -> Bytes { + self.bytes.freeze() + } +} diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs new file mode 100644 index 0000000000..fb137b81e0 --- /dev/null +++ b/gateways/kafka/src/protocol/header.rs @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use crate::error::{KafkaProtocolError, Result}; +use crate::protocol::codec::{Decoder, Encoder}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RequestHeader { + pub api_key: i16, + pub api_version: i16, + pub correlation_id: i32, + pub client_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResponseHeader { + pub correlation_id: i32, +} + +/// Returns the request header version to use for a given (api_key, api_version) pair. +/// +/// Header v1 is the standard non-flexible format (nullable string client_id). +/// Header v2 is the flexible format (compact nullable string client_id + empty tagged fields). +/// The threshold at which each API key switches from v1 to v2 is defined by the Kafka protocol. +pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { + let flexible_from: i16 = match api_key { + 0 => 9, // Produce + 1 => 12, // Fetch + 2 => 6, // ListOffsets + 3 => 9, // Metadata + 4 => 4, // LeaderAndIsr + 5 => 2, // StopReplica + 6 => 6, // UpdateMetadata + 7 => 3, // ControlledShutdown + 8 => 8, // OffsetCommit + 9 => 6, // OffsetFetch + 10 => 3, // FindCoordinator + 11 => 6, // JoinGroup + 12 => 4, // Heartbeat + 13 => 4, // LeaveGroup + 14 => 4, // SyncGroup + 15 => 5, // DescribeGroups + 16 => 3, // ListGroups + 17 => i16::MAX, // SaslHandshake — never flexible + 18 => 3, // ApiVersions + 19 => 5, // CreateTopics + 20 => 4, // DeleteTopics + 21 => 2, // DeleteRecords + 22 => 2, // InitProducerId + 23 => 4, // OffsetForLeaderEpoch + 24 => 3, // AddPartitionsToTxn + 25 => 3, // AddOffsetsToTxn + 26 => 3, // EndTxn + 27 => 1, // WriteTxnMarkers + 28 => 3, // TxnOffsetCommit + 29 => 2, // DescribeAcls + 30 => 2, // CreateAcls + 31 => 2, // DeleteAcls + 32 => 4, // DescribeConfigs + 33 => 2, // AlterConfigs + 34 => 2, // AlterReplicaLogDirs + 35 => 2, // DescribeLogDirs + 36 => 2, // SaslAuthenticate + 37 => 2, // CreatePartitions + 38 => 2, // CreateDelegationToken + 39 => 2, // RenewDelegationToken + 40 => 2, // ExpireDelegationToken + 41 => 2, // DescribeDelegationToken + 42 => 2, // DeleteGroups + 43 => 2, // ElectLeaders + 44 => 1, // IncrementalAlterConfigs + 45 => 0, // AlterPartitionReassignments — always flexible + 46 => 0, // ListPartitionReassignments — always flexible + 47 => i16::MAX, // OffsetDelete — never flexible + 48 => 1, // DescribeClientQuotas + 49 => 1, // AlterClientQuotas + 50 => 0, // DescribeUserScramCredentials — always flexible + 51 => 0, // AlterUserScramCredentials — always flexible + 55 => 0, // DescribeQuorum — always flexible + 56 => 0, // AlterPartition — always flexible + 57 => 1, // UpdateFeatures + 60 => 0, // DescribeCluster — always flexible + 61 => 0, // DescribeProducers — always flexible + 64 => 0, // UnregisterBroker — always flexible + 65 => 0, // DescribeTransactions — always flexible + 66 => 0, // ListTransactions — always flexible + 67 => 0, // AllocateProducerIds — always flexible + 68 => 0, // ConsumerGroupHeartbeat — always flexible + 69 => 0, // ConsumerGroupDescribe — always flexible + 71 => 0, // GetTelemetrySubscriptions — always flexible + 72 => 0, // PushTelemetry — always flexible + 74 => 0, // AssignReplicasToDirs — always flexible + 75 => 0, // DescribeTopicPartitions — always flexible + 76 => 0, // ListClientMetricsResources — always flexible + _ => i16::MAX, // Unknown API — assume non-flexible + }; + if api_version >= flexible_from { 2 } else { 1 } +} + +/// Returns the response header version to use when replying to a given (api_key, api_version). +/// +/// ApiVersions (18) is a special case: the server ALWAYS returns response header v0 (no tagged +/// fields) so that clients that don't yet know the server supports flexible encoding can still +/// parse the discovery response. All other flexible-version APIs use response header v1. +pub fn response_header_version(api_key: i16, api_version: i16) -> i16 { + if api_key == 18 { + return 0; + } + if request_header_version(api_key, api_version) >= 2 { + 1 + } else { + 0 + } +} + +impl RequestHeader { + pub fn decode(bytes: Bytes, header_version: i16) -> Result { + let mut d = Decoder::new(bytes); + Self::decode_from(&mut d, header_version) + } + + /// Decode from a shared `Decoder`. + /// + /// Header v1 (non-flexible): + /// api_key i16 | api_version i16 | correlation_id i32 | client_id NULLABLE_STRING + /// + /// Header v2 (flexible): + /// api_key i16 | api_version i16 | correlation_id i32 + /// | client_id COMPACT_NULLABLE_STRING | _tagged_fields UNSIGNED_VARINT + pub fn decode_from(d: &mut Decoder, header_version: i16) -> Result { + match header_version { + 1 => { + let api_key = d.read_i16()?; + let api_version = d.read_i16()?; + let correlation_id = d.read_i32()?; + let client_id = d.read_nullable_string()?; + Ok(Self { + api_key, + api_version, + correlation_id, + client_id, + }) + } + 2 => { + let api_key = d.read_i16()?; + let api_version = d.read_i16()?; + let correlation_id = d.read_i32()?; + let client_id = d.read_compact_nullable_string()?; + d.read_tagged_fields()?; + Ok(Self { + api_key, + api_version, + correlation_id, + client_id, + }) + } + v => Err(KafkaProtocolError::UnsupportedHeaderVersion(v)), + } + } +} + +impl ResponseHeader { + /// Encode the response header. + /// + /// v0: correlation_id i32 (non-flexible APIs and ApiVersions) + /// v1: correlation_id i32 + empty tagged fields (flexible APIs) + pub fn encode(&self, header_version: i16) -> Bytes { + let mut e = Encoder::with_capacity(5); + e.write_i32(self.correlation_id); + if header_version >= 1 { + e.write_empty_tagged_fields(); + } + e.freeze() + } +} diff --git a/gateways/kafka/src/protocol/mod.rs b/gateways/kafka/src/protocol/mod.rs new file mode 100644 index 0000000000..3fe1fb544f --- /dev/null +++ b/gateways/kafka/src/protocol/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod api; +pub mod codec; +pub mod header; +pub mod requests; +pub mod responses; diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs new file mode 100644 index 0000000000..786fa0b88c --- /dev/null +++ b/gateways/kafka/src/protocol/requests.rs @@ -0,0 +1,450 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka request decoders for critical API keys + +use crate::error::Result; +use crate::protocol::codec::Decoder; +use bytes::Bytes; + +/// Produce Request (API Key 0) +#[derive(Debug, Clone)] +pub struct ProduceRequest { + pub transactional_id: Option, + pub acks: i16, + pub timeout_ms: i32, + pub topics: Vec, +} + +#[derive(Debug, Clone)] +pub struct ProduceTopicData { + pub topic: String, + pub partitions: Vec, +} + +#[derive(Debug, Clone)] +pub struct ProducePartitionData { + pub partition: i32, + pub records: Option, // Raw RecordBatch bytes +} + +pub fn decode_produce_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 9; + + // transactional_id (v3+) + let transactional_id = if version >= 3 { + if flexible { + d.read_compact_nullable_string()? + } else { + d.read_nullable_string()? + } + } else { + None + }; + + let acks = d.read_i16()?; + let timeout_ms = d.read_i32()?; + + // topics array + let topics_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let topic = if flexible { + d.read_compact_nullable_string()?.unwrap_or_default() + } else { + d.read_nullable_string()?.unwrap_or_default() + }; + + let partitions_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut partitions = Vec::with_capacity(partitions_count); + for _ in 0..partitions_count { + let partition = d.read_i32()?; + let records = if flexible { + d.read_compact_nullable_bytes()? + } else { + d.read_nullable_bytes()? + }; + partitions.push(ProducePartitionData { partition, records }); + if flexible { + d.read_tagged_fields()?; + } + } + + topics.push(ProduceTopicData { topic, partitions }); + if flexible { + d.read_tagged_fields()?; + } + } + + if flexible { + d.read_tagged_fields()?; + } + + Ok(ProduceRequest { + transactional_id, + acks, + timeout_ms, + topics, + }) +} + +/// Fetch Request (API Key 1) +#[derive(Debug, Clone)] +pub struct FetchRequest { + pub max_wait_ms: i32, + pub min_bytes: i32, + pub max_bytes: i32, + pub isolation_level: i8, + pub topics: Vec, +} + +#[derive(Debug, Clone)] +pub struct FetchTopic { + pub topic: String, + pub partitions: Vec, +} + +#[derive(Debug, Clone)] +pub struct FetchPartition { + pub partition: i32, + pub fetch_offset: i64, + pub partition_max_bytes: i32, +} + +pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 12; + + let _replica_id = d.read_i32()?; + let max_wait_ms = d.read_i32()?; + let min_bytes = d.read_i32()?; + + let max_bytes = if version >= 3 { + d.read_i32()? + } else { + 52_428_800 // default 50MB + }; + + let isolation_level = if version >= 4 { d.read_i8()? } else { 0 }; + + // session_id and session_epoch (v7+) — we'll skip for now + if version >= 7 { + d.read_i32()?; // session_id + d.read_i32()?; // session_epoch + } + + // topics array + let topics_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let topic = if flexible { + d.read_compact_nullable_string()?.unwrap_or_default() + } else { + d.read_nullable_string()?.unwrap_or_default() + }; + + let partitions_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut partitions = Vec::with_capacity(partitions_count); + for _ in 0..partitions_count { + let partition = d.read_i32()?; + + if version >= 9 { + d.read_i32()?; // current_leader_epoch + } + + let fetch_offset = d.read_i64()?; + + if version >= 12 { + d.read_i32()?; // last_fetched_epoch + } + + if version >= 5 { + d.read_i64()?; // log_start_offset + } + + let partition_max_bytes = d.read_i32()?; + + partitions.push(FetchPartition { + partition, + fetch_offset, + partition_max_bytes, + }); + + if flexible { + d.read_tagged_fields()?; + } + } + + topics.push(FetchTopic { topic, partitions }); + if flexible { + d.read_tagged_fields()?; + } + } + + // forgotten_topics_data (v7+) — skip + if version >= 7 { + let forgotten_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + for _ in 0..forgotten_count { + if flexible { + d.read_compact_nullable_string()?; + let partitions_count = (d.read_varint()? - 1) as usize; + for _ in 0..partitions_count { + d.read_i32()?; + } + d.read_tagged_fields()?; + } else { + d.read_nullable_string()?; + let partitions_count = d.read_i32()? as usize; + for _ in 0..partitions_count { + d.read_i32()?; + } + } + } + } + + // rack_id (v11+) + if version >= 11 { + if flexible { + d.read_compact_nullable_string()?; + } else { + d.read_nullable_string()?; + } + } + + if flexible { + d.read_tagged_fields()?; + } + + Ok(FetchRequest { + max_wait_ms, + min_bytes, + max_bytes, + isolation_level, + topics, + }) +} + +/// ListOffsets Request (API Key 2) +#[derive(Debug, Clone)] +pub struct ListOffsetsRequest { + pub isolation_level: i8, + pub topics: Vec, +} + +#[derive(Debug, Clone)] +pub struct ListOffsetsTopic { + pub topic: String, + pub partitions: Vec, +} + +#[derive(Debug, Clone)] +pub struct ListOffsetsPartition { + pub partition: i32, + pub timestamp: i64, // -2 = earliest, -1 = latest +} + +pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 6; + + let _replica_id = d.read_i32()?; + + let isolation_level = if version >= 2 { d.read_i8()? } else { 0 }; + + let topics_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let topic = if flexible { + d.read_compact_nullable_string()?.unwrap_or_default() + } else { + d.read_nullable_string()?.unwrap_or_default() + }; + + let partitions_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut partitions = Vec::with_capacity(partitions_count); + for _ in 0..partitions_count { + let partition = d.read_i32()?; + + if version >= 4 { + d.read_i32()?; // current_leader_epoch + } + + let timestamp = d.read_i64()?; + + if version == 0 { + d.read_i32()?; // max_num_offsets (deprecated) + } + + partitions.push(ListOffsetsPartition { + partition, + timestamp, + }); + + if flexible { + d.read_tagged_fields()?; + } + } + + topics.push(ListOffsetsTopic { topic, partitions }); + if flexible { + d.read_tagged_fields()?; + } + } + + if flexible { + d.read_tagged_fields()?; + } + + Ok(ListOffsetsRequest { + isolation_level, + topics, + }) +} + +/// CreateTopics Request (API Key 19) +#[derive(Debug, Clone)] +pub struct CreateTopicsRequest { + pub topics: Vec, + pub timeout_ms: i32, + pub validate_only: bool, +} + +#[derive(Debug, Clone)] +pub struct CreatableTopic { + pub name: String, + pub num_partitions: i32, + pub replication_factor: i16, +} + +pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 5; + + let topics_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let name = if flexible { + d.read_compact_nullable_string()?.unwrap_or_default() + } else { + d.read_nullable_string()?.unwrap_or_default() + }; + + let num_partitions = d.read_i32()?; + let replication_factor = d.read_i16()?; + + // assignments (COMPACT_ARRAY or ARRAY) — skip + let assignments_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + for _ in 0..assignments_count { + d.read_i32()?; // partition_index + let replicas_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + for _ in 0..replicas_count { + d.read_i32()?; // broker_id + } + if flexible { + d.read_tagged_fields()?; + } + } + + // configs (COMPACT_ARRAY or ARRAY) — skip + let configs_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + for _ in 0..configs_count { + if flexible { + d.read_compact_nullable_string()?; // name + d.read_compact_nullable_string()?; // value + d.read_tagged_fields()?; + } else { + d.read_nullable_string()?; + d.read_nullable_string()?; + } + } + + topics.push(CreatableTopic { + name, + num_partitions, + replication_factor, + }); + + if flexible { + d.read_tagged_fields()?; + } + } + + let timeout_ms = d.read_i32()?; + let validate_only = if version >= 1 { d.read_bool()? } else { false }; + + if flexible { + d.read_tagged_fields()?; + } + + Ok(CreateTopicsRequest { + topics, + timeout_ms, + validate_only, + }) +} diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs new file mode 100644 index 0000000000..55f332b273 --- /dev/null +++ b/gateways/kafka/src/protocol/responses.rs @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka response encoders (stub implementations — will call Iggy SDK in production) + +use crate::protocol::api::*; +use crate::protocol::codec::Encoder; +use crate::protocol::requests::*; +use bytes::Bytes; + +pub fn encode_produce_response(version: i16, req: ProduceRequest) -> Bytes { + let flexible = version >= 9; + let mut e = Encoder::with_capacity(512); + + if flexible { + e.write_varint((req.topics.len() + 1) as u64); + } else { + e.write_i32(req.topics.len() as i32); + } + + for topic in &req.topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.topic)); + } else { + e.write_nullable_string(Some(&topic.topic)); + } + + if flexible { + e.write_varint((topic.partitions.len() + 1) as u64); + } else { + e.write_i32(topic.partitions.len() as i32); + } + + for p in &topic.partitions { + e.write_i32(p.partition); + e.write_i16(ERROR_NONE); + e.write_i64(0); // base_offset — TODO: return real offset from Iggy + if version >= 2 { + e.write_i64(-1); // log_append_time_ms (-1 = not set) + } + if version >= 5 { + e.write_i64(0); // log_start_offset + } + // record_errors[] and error_message added in v8 + if version >= 8 { + if flexible { + e.write_varint(1); // empty COMPACT_ARRAY + e.write_compact_nullable_string(None); // error_message = null + } else { + e.write_i32(0); // empty ARRAY + e.write_nullable_string(None); // error_message = null + } + } + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if version >= 1 { + e.write_i32(0); // throttle_time_ms + } + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +pub fn encode_fetch_response(version: i16, req: FetchRequest) -> Bytes { + let flexible = version >= 12; + let mut e = Encoder::with_capacity(512); + + if version >= 1 { + e.write_i32(0); // throttle_time_ms + } + if version >= 7 { + e.write_i16(ERROR_NONE); // error_code + e.write_i32(0); // session_id + } + + if flexible { + e.write_varint((req.topics.len() + 1) as u64); + } else { + e.write_i32(req.topics.len() as i32); + } + + for topic in &req.topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.topic)); + } else { + e.write_nullable_string(Some(&topic.topic)); + } + + if flexible { + e.write_varint((topic.partitions.len() + 1) as u64); + } else { + e.write_i32(topic.partitions.len() as i32); + } + + for partition in &topic.partitions { + e.write_i32(partition.partition); + e.write_i16(ERROR_NONE); + e.write_i64(0); // high_watermark — TODO: get from Iggy + if version >= 4 { + e.write_i64(0); // last_stable_offset + } + if version >= 5 { + e.write_i64(0); // log_start_offset + } + if version >= 4 { + // aborted_transactions[] + if flexible { + e.write_varint(1); + } else { + e.write_i32(0); + } + } + if version >= 11 { + e.write_i32(-1); // preferred_read_replica + } + // records (empty — TODO: call Iggy poll_messages) + if flexible { + e.write_compact_nullable_bytes(None); + } else { + e.write_nullable_bytes(None); + } + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +pub fn encode_list_offsets_response(version: i16, req: ListOffsetsRequest) -> Bytes { + let flexible = version >= 6; + let mut e = Encoder::with_capacity(256); + + if version >= 2 { + e.write_i32(0); // throttle_time_ms + } + + if flexible { + e.write_varint((req.topics.len() + 1) as u64); + } else { + e.write_i32(req.topics.len() as i32); + } + + for topic in &req.topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.topic)); + } else { + e.write_nullable_string(Some(&topic.topic)); + } + + if flexible { + e.write_varint((topic.partitions.len() + 1) as u64); + } else { + e.write_i32(topic.partitions.len() as i32); + } + + for partition in &topic.partitions { + e.write_i32(partition.partition); + e.write_i16(ERROR_NONE); + + // TODO: query Iggy for actual offsets + let offset = 0i64; + if version >= 1 { + e.write_i64(1_700_000_000_000); // timestamp placeholder + } + e.write_i64(offset); + // leader_epoch was added in v4, not v1 + if version >= 4 { + e.write_i32(-1); + } + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +pub fn encode_create_topics_response(version: i16, req: CreateTopicsRequest) -> Bytes { + let flexible = version >= 5; + let mut e = Encoder::with_capacity(256); + + if version >= 2 { + e.write_i32(0); // throttle_time_ms + } + + if flexible { + e.write_varint((req.topics.len() + 1) as u64); + } else { + e.write_i32(req.topics.len() as i32); + } + + for topic in &req.topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.name)); + } else { + e.write_nullable_string(Some(&topic.name)); + } + + let error_code = if topic.num_partitions <= 0 { + ERROR_INVALID_PARTITIONS + } else { + ERROR_NONE + }; + e.write_i16(error_code); + + if version >= 1 { + if flexible { + e.write_compact_nullable_string(None); // error_message + } else { + e.write_nullable_string(None); + } + } + + if version >= 5 { + e.write_i16(ERROR_NONE); // topic_config_error_code (added in v5) + e.write_i32(topic.num_partitions); + e.write_i16(topic.replication_factor); + e.write_varint(1); // configs[] empty COMPACT_ARRAY + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs new file mode 100644 index 0000000000..c6465eb0ca --- /dev/null +++ b/gateways/kafka/src/server.rs @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use bytes::{BufMut, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::broadcast; +use tokio::time::timeout; +use tracing::{error, info, warn}; + +use crate::error::{KafkaProtocolError, Result}; +use crate::protocol::api::handle_request; +use crate::protocol::codec::Decoder; +use crate::protocol::header::{ + RequestHeader, ResponseHeader, request_header_version, response_header_version, +}; + +#[derive(Debug, Clone)] +pub struct ServerConfig { + pub bind_addr: String, + pub max_frame_size: usize, + pub read_timeout: Duration, + pub write_timeout: Duration, +} + +impl Default for ServerConfig { + fn default() -> Self { + Self { + bind_addr: "127.0.0.1:9093".to_string(), + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(15), + write_timeout: Duration::from_secs(10), + } + } +} + +pub struct KafkaServer { + config: Arc, +} + +impl KafkaServer { + pub fn new(config: ServerConfig) -> Self { + Self { + config: Arc::new(config), + } + } + + pub async fn run(self, mut shutdown: broadcast::Receiver<()>) -> Result<()> { + let listener = TcpListener::bind(&self.config.bind_addr).await?; + info!("kafka listener bound on {}", self.config.bind_addr); + + loop { + tokio::select! { + _ = shutdown.recv() => { + info!("kafka listener shutdown requested"); + break; + } + accept_result = listener.accept() => { + let (stream, peer) = accept_result?; + let cfg = Arc::clone(&self.config); + tokio::spawn(async move { + if let Err(err) = handle_connection(stream, cfg, peer).await { + warn!(%peer, "connection closed with error: {err}"); + } + }); + } + } + } + Ok(()) + } +} + +async fn handle_connection( + mut stream: TcpStream, + config: Arc, + peer: SocketAddr, +) -> Result<()> { + info!(%peer, "connection accepted"); + + loop { + let frame = match read_frame(&mut stream, config.max_frame_size, config.read_timeout).await + { + Ok(f) => f, + Err(KafkaProtocolError::Io(ref e)) + if e.kind() == std::io::ErrorKind::UnexpectedEof + || e.kind() == std::io::ErrorKind::ConnectionReset => + { + info!(%peer, "connection closed by client"); + return Ok(()); + } + Err(e) => return Err(e), + }; + + if frame.len() < 4 { + return Err(KafkaProtocolError::BufferUnderflow { + needed: 4, + remaining: frame.len(), + }); + } + let api_key = i16::from_be_bytes([frame[0], frame[1]]); + let api_version = i16::from_be_bytes([frame[2], frame[3]]); + let req_hdr_ver = request_header_version(api_key, api_version); + let resp_hdr_ver = response_header_version(api_key, api_version); + + let mut decoder = Decoder::new(frame); + let req = RequestHeader::decode_from(&mut decoder, req_hdr_ver)?; + info!( + %peer, + api_key = req.api_key, + api_version = req.api_version, + correlation_id = req.correlation_id, + client_id = req.client_id.as_deref().unwrap_or(""), + "received request" + ); + + let body = decoder.read_bytes(decoder.remaining())?; + let body_response = handle_request(req.api_key, req.api_version, body); + + let resp_header = ResponseHeader { + correlation_id: req.correlation_id, + }; + let encoded_header = resp_header.encode(resp_hdr_ver); + let mut payload = BytesMut::with_capacity(encoded_header.len() + body_response.len()); + payload.put_slice(&encoded_header); + payload.put_slice(&body_response); + + write_frame(&mut stream, &payload, config.write_timeout).await?; + } +} + +pub async fn read_frame( + stream: &mut TcpStream, + max_frame_size: usize, + read_timeout: Duration, +) -> Result { + let mut len_buf = [0u8; 4]; + timeout(read_timeout, stream.read_exact(&mut len_buf)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; + + let frame_len = i32::from_be_bytes(len_buf); + if frame_len <= 0 { + return Err(KafkaProtocolError::InvalidFrameLength(frame_len)); + } + + let frame_len = frame_len as usize; + if frame_len > max_frame_size { + return Err(KafkaProtocolError::FrameTooLarge { + max_bytes: max_frame_size, + actual_bytes: frame_len, + }); + } + + let mut data = vec![0u8; frame_len]; + timeout(read_timeout, stream.read_exact(&mut data)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; + Ok(bytes::Bytes::from(data)) +} + +pub async fn write_frame( + stream: &mut TcpStream, + payload: &[u8], + write_timeout: Duration, +) -> Result<()> { + let len = payload.len(); + if len > i32::MAX as usize { + return Err(KafkaProtocolError::FrameTooLarge { + max_bytes: i32::MAX as usize, + actual_bytes: len, + }); + } + let mut frame = BytesMut::with_capacity(4 + len); + frame.put_i32(len as i32); + frame.extend_from_slice(payload); + timeout(write_timeout, stream.write_all(&frame)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timeout"))??; + Ok(()) +} + +pub fn init_tracing() { + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + let _ = tracing_subscriber::fmt() + .with_env_filter(filter) + .try_init() + .map_err(|e| error!("failed to initialize tracing: {e}")); +} diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs new file mode 100644 index 0000000000..5fb8b0c6c2 --- /dev/null +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_METADATA, ERROR_UNSUPPORTED_VERSION, handle_request, + is_supported_version, split_metadata_request_topics, supported_api_ranges, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +// ── ApiVersions ───────────────────────────────────────────────────────────── + +#[test] +fn api_versions_v1_response_non_flexible_format() { + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new()); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i16().unwrap(), 0); // error_code + + // Non-flexible: i32 array count + let count = d.read_i32().unwrap(); + assert!(count >= 2); + let mut keys = Vec::new(); + for _ in 0..count { + keys.push(d.read_i16().unwrap()); + d.read_i16().unwrap(); // min + d.read_i16().unwrap(); // max + } + assert_eq!(d.read_i32().unwrap(), 0); // throttle_time_ms + + let expected_keys: Vec = supported_api_ranges().iter().map(|r| r.api_key).collect(); + for k in expected_keys { + assert!(keys.contains(&k)); + } +} + +#[test] +fn api_versions_v3_response_flexible_format() { + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new()); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i16().unwrap(), 0); // error_code + + // Flexible: varint(len+1) compact array + let count_plus_one = d.read_varint().unwrap(); + assert!(count_plus_one >= 3); // at least 2 entries → varint = 3+ + let count = (count_plus_one - 1) as i32; + + let mut keys = Vec::new(); + for _ in 0..count { + keys.push(d.read_i16().unwrap()); + d.read_i16().unwrap(); // min + d.read_i16().unwrap(); // max + d.read_tagged_fields().unwrap(); // per-entry tagged fields + } + assert_eq!(d.read_i32().unwrap(), 0); // throttle_time_ms + d.read_tagged_fields().unwrap(); // top-level tagged fields + + let expected_keys: Vec = supported_api_ranges().iter().map(|r| r.api_key).collect(); + for k in expected_keys { + assert!(keys.contains(&k)); + } +} + +// ── Metadata ───────────────────────────────────────────────────────────────── + +#[test] +fn metadata_response_has_broker_array_and_topic_array() { + let body = handle_request(API_KEY_METADATA, 0, Bytes::new()); + let mut d = Decoder::new(body); + + let broker_count = d.read_i32().unwrap(); + assert_eq!(broker_count, 1); + let node_id = d.read_i32().unwrap(); + assert_eq!(node_id, 1); + let host = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(host, "127.0.0.1"); + let port = d.read_i32().unwrap(); + assert_eq!(port, 9093); + + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 0); +} + +#[test] +fn unsupported_version_returns_protocol_error() { + let mut req = Vec::new(); + req.extend_from_slice(&1_i32.to_be_bytes()); + let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req)); + let mut d = Decoder::new(body); + let _broker_count = d.read_i32().unwrap(); + let _ = d.read_i32().unwrap(); + let _ = d.read_nullable_string().unwrap(); + let _ = d.read_i32().unwrap(); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let topic_error = d.read_i16().unwrap(); + assert_eq!(topic_error, ERROR_UNSUPPORTED_VERSION); + let topic_name = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(topic_name, "unknown-topic"); + let partitions_count = d.read_i32().unwrap(); + assert_eq!(partitions_count, 0); + let controller_id = d.read_i32().unwrap(); + assert_eq!(controller_id, 1); +} + +// ── Misc ──────────────────────────────────────────────────────────────────── + +#[test] +fn unknown_api_key_returns_error_only_payload() { + let body = handle_request(999, 0, Bytes::new()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn metadata_topic_split_reads_array_count() { + let mut raw = Vec::new(); + raw.extend_from_slice(&2_i32.to_be_bytes()); + assert_eq!(split_metadata_request_topics(Bytes::from(raw)), 2); +} + +#[test] +fn version_support_table_is_applied() { + assert!(is_supported_version(API_KEY_API_VERSIONS, 3)); + assert!(!is_supported_version(API_KEY_API_VERSIONS, 10)); + assert!(is_supported_version(API_KEY_METADATA, 1)); + assert!(!is_supported_version(API_KEY_METADATA, -1)); +} diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs new file mode 100644 index 0000000000..093d123451 --- /dev/null +++ b/gateways/kafka/tests/codec_tests.rs @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +#[test] +fn codec_round_trip_primitives_and_nullable_fields() { + let mut enc = Encoder::with_capacity(128); + enc.write_i8(-3); + enc.write_i16(42); + enc.write_i32(123_456); + enc.write_i64(9_999_999); + enc.write_nullable_string(Some("client-a")); + enc.write_nullable_string(None); + enc.write_nullable_bytes(Some(&[1, 2, 3])); + enc.write_nullable_bytes(None); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!(dec.read_i8().unwrap(), -3); + assert_eq!(dec.read_i16().unwrap(), 42); + assert_eq!(dec.read_i32().unwrap(), 123_456); + assert_eq!(dec.read_i64().unwrap(), 9_999_999); + assert_eq!( + dec.read_nullable_string().unwrap().as_deref(), + Some("client-a") + ); + assert_eq!(dec.read_nullable_string().unwrap(), None); + assert_eq!( + dec.read_nullable_bytes().unwrap().unwrap(), + Bytes::from_static(&[1, 2, 3]) + ); + assert_eq!(dec.read_nullable_bytes().unwrap(), None); +} + +#[test] +fn decoder_returns_underflow_error() { + let mut dec = Decoder::new(Bytes::from_static(&[0x00])); + let err = dec.read_i32().expect_err("must fail"); + assert!(err.to_string().contains("buffer underflow")); +} + +#[test] +fn codec_u8_and_bool() { + let mut enc = Encoder::with_capacity(8); + enc.write_u8(0xFF); + enc.write_bool(true); + enc.write_bool(false); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!(dec.read_u8().unwrap(), 0xFF); + assert!(dec.read_bool().unwrap()); + assert!(!dec.read_bool().unwrap()); +} + +#[test] +fn varint_round_trip_small_values() { + for v in [0u64, 1, 127, 128, 255, 300, 16383, 16384, u32::MAX as u64] { + let mut enc = Encoder::with_capacity(16); + enc.write_varint(v); + let mut dec = Decoder::new(enc.freeze()); + assert_eq!(dec.read_varint().unwrap(), v, "failed for v={v}"); + } +} + +#[test] +fn varint_single_byte_for_values_below_128() { + let mut enc = Encoder::with_capacity(1); + enc.write_varint(42); + let bytes = enc.freeze(); + assert_eq!(bytes.len(), 1); + assert_eq!(bytes[0], 42); +} + +#[test] +fn varint_two_bytes_for_128() { + let mut enc = Encoder::with_capacity(2); + enc.write_varint(128); + let bytes = enc.freeze(); + // 128 = 0x80 → first byte 0x80 | 0x80 = 0x80 (continue), second byte 0x01 + assert_eq!(bytes.as_ref(), &[0x80, 0x01]); +} + +#[test] +fn compact_nullable_string_round_trip() { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_string(Some("hello")); + enc.write_compact_nullable_string(None); + enc.write_compact_nullable_string(Some("")); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!( + dec.read_compact_nullable_string().unwrap().as_deref(), + Some("hello") + ); + assert_eq!(dec.read_compact_nullable_string().unwrap(), None); + assert_eq!( + dec.read_compact_nullable_string().unwrap().as_deref(), + Some("") + ); +} + +#[test] +fn compact_nullable_bytes_round_trip() { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_bytes(Some(&[10, 20, 30])); + enc.write_compact_nullable_bytes(None); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!( + dec.read_compact_nullable_bytes().unwrap().unwrap(), + Bytes::from_static(&[10, 20, 30]) + ); + assert_eq!(dec.read_compact_nullable_bytes().unwrap(), None); +} + +#[test] +fn tagged_fields_empty_section_round_trip() { + let mut enc = Encoder::with_capacity(8); + enc.write_i32(42); + enc.write_empty_tagged_fields(); + enc.write_i16(7); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!(dec.read_i32().unwrap(), 42); + dec.read_tagged_fields().unwrap(); // should consume the single 0x00 byte + assert_eq!(dec.read_i16().unwrap(), 7); + assert_eq!(dec.remaining(), 0); +} diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs new file mode 100644 index 0000000000..d9f9cab3f9 --- /dev/null +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -0,0 +1,449 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Validates request decoders and response encoders against the binary fixtures +//! produced by tools/kafka-tool. +//! +//! Frame layout written by kafka-tool (all versions): +//! [4-byte length prefix] +//! [api_key i16][api_version i16][correlation_id i32] +//! [client_id_len i16][client_id bytes] ← always legacy i16, even for flexible APIs +//! [0x00 tagged-fields byte] ← only for flexible API versions +//! [request body] ← properly encoded per spec (flexible or not) + +use std::path::PathBuf; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::header::request_header_version; +use iggy_gateway_kafka::protocol::requests::{ + decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, + decode_produce_request, +}; +use iggy_gateway_kafka::protocol::responses::{ + encode_create_topics_response, encode_fetch_response, encode_list_offsets_response, + encode_produce_response, +}; + +// ── helpers ─────────────────────────────────────────────────────────────────── + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") +} + +/// Load a kafka-tool .bin file and return just the request body bytes, correctly +/// skipping the outer Kafka frame header (api_key, api_version, correlation_id, +/// legacy-i16 client_id, and — for flexible versions — the 0x00 tagged-fields byte). +fn load_body(api_key: i16, api_name: &str, version: i16) -> Bytes { + let filename = format!("{:03}_{}_v{}.bin", api_key, api_name, version); + let path = fixtures_dir().join(&filename); + let data = std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {filename}: {e}")); + + // Skip 4-byte length prefix → frame starts here + let frame = &data[4..]; + + // Bytes 0-7: api_key(2) + api_version(2) + correlation_id(4) + // Bytes 8-9: client_id_len (legacy i16) + let client_id_len = i16::from_be_bytes([frame[8], frame[9]]) as usize; + let body_start_after_client_id = 10 + client_id_len; + + // kafka-tool appends a 0x00 tagged-fields byte for flexible-version APIs + let is_flexible = request_header_version(api_key, version) >= 2; + let body_start = if is_flexible { + body_start_after_client_id + 1 + } else { + body_start_after_client_id + }; + + Bytes::copy_from_slice(&frame[body_start..]) +} + +// ── Produce (API key 0) ─────────────────────────────────────────────────────── + +#[test] +fn produce_all_supported_versions_decode() { + for version in 3i16..=9 { + let body = load_body(0, "Produce", version); + let req = decode_produce_request(version, body) + .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); + + assert_eq!(req.acks, -1, "Produce v{version}: unexpected acks"); + assert_eq!( + req.timeout_ms, 5000, + "Produce v{version}: unexpected timeout_ms" + ); + assert_eq!(req.topics.len(), 1, "Produce v{version}: expected 1 topic"); + assert_eq!( + req.topics[0].topic, "test-topic", + "Produce v{version}: wrong topic name" + ); + assert_eq!( + req.topics[0].partitions.len(), + 1, + "Produce v{version}: expected 1 partition" + ); + assert_eq!( + req.topics[0].partitions[0].partition, 0, + "Produce v{version}: wrong partition index" + ); + assert!( + req.topics[0].partitions[0].records.is_some(), + "Produce v{version}: records should be present" + ); + } +} + +#[test] +fn produce_response_encodes_for_all_supported_versions() { + for version in 3i16..=9 { + let body = load_body(0, "Produce", version); + let req = decode_produce_request(version, body) + .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); + let resp = encode_produce_response(version, req); + assert!( + !resp.is_empty(), + "Produce v{version}: response must not be empty" + ); + } +} + +#[test] +fn produce_response_v3_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(0, "Produce", 3); + let req = decode_produce_request(3, body).unwrap(); + let resp = encode_produce_response(3, req); + + let mut d = Decoder::new(resp); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let topic_name = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(topic_name, "test-topic"); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let partition = d.read_i32().unwrap(); + assert_eq!(partition, 0); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let base_offset = d.read_i64().unwrap(); + assert_eq!(base_offset, 0); + // log_append_time_ms (v2+) + let _log_append = d.read_i64().unwrap(); + // log_start_offset (v5+) — not present for v3 + let throttle = d.read_i32().unwrap(); + assert_eq!(throttle, 0); +} + +#[test] +fn produce_response_v8_includes_record_errors() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(0, "Produce", 8); + let req = decode_produce_request(8, body).unwrap(); + let resp = encode_produce_response(8, req); + + let mut d = Decoder::new(resp); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let _topic_name = d.read_nullable_string().unwrap(); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let _partition = d.read_i32().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _base_offset = d.read_i64().unwrap(); + let _log_append_time = d.read_i64().unwrap(); // v2+ + let _log_start_offset = d.read_i64().unwrap(); // v5+ + let record_errors_count = d.read_i32().unwrap(); // v8+: should be 0 + assert_eq!( + record_errors_count, 0, + "v8 must emit empty record_errors array" + ); + let error_message = d.read_nullable_string().unwrap(); // v8+: should be null + assert!(error_message.is_none(), "v8 error_message must be null"); +} + +// ── Fetch (API key 1) ───────────────────────────────────────────────────────── + +#[test] +fn fetch_all_supported_versions_decode() { + for version in 4i16..=12 { + let body = load_body(1, "Fetch", version); + let req = decode_fetch_request(version, body) + .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); + + assert_eq!( + req.max_wait_ms, 500, + "Fetch v{version}: unexpected max_wait_ms" + ); + assert_eq!(req.min_bytes, 1, "Fetch v{version}: unexpected min_bytes"); + assert_eq!(req.topics.len(), 1, "Fetch v{version}: expected 1 topic"); + assert_eq!( + req.topics[0].topic, "test-topic", + "Fetch v{version}: wrong topic name" + ); + assert_eq!( + req.topics[0].partitions.len(), + 1, + "Fetch v{version}: expected 1 partition" + ); + assert_eq!( + req.topics[0].partitions[0].partition, 0, + "Fetch v{version}: wrong partition index" + ); + assert_eq!( + req.topics[0].partitions[0].fetch_offset, 0, + "Fetch v{version}: wrong fetch_offset" + ); + } +} + +#[test] +fn fetch_response_encodes_for_all_supported_versions() { + for version in 4i16..=12 { + let body = load_body(1, "Fetch", version); + let req = decode_fetch_request(version, body) + .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); + let resp = encode_fetch_response(version, req); + assert!( + !resp.is_empty(), + "Fetch v{version}: response must not be empty" + ); + } +} + +#[test] +fn fetch_response_v7_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(1, "Fetch", 7); + let req = decode_fetch_request(7, body).unwrap(); + let resp = encode_fetch_response(7, req); + + let mut d = Decoder::new(resp); + let throttle_ms = d.read_i32().unwrap(); // v1+ + assert_eq!(throttle_ms, 0); + let error_code = d.read_i16().unwrap(); // v7+ + assert_eq!(error_code, 0); + let session_id = d.read_i32().unwrap(); // v7+ + assert_eq!(session_id, 0); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let topic_name = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(topic_name, "test-topic"); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let partition = d.read_i32().unwrap(); + assert_eq!(partition, 0); + let partition_error = d.read_i16().unwrap(); + assert_eq!(partition_error, 0); + let high_watermark = d.read_i64().unwrap(); + assert_eq!(high_watermark, 0); +} + +// ── ListOffsets (API key 2) ─────────────────────────────────────────────────── + +#[test] +fn list_offsets_all_supported_versions_decode() { + for version in 1i16..=6 { + let body = load_body(2, "ListOffsets", version); + let req = decode_list_offsets_request(version, body) + .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); + + assert_eq!( + req.topics.len(), + 1, + "ListOffsets v{version}: expected 1 topic" + ); + assert_eq!( + req.topics[0].topic, "test-topic", + "ListOffsets v{version}: wrong topic name" + ); + assert_eq!( + req.topics[0].partitions.len(), + 1, + "ListOffsets v{version}: expected 1 partition" + ); + assert_eq!( + req.topics[0].partitions[0].partition, 0, + "ListOffsets v{version}: wrong partition index" + ); + } +} + +#[test] +fn list_offsets_response_encodes_for_all_supported_versions() { + for version in 1i16..=6 { + let body = load_body(2, "ListOffsets", version); + let req = decode_list_offsets_request(version, body) + .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); + let resp = encode_list_offsets_response(version, req); + assert!( + !resp.is_empty(), + "ListOffsets v{version}: response must not be empty" + ); + } +} + +#[test] +fn list_offsets_response_v1_no_leader_epoch() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(2, "ListOffsets", 1); + let req = decode_list_offsets_request(1, body).unwrap(); + let resp = encode_list_offsets_response(1, req); + + let mut d = Decoder::new(resp); + // v1: no throttle_time_ms + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let _topic_name = d.read_nullable_string().unwrap(); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let _partition = d.read_i32().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _timestamp = d.read_i64().unwrap(); // v1+ + let _offset = d.read_i64().unwrap(); + // v1 must NOT have a leader_epoch field — assert all bytes consumed + assert_eq!( + d.remaining(), + 0, + "v1 response must have no trailing bytes (leader_epoch must NOT be written)" + ); +} + +#[test] +fn list_offsets_response_v4_has_leader_epoch() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(2, "ListOffsets", 4); + let req = decode_list_offsets_request(4, body).unwrap(); + let resp = encode_list_offsets_response(4, req); + + let mut d = Decoder::new(resp); + let _throttle = d.read_i32().unwrap(); // v2+ + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let _topic_name = d.read_nullable_string().unwrap(); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let _partition = d.read_i32().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _timestamp = d.read_i64().unwrap(); + let _offset = d.read_i64().unwrap(); + let leader_epoch = d.read_i32().unwrap(); // v4+ + assert_eq!(leader_epoch, -1, "v4 must have leader_epoch = -1"); + assert_eq!(d.remaining(), 0); +} + +// ── CreateTopics (API key 19) ───────────────────────────────────────────────── + +#[test] +fn create_topics_all_supported_versions_decode() { + for version in 2i16..=5 { + let body = load_body(19, "CreateTopics", version); + let req = decode_create_topics_request(version, body) + .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); + + assert_eq!( + req.topics.len(), + 1, + "CreateTopics v{version}: expected 1 topic" + ); + assert_eq!( + req.topics[0].num_partitions, 1, + "CreateTopics v{version}: wrong num_partitions" + ); + assert_eq!( + req.topics[0].replication_factor, 1, + "CreateTopics v{version}: wrong replication_factor" + ); + assert!( + !req.topics[0].name.is_empty(), + "CreateTopics v{version}: topic name must not be empty" + ); + assert_eq!( + req.timeout_ms, 30000, + "CreateTopics v{version}: unexpected timeout_ms" + ); + } +} + +#[test] +fn create_topics_response_encodes_for_all_supported_versions() { + for version in 2i16..=5 { + let body = load_body(19, "CreateTopics", version); + let req = decode_create_topics_request(version, body) + .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); + let resp = encode_create_topics_response(version, req); + assert!( + !resp.is_empty(), + "CreateTopics v{version}: response must not be empty" + ); + } +} + +#[test] +fn create_topics_response_v2_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(19, "CreateTopics", 2); + let req = decode_create_topics_request(2, body).unwrap(); + let topic_name = req.topics[0].name.clone(); + let resp = encode_create_topics_response(2, req); + + let mut d = Decoder::new(resp); + let _throttle = d.read_i32().unwrap(); // v2+ + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let resp_topic = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(resp_topic, topic_name); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let error_msg = d.read_nullable_string().unwrap(); // v1+ + assert!(error_msg.is_none()); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn create_topics_response_v5_has_topic_config_error_code() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(19, "CreateTopics", 5); + let req = decode_create_topics_request(5, body).unwrap(); + let resp = encode_create_topics_response(5, req); + + let mut d = Decoder::new(resp); + let _throttle = d.read_i32().unwrap(); // v2+ + let topic_count_plus_one = d.read_varint().unwrap(); // flexible compact array + assert_eq!(topic_count_plus_one, 2); // 1 topic → varint = 2 + + let _topic_name = d.read_compact_nullable_string().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _error_msg = d.read_compact_nullable_string().unwrap(); // v1+ + let topic_config_err = d.read_i16().unwrap(); // v5+: MUST be present + assert_eq!( + topic_config_err, 0, + "v5 must include topic_config_error_code" + ); + let num_partitions = d.read_i32().unwrap(); + assert_eq!(num_partitions, 1); + let replication_factor = d.read_i16().unwrap(); + assert_eq!(replication_factor, 1); + let configs_count_plus_one = d.read_varint().unwrap(); // empty compact array + assert_eq!(configs_count_plus_one, 1); // empty = varint(1) + d.read_tagged_fields().unwrap(); // per-entry tagged_fields + d.read_tagged_fields().unwrap(); // top-level tagged_fields + assert_eq!(d.remaining(), 0); +} diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs new file mode 100644 index 0000000000..121074a191 --- /dev/null +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{API_KEY_API_VERSIONS, API_KEY_METADATA, handle_request}; +use iggy_gateway_kafka::protocol::codec::Encoder; + +#[test] +fn golden_apiversions_v1_response_fixture() { + let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new()); + + // error_code=0, api_count=6 + // key 0 (Produce) min=3 max=9 + // key 1 (Fetch) min=4 max=12 + // key 2 (ListOffsets) min=1 max=6 + // key 3 (Metadata) min=0 max=9 + // key 18 (ApiVersions) min=0 max=3 + // key 19 (CreateTopics) min=2 max=5 + // throttle_ms=0 + let expected: [u8; 46] = [ + 0x00, 0x00, // error_code + 0x00, 0x00, 0x00, 0x06, // api count = 6 + 0x00, 0x00, 0x00, 0x03, 0x00, 0x09, // key 0: Produce 3–9 + 0x00, 0x01, 0x00, 0x04, 0x00, 0x0C, // key 1: Fetch 4–12 + 0x00, 0x02, 0x00, 0x01, 0x00, 0x06, // key 2: ListOffsets 1–6 + 0x00, 0x03, 0x00, 0x00, 0x00, 0x09, // key 3: Metadata 0–9 + 0x00, 0x12, 0x00, 0x00, 0x00, 0x03, // key 18: ApiVersions 0–3 + 0x00, 0x13, 0x00, 0x02, 0x00, 0x05, // key 19: CreateTopics 2–5 + 0x00, 0x00, 0x00, 0x00, // throttle_ms + ]; + assert_eq!(actual.as_ref(), &expected); +} + +#[test] +fn golden_metadata_v0_single_topic_response_fixture() { + let mut request = Encoder::with_capacity(32); + request.write_i32(1); // one topic + let req_bytes = request.freeze(); + + let actual = handle_request(API_KEY_METADATA, 0, req_bytes); + + // brokers[1]: node_id=1, host=127.0.0.1, port=9093 + // topics[1]: topic_error=3, topic_name=unknown-topic, partitions[0] + // controller_id=1 (included by this implementation baseline) + let expected: [u8; 52] = [ + 0x00, 0x00, 0x00, 0x01, // broker count + 0x00, 0x00, 0x00, 0x01, // node id + 0x00, 0x09, // host len + 0x31, 0x32, 0x37, 0x2e, 0x30, 0x2e, 0x30, 0x2e, 0x31, // "127.0.0.1" + 0x00, 0x00, 0x23, 0x85, // port 9093 + 0x00, 0x00, 0x00, 0x01, // topic count + 0x00, 0x03, // topic error code + 0x00, 0x0d, // topic name len + 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x2d, 0x74, 0x6f, 0x70, 0x69, + 0x63, // unknown-topic + 0x00, 0x00, 0x00, 0x00, // partition count + 0x00, 0x00, 0x00, 0x01, // controller id + ]; + assert_eq!(actual.as_ref(), &expected); +} diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs new file mode 100644 index 0000000000..d4eef81db6 --- /dev/null +++ b/gateways/kafka/tests/header_tests.rs @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_gateway_kafka::protocol::codec::Encoder; +use iggy_gateway_kafka::protocol::header::{ + RequestHeader, ResponseHeader, request_header_version, response_header_version, +}; + +// ── Request header v1 (non-flexible) ─────────────────────────────────────── + +#[test] +fn request_header_v1_decodes() { + let mut enc = Encoder::with_capacity(64); + enc.write_i16(18); // api_key: ApiVersions + enc.write_i16(2); // api_version + enc.write_i32(101); + enc.write_nullable_string(Some("kafka-cli")); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 1).expect("decode should succeed"); + assert_eq!(header.api_key, 18); + assert_eq!(header.api_version, 2); + assert_eq!(header.correlation_id, 101); + assert_eq!(header.client_id.as_deref(), Some("kafka-cli")); +} + +#[test] +fn request_header_v1_null_client_id() { + let mut enc = Encoder::with_capacity(32); + enc.write_i16(18); + enc.write_i16(1); + enc.write_i32(5); + enc.write_nullable_string(None); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 1).unwrap(); + assert_eq!(header.client_id, None); +} + +// ── Request header v2 (flexible — compact client_id + tagged fields) ─────── + +#[test] +fn request_header_v2_decodes() { + let mut enc = Encoder::with_capacity(64); + enc.write_i16(18); // api_key: ApiVersions + enc.write_i16(3); // api_version (flexible threshold for ApiVersions is 3) + enc.write_i32(202); + enc.write_compact_nullable_string(Some("my-client")); + enc.write_empty_tagged_fields(); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 2).expect("flexible decode should succeed"); + assert_eq!(header.api_key, 18); + assert_eq!(header.api_version, 3); + assert_eq!(header.correlation_id, 202); + assert_eq!(header.client_id.as_deref(), Some("my-client")); +} + +#[test] +fn request_header_v2_null_client_id() { + let mut enc = Encoder::with_capacity(32); + enc.write_i16(18); + enc.write_i16(3); + enc.write_i32(303); + enc.write_compact_nullable_string(None); + enc.write_empty_tagged_fields(); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 2).unwrap(); + assert_eq!(header.client_id, None); +} + +// ── Response header encode ────────────────────────────────────────────────── + +#[test] +fn response_header_v0_encodes_correlation_id_only() { + let header = ResponseHeader { correlation_id: 77 }; + let bytes = header.encode(0); + assert_eq!(bytes.as_ref(), &[0, 0, 0, 77]); +} + +#[test] +fn response_header_v1_encodes_correlation_id_plus_tagged_fields() { + let header = ResponseHeader { correlation_id: 1 }; + let bytes = header.encode(1); + // [0,0,0,1] correlation_id + [0x00] empty tagged fields + assert_eq!(bytes.as_ref(), &[0, 0, 0, 1, 0x00]); +} + +// ── Header version lookup ─────────────────────────────────────────────────── + +#[test] +fn request_header_version_non_flexible_below_threshold() { + // ApiVersions v0-2 → header v1 + assert_eq!(request_header_version(18, 0), 1); + assert_eq!(request_header_version(18, 2), 1); + // Metadata v0-8 → header v1 + assert_eq!(request_header_version(3, 0), 1); + assert_eq!(request_header_version(3, 8), 1); +} + +#[test] +fn request_header_version_flexible_at_threshold() { + // ApiVersions v3 → header v2 + assert_eq!(request_header_version(18, 3), 2); + // Metadata v9 → header v2 + assert_eq!(request_header_version(3, 9), 2); + // ConsumerGroupHeartbeat (68) always flexible + assert_eq!(request_header_version(68, 0), 2); +} + +#[test] +fn response_header_version_apiversions_always_zero() { + // ApiVersions is a special case: response header is always v0 + assert_eq!(response_header_version(18, 0), 0); + assert_eq!(response_header_version(18, 3), 0); // even flexible request → v0 response +} + +#[test] +fn response_header_version_flexible_non_apiversions() { + // Metadata v9+ is flexible → response header v1 + assert_eq!(response_header_version(3, 9), 1); + // Metadata v0 is non-flexible → response header v0 + assert_eq!(response_header_version(3, 0), 0); +} diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs new file mode 100644 index 0000000000..7a672348ac --- /dev/null +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::time::Duration; + +use bytes::{Buf, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +use iggy_gateway_kafka::protocol::codec::Encoder; +use iggy_gateway_kafka::server::{read_frame, write_frame}; + +async fn tcp_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let client = tokio::spawn(async move { TcpStream::connect(addr).await.unwrap() }); + let (server, _) = listener.accept().await.unwrap(); + let client = client.await.unwrap(); + (client, server) +} + +#[tokio::test] +async fn read_frame_reads_valid_payload() { + let (mut client, mut server) = tcp_pair().await; + + let mut enc = Encoder::with_capacity(64); + enc.write_i16(18); + enc.write_i16(3); + enc.write_i32(123); + enc.write_nullable_string(Some("test-client")); + let payload = enc.freeze(); + + let mut frame = BytesMut::with_capacity(4 + payload.len()); + frame.extend_from_slice(&(payload.len() as i32).to_be_bytes()); + frame.extend_from_slice(&payload); + client.write_all(&frame).await.unwrap(); + + let parsed = read_frame(&mut server, 4096, Duration::from_secs(1)) + .await + .unwrap(); + assert_eq!(parsed, payload); +} + +#[tokio::test] +async fn write_frame_writes_length_prefixed_payload() { + let (mut client, mut server) = tcp_pair().await; + let payload = b"abc123"; + write_frame(&mut server, payload, Duration::from_secs(1)) + .await + .unwrap(); + + let mut len = [0u8; 4]; + client.read_exact(&mut len).await.unwrap(); + let len = i32::from_be_bytes(len) as usize; + assert_eq!(len, payload.len()); + + let mut body = vec![0u8; len]; + client.read_exact(&mut body).await.unwrap(); + assert_eq!(body, payload); +} + +#[tokio::test] +async fn read_frame_rejects_invalid_lengths() { + let (mut client, mut server) = tcp_pair().await; + + client.write_all(&0i32.to_be_bytes()).await.unwrap(); + let err = read_frame(&mut server, 128, Duration::from_secs(1)) + .await + .expect_err("zero frame must fail"); + assert!(err.to_string().contains("invalid frame length")); + + // Ensure connection can still be reused for a second scenario by writing a valid new prefix+payload. + let mut frame = BytesMut::new(); + frame.extend_from_slice(&(200i32).to_be_bytes()); + frame.resize(4 + 200, 0); + client.write_all(&frame).await.unwrap(); + let err = read_frame(&mut server, 64, Duration::from_secs(1)) + .await + .expect_err("large frame must fail"); + assert!(err.to_string().contains("exceeds max frame size")); +} + +#[tokio::test] +async fn write_frame_length_prefix_is_big_endian() { + let (mut client, mut server) = tcp_pair().await; + write_frame(&mut server, &[1, 2, 3, 4], Duration::from_secs(1)) + .await + .unwrap(); + + let mut len_and_data = [0u8; 8]; + client.read_exact(&mut len_and_data).await.unwrap(); + let mut buf = &len_and_data[..]; + let len = buf.get_i32(); + assert_eq!(len, 4); + assert_eq!(&len_and_data[4..], &[1, 2, 3, 4]); +} diff --git a/gateways/kafka/tools/kafka-tool/Cargo.toml b/gateways/kafka/tools/kafka-tool/Cargo.toml new file mode 100644 index 0000000000..a73d8addd9 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/Cargo.toml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "kafka-message-gen" +version = "0.1.0" +edition = "2024" +description = "Generates binary Kafka protocol messages for testing the Iggy Kafka gateway" +license = "Apache-2.0" +repository = "https://github.com/apache/iggy" +keywords = ["kafka", "protocol", "testing", "iggy", "wire-format"] +publish = false + +[[bin]] +name = "kafka-message-gen" +path = "src/main.rs" + +[dependencies] +anyhow = { workspace = true } +bytes = { workspace = true } +clap = { workspace = true } +hex = "0.4" +indexmap = "2" +kafka-protocol = "0.17" +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/gateways/kafka/tools/kafka-tool/README.md b/gateways/kafka/tools/kafka-tool/README.md new file mode 100644 index 0000000000..a65eb2a005 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/README.md @@ -0,0 +1,249 @@ +# kafka-message-gen + +A Rust CLI tool that generates correct, fully-framed Kafka binary wire protocol messages for **every API key** and **every supported version** — built for testing Kafka-compatible server implementations such as [Apache Iggy](https://iggy.apache.org)'s Kafka compatibility listener. + +## What It Does + +Each output `.bin` file is a complete, TCP-ready Kafka request: + +``` +[total_length: i32][api_key: i16][api_version: i16] +[correlation_id: i32][client_id: NULLABLE_STRING] +[tagged_fields: 0x00] ← only for flexible versions +[payload: bytes] ← API-specific encoded body +``` + +The tool covers **65 API keys** and **~280 versioned messages**, sourced directly from the official [Apache Kafka JSON schema files](https://github.com/apache/kafka/tree/trunk/clients/src/main/resources/common/message) (Kafka 4.1.0). + +--- + +## Why This Tool Exists + +When implementing a Kafka protocol compatibility layer (e.g. inside Apache Iggy), you need to: + +1. Verify your server correctly **parses** every API key at every version +2. Verify your server sends **valid responses** back (correct correlation ID, error codes) +3. Do this without spinning up a full Kafka client or running JVM-based tests + +This tool solves all three: generate the binary messages once, then `cat` them directly to your server's port 9092 and inspect the response. + +--- + +## Dependency on `kafka-protocol` + +This tool is built on the [`kafka-protocol`](https://crates.io/crates/kafka-protocol) Rust crate, which is itself **code-generated from Kafka's official JSON schema files**. This ensures byte-perfect correctness — the same schemas that generate Kafka's own Java serialization code generate the Rust structs used here. + +--- + +## Installation + +```bash +cd gateways/kafka/tools/kafka-tool +cargo build --release +# Binary is at: ./target/release/kafka-message-gen +``` + +**Requirements:** Rust 1.75+ (MSRV follows `kafka-protocol` crate) + +--- + +## Usage + +### List all API keys and version ranges + +```bash +cargo run -- list +``` + +Output: +``` +Key Name MinVer MaxVer Count +────────────────────────────────────────────────────────────────────────────── +0 Produce 3 13 11 +1 Fetch 4 18 15 +2 ListOffsets 1 11 11 +3 Metadata 0 13 14 +8 OffsetCommit 2 10 9 +... +────────────────────────────────────────────────────────────────────────────── +Total: 65 API keys | ~280 versioned messages +``` + +--- + +### Generate all binary messages + +```bash +cargo run -- generate --output ./kafka_messages/ +``` + +Creates one `.bin` file per API key × version: + +``` +kafka_messages/ + 000_Produce_v3.bin + 000_Produce_v4.bin + ... + 000_Produce_v13.bin + 001_Fetch_v4.bin + ... + 003_Metadata_v0.bin + 003_Metadata_v13.bin + 018_ApiVersions_v0.bin + 018_ApiVersions_v3.bin + ... +``` + +#### Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--output` | Output directory | `kafka_messages/` | +| `--api-key N` | Generate only for API key N | all | +| `--version N` | Generate only for version N | all | +| `--hex` | Print hex dump to stdout | off | + +```bash +# Generate only Metadata messages +cargo run -- generate --api-key 3 + +# Generate only ApiVersions v3 with hex dump +cargo run -- generate --api-key 18 --version 3 --hex +``` + +--- + +### Send messages to a live server + +```bash +# Start Iggy with Kafka compat listener on port 9092, then: +cargo run -- send --host 127.0.0.1:9092 +``` + +Output (one line per API key × version): +``` +✓ ApiVersions v3 → 32 bytes ec=0 +✓ Metadata v12 → 148 bytes ec=0 +⚠ Produce v9 → 24 bytes ec=3 ← ec=3 = UnknownTopicOrPartition (expected) +✓ Fetch v12 → 36 bytes ec=0 +... +Result: 243 OK 37 failed +``` + +#### Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--host` | Server address | `127.0.0.1:9092` | +| `--api-key N` | Test only API key N | all | +| `--version N` | Test only version N | all | +| `--timeout-ms N` | Per-request timeout | `5000` | + +--- + +### Verify compatibility (CI-friendly) + +```bash +cargo run -- verify --host 127.0.0.1:9092 +``` + +Exits with code **0** if all messages get a response, **1** if any fail (timeout or IO error). Useful in CI pipelines testing a Kafka-compatible server implementation. + +--- + +### Quick raw test with netcat + +No Rust needed for a quick smoke test: + +```bash +# Generate first +cargo run -- generate + +# Send ApiVersions v3 directly via netcat and inspect response +cat kafka_messages/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd | head + +# Send and decode with Wireshark (capture on loopback, filter: kafka) +``` + +--- + +## Supported API Keys (Kafka 4.1.0) + +| Key | Name | Versions | Phase 1 Priority | +|-----|------|----------|-----------------| +| 0 | Produce | v3–v13 | ✅ Critical | +| 1 | Fetch | v4–v18 | ✅ Critical | +| 2 | ListOffsets | v1–v11 | ✅ Critical | +| 3 | Metadata | v0–v13 | ✅ Critical | +| 8 | OffsetCommit | v2–v10 | ✅ Critical | +| 9 | OffsetFetch | v1–v10 | ✅ Critical | +| 10 | FindCoordinator | v0–v6 | ✅ Critical | +| 11 | JoinGroup | v0–v9 | ✅ Critical | +| 12 | Heartbeat | v0–v4 | ✅ Critical | +| 13 | LeaveGroup | v0–v5 | ✅ Critical | +| 14 | SyncGroup | v0–v5 | ✅ Critical | +| 15 | DescribeGroups | v0–v6 | 🟡 Important | +| 16 | ListGroups | v0–v5 | 🟡 Important | +| 17 | SaslHandshake | v0–v1 | 🟡 Important | +| 18 | ApiVersions | v0–v5 | ✅ Critical | +| 19 | CreateTopics | v2–v7 | ✅ Critical | +| 20 | DeleteTopics | v1–v6 | 🟡 Important | +| 21 | DeleteRecords | v0–v2 | 🔵 Phase 2 | +| 22 | InitProducerId | v0–v6 | 🔵 Phase 2 | +| 24 | AddPartitionsToTxn | v0–v5 | 🔵 Phase 2 | +| 25 | AddOffsetsToTxn | v0–v4 | 🔵 Phase 2 | +| 26 | EndTxn | v0–v5 | 🔵 Phase 2 | +| 28 | TxnOffsetCommit | v0–v5 | 🔵 Phase 2 | +| 29–31 | ACL APIs | v1–v3 | 🔵 Phase 2 | +| 32 | DescribeConfigs | v1–v4 | 🟡 Important | +| 36 | SaslAuthenticate | v0–v2 | 🟡 Important | +| ... | 40+ more | various | 🔵 Phase 3 | + +--- + +## Project Structure + +``` +tools/kafka-tool/ +├── Cargo.toml ← package manifest and dependencies +├── src/ +│ └── main.rs ← complete CLI implementation +└── README.md ← this file +``` + +--- + +## How It Works + +### Protocol Source + +All API schemas come from the official Kafka repository: +`apache/kafka/trunk/clients/src/main/resources/common/message/*.json` + +The `kafka-protocol` crate processes these JSON files and generates Rust structs with `encode()` and `decode()` methods. This guarantees byte-level compatibility with what official Kafka clients send. + +### Flexible vs Legacy Encoding + +Kafka introduced "flexible" encoding (compact ULEB128 strings/arrays) starting at different versions per API. The tool automatically detects whether a version uses flexible or legacy encoding and sets the request header format accordingly (header v1 for legacy, header v2 for flexible with tagged fields section). + +### API Key Coverage + +- **Explicit builders (23 API keys):** Produce, Fetch, ListOffsets, Metadata, OffsetCommit, OffsetFetch, FindCoordinator, JoinGroup, Heartbeat, LeaveGroup, SyncGroup, DescribeGroups, ListGroups, SaslHandshake, ApiVersions, CreateTopics, DeleteTopics, DeleteRecords, InitProducerId, AddPartitionsToTxn, AddOffsetsToTxn, EndTxn, TxnOffsetCommit, DescribeConfigs, SaslAuthenticate +- **Header-framing test (42 API keys):** All remaining API keys are framed correctly with an empty payload — useful for testing that your server returns a proper error response rather than crashing + +--- + +## Contributing + +The tool is intentionally simple. To add an explicit builder for a new API key: + +1. Find the JSON schema in `apache/kafka/.../message/YourRequest.json` +2. Add a new match arm in `build_payload()` in `src/main.rs` +3. Use the kafka-protocol crate's generated struct (e.g. `YourRequest::default().with_field(value)`) +4. Open a PR + +--- + +## License + +Apache License 2.0 — same as Apache Kafka and Apache Iggy. diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs new file mode 100644 index 0000000000..e1b29f08b7 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -0,0 +1,770 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::{Context, Result}; +use bytes::{BufMut, Bytes, BytesMut}; +use clap::{Parser, Subcommand}; +use kafka_protocol::messages::*; +use kafka_protocol::protocol::{Encodable, StrBytes}; +use std::path::PathBuf; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; +use tracing::{info, warn}; + +#[derive(Parser)] +#[command( + name = "kafka-message-gen", + about = "Generate Kafka wire protocol binary messages for all API keys and versions", + long_about = "Generates correctly-framed Kafka protocol requests from Kafka 4.1.0 schemas.\n\ +Each output .bin file is TCP-ready: [len:i32][api_key:i16][api_version:i16][correlation_id:i32][client_id][payload]" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// List all supported API keys with name and version range + List, + /// Generate binary .bin files for all API keys and versions + Generate { + #[arg(short, long, default_value = "kafka_messages")] + output: PathBuf, + /// Filter to a single API key integer + #[arg(long)] + api_key: Option, + /// Filter to a single version + #[arg(long)] + version: Option, + /// Print hex dump to stdout + #[arg(long)] + hex: bool, + }, + /// Send messages to a live Kafka-compatible server and show responses + Send { + #[arg(long, default_value = "127.0.0.1:9092")] + host: String, + #[arg(long)] + api_key: Option, + #[arg(long)] + version: Option, + #[arg(long, default_value = "5000")] + timeout_ms: u64, + }, + /// Send all messages and report pass/fail — exit code 1 if any fail + Verify { + #[arg(long, default_value = "127.0.0.1:9092")] + host: String, + #[arg(long)] + fail_fast: bool, + }, +} + +// ── API Registry ───────────────────────────────────────────────────────────── +// Source: validVersions in apache/kafka trunk JSON schema files, Kafka 4.1.0 +// Format: (api_key, name, min_version, max_version) +const API_REGISTRY: &[(i16, &str, i16, i16)] = &[ + (0, "Produce", 3, 13), + (1, "Fetch", 4, 18), + (2, "ListOffsets", 1, 11), + (3, "Metadata", 0, 13), + (8, "OffsetCommit", 2, 10), + (9, "OffsetFetch", 1, 10), + (10, "FindCoordinator", 0, 6), + (11, "JoinGroup", 0, 9), + (12, "Heartbeat", 0, 4), + (13, "LeaveGroup", 0, 5), + (14, "SyncGroup", 0, 5), + (15, "DescribeGroups", 0, 6), + (16, "ListGroups", 0, 5), + (17, "SaslHandshake", 0, 1), + (18, "ApiVersions", 0, 5), + (19, "CreateTopics", 2, 7), + (20, "DeleteTopics", 1, 6), + (21, "DeleteRecords", 0, 2), + (22, "InitProducerId", 0, 6), + (23, "OffsetForLeaderEpoch", 2, 4), + (24, "AddPartitionsToTxn", 0, 5), + (25, "AddOffsetsToTxn", 0, 4), + (26, "EndTxn", 0, 5), + (27, "WriteTxnMarkers", 1, 2), + (28, "TxnOffsetCommit", 0, 5), + (29, "DescribeAcls", 1, 3), + (30, "CreateAcls", 1, 3), + (31, "DeleteAcls", 1, 3), + (32, "DescribeConfigs", 1, 4), + (33, "AlterConfigs", 0, 2), + (34, "AlterReplicaLogDirs", 1, 2), + (35, "DescribeLogDirs", 1, 5), + (36, "SaslAuthenticate", 0, 2), + (37, "CreatePartitions", 0, 3), + (38, "CreateDelegationToken", 1, 3), + (39, "RenewDelegationToken", 1, 2), + (40, "ExpireDelegationToken", 1, 2), + (41, "DescribeDelegationToken", 1, 3), + (42, "DeleteGroups", 0, 2), + (43, "ElectLeaders", 0, 2), + (44, "IncrementalAlterConfigs", 0, 1), + (45, "AlterPartitionReassignments", 0, 1), + (46, "ListPartitionReassignments", 0, 1), + (47, "OffsetDelete", 0, 0), + (48, "DescribeClientQuotas", 0, 1), + (49, "AlterClientQuotas", 0, 1), + (50, "DescribeUserScramCredentials", 0, 0), + (51, "AlterUserScramCredentials", 0, 0), + (55, "DescribeQuorum", 2, 3), + (56, "AlterPartition", 2, 3), + (57, "UpdateFeatures", 0, 2), + (60, "DescribeCluster", 0, 2), + (61, "DescribeProducers", 0, 0), + (64, "UnregisterBroker", 0, 0), + (65, "DescribeTransactions", 0, 0), + (66, "ListTransactions", 0, 1), + (67, "AllocateProducerIds", 0, 0), + (68, "ConsumerGroupHeartbeat", 0, 1), + (69, "ConsumerGroupDescribe", 0, 1), + (71, "GetTelemetrySubscriptions", 0, 0), + (72, "PushTelemetry", 0, 0), + (74, "AssignReplicasToDirs", 0, 0), + (75, "DescribeTopicPartitions", 0, 0), + (76, "ListClientMetricsResources", 0, 0), +]; + +// ── Flexible version table ──────────────────────────────────────────────────── +// Source: flexibleVersions field in each Kafka JSON schema. +// Returns the first version using compact encoding, or None if never flexible. +fn first_flexible_version(api_key: i16) -> Option { + match api_key { + 0 => Some(9), + 1 => Some(12), + 2 => Some(6), + 3 => Some(9), + 8 => Some(8), + 9 => Some(6), + 10 => Some(3), + 11 => Some(6), + 12 => Some(4), + 13 => Some(4), + 14 => Some(4), + 15 => Some(5), + 16 => Some(3), + 17 => None, + 18 => Some(3), + 19 => Some(5), + 20 => Some(4), + 21 => Some(2), + 22 => Some(2), + 23 => Some(4), + 24 => Some(3), + 25 => Some(3), + 26 => Some(3), + 27 => Some(1), + 28 => Some(3), + 29 => Some(2), + 30 => Some(2), + 31 => Some(2), + 32 => Some(4), + 33 => Some(2), + 34 => Some(2), + 35 => Some(2), + 36 => Some(2), + 37 => Some(2), + 38 => Some(2), + 39 => Some(2), + 40 => Some(2), + 41 => Some(2), + 42 => Some(2), + 43 => Some(2), + 44 => Some(1), + 45 => Some(1), + 46 => Some(1), + 47 => Some(0), + 48 => Some(1), + 49 => Some(1), + 50 => Some(0), + 51 => Some(0), + 55 => Some(2), + 56 => Some(2), + 57 => Some(1), + 60 => Some(0), + 61 => Some(0), + 64 => Some(0), + 65 => Some(0), + 66 => Some(0), + 67 => Some(0), + 68 => Some(0), + 69 => Some(0), + 71 => Some(0), + 72 => Some(0), + 74 => Some(0), + 75 => Some(0), + 76 => Some(0), + _ => None, + } +} + +// ── Request framing ─────────────────────────────────────────────────────────── +// Wire format (Kafka protocol spec): +// [total_length: i32] big-endian, excludes self +// [api_key: i16] +// [api_version: i16] +// [correlation_id: i32] +// [client_id_len: i16] -1 = null +// [client_id: bytes] +// [tagged_fields: u8(0)] only present for flexible versions +// [payload: bytes] +fn frame_request( + api_key: i16, + api_version: i16, + correlation_id: i32, + client_id: &str, + payload: &[u8], + flexible: bool, +) -> Bytes { + let cid = client_id.as_bytes(); + let hlen = 2 + 2 + 4 + 2 + cid.len() + if flexible { 1 } else { 0 }; + let blen = hlen + payload.len(); + let mut buf = BytesMut::with_capacity(4 + blen); + buf.put_i32(blen as i32); + buf.put_i16(api_key); + buf.put_i16(api_version); + buf.put_i32(correlation_id); + buf.put_i16(cid.len() as i16); + buf.put_slice(cid); + if flexible { + buf.put_u8(0x00); + } + buf.put_slice(payload); + buf.freeze() +} + +// ── Payload builders ────────────────────────────────────────────────────────── +// Build the API-specific encoded body for a given api_key and version. +// All required fields contain realistic non-zero values. +// Returns raw bytes WITHOUT the framing header. +fn build_payload(api_key: i16, version: i16) -> Result { + let mut buf = BytesMut::new(); + match api_key { + 18 => { + let mut r = ApiVersionsRequest::default(); + if version >= 3 { + r.client_software_name = StrBytes::from_static_str("kafka-message-gen"); + r.client_software_version = StrBytes::from_static_str("0.1.0"); + } + r.encode(&mut buf, version).context("ApiVersions")?; + } + 3 => { + let mut r = MetadataRequest::default(); + if version >= 1 { + r.topics = None; + } + if version >= 4 { + r.allow_auto_topic_creation = true; + } + if version >= 8 { + r.include_cluster_authorized_operations = false; + r.include_topic_authorized_operations = false; + } + r.encode(&mut buf, version).context("Metadata")?; + } + 0 => { + use kafka_protocol::messages::produce_request::*; + use kafka_protocol::records::{ + Compression, Record, RecordBatchEncoder, RecordEncodeOptions, TimestampType, + }; + let rec = Record { + transactional: false, + control: false, + partition_leader_epoch: 0, + producer_id: -1, + producer_epoch: -1, + timestamp_type: TimestampType::Creation, + offset: 0, + sequence: 0, + timestamp: 1_700_000_000_000, + key: Some(Bytes::from_static(b"test-key")), + value: Some(Bytes::from_static(b"test-value")), + headers: indexmap::IndexMap::new(), + }; + let mut rb = BytesMut::new(); + RecordBatchEncoder::encode( + &mut rb, + [rec].iter(), + &RecordEncodeOptions { + version: 2, + compression: Compression::None, + }, + ) + .context("RecordBatch encode")?; + let pd = TopicProduceData::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partition_data(vec![ + PartitionProduceData::default() + .with_index(0) + .with_records(Some(rb.freeze())), + ]); + let mut r = ProduceRequest::default() + .with_acks(-1) + .with_timeout_ms(5000) + .with_topic_data(vec![pd]); + if version >= 3 { + r.transactional_id = None; + } + r.encode(&mut buf, version).context("Produce")?; + } + 1 => { + use kafka_protocol::messages::fetch_request::*; + let fp = FetchPartition::default() + .with_partition(0) + .with_fetch_offset(0) + .with_partition_max_bytes(1_048_576); + let ft = FetchTopic::default() + .with_topic(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![fp]); + let mut r = FetchRequest::default() + .with_replica_id(BrokerId(-1)) + .with_max_wait_ms(500) + .with_min_bytes(1) + .with_topics(vec![ft]); + if version >= 3 { + r.max_bytes = 52_428_800; + } + if version >= 4 { + r.isolation_level = 0; + } + if version >= 7 { + r.session_id = 0; + r.session_epoch = -1; + } + r.encode(&mut buf, version).context("Fetch")?; + } + 2 => { + use kafka_protocol::messages::list_offsets_request::*; + let p = ListOffsetsPartition::default() + .with_partition_index(0) + .with_timestamp(-1); + let t = ListOffsetsTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + ListOffsetsRequest::default() + .with_replica_id(BrokerId(-1)) + .with_isolation_level(0) + .with_topics(vec![t]) + .encode(&mut buf, version) + .context("ListOffsets")?; + } + 8 => { + use kafka_protocol::messages::offset_commit_request::*; + let p = OffsetCommitRequestPartition::default() + .with_partition_index(0) + .with_committed_offset(42) + .with_committed_metadata(Some(StrBytes::from_static_str(""))); + let t = OffsetCommitRequestTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + OffsetCommitRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_topics(vec![t]) + .encode(&mut buf, version) + .context("OffsetCommit")?; + } + 9 => { + OffsetFetchRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .encode(&mut buf, version) + .context("OffsetFetch")?; + } + 10 => { + FindCoordinatorRequest::default() + .with_key(StrBytes::from_static_str("test-group")) + .with_key_type(0) + .encode(&mut buf, version) + .context("FindCoordinator")?; + } + 11 => { + use kafka_protocol::messages::join_group_request::*; + let p = JoinGroupRequestProtocol::default() + .with_name(StrBytes::from_static_str("range")) + .with_metadata(Bytes::from_static(b"\x00\x00\x00\x01\x00\x0atest-topic")); + JoinGroupRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_session_timeout_ms(30_000) + .with_rebalance_timeout_ms(300_000) + .with_member_id(StrBytes::from_static_str("")) + .with_protocol_type(StrBytes::from_static_str("consumer")) + .with_protocols(vec![p]) + .encode(&mut buf, version) + .context("JoinGroup")?; + } + 12 => { + HeartbeatRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_generation_id(1) + .with_member_id(StrBytes::from_static_str("test-member-1")) + .encode(&mut buf, version) + .context("Heartbeat")?; + } + 13 => { + LeaveGroupRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_member_id(StrBytes::from_static_str("test-member-1")) + .encode(&mut buf, version) + .context("LeaveGroup")?; + } + 14 => { + SyncGroupRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_generation_id(1) + .with_member_id(StrBytes::from_static_str("test-member-1")) + .with_protocol_type(Some(StrBytes::from_static_str("consumer"))) + .with_protocol_name(Some(StrBytes::from_static_str("range"))) + .encode(&mut buf, version) + .context("SyncGroup")?; + } + 15 => { + DescribeGroupsRequest::default() + .with_groups(vec![GroupId::from(StrBytes::from_static_str("test-group"))]) + .with_include_authorized_operations(false) + .encode(&mut buf, version) + .context("DescribeGroups")?; + } + 16 => { + ListGroupsRequest::default() + .encode(&mut buf, version) + .context("ListGroups")?; + } + 17 => { + SaslHandshakeRequest::default() + .with_mechanism(StrBytes::from_static_str("PLAIN")) + .encode(&mut buf, version) + .context("SaslHandshake")?; + } + 19 => { + use kafka_protocol::messages::create_topics_request::*; + let t = CreatableTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str( + "iggy-test-topic", + ))) + .with_num_partitions(1) + .with_replication_factor(1); + CreateTopicsRequest::default() + .with_topics(vec![t]) + .with_timeout_ms(30_000) + .with_validate_only(false) + .encode(&mut buf, version) + .context("CreateTopics")?; + } + 20 => { + use kafka_protocol::messages::delete_topics_request::*; + let r = if version >= 6 { + DeleteTopicsRequest::default() + .with_topics(vec![DeleteTopicState::default().with_name(Some( + TopicName::from(StrBytes::from_static_str("iggy-test-topic")), + ))]) + .with_timeout_ms(30_000) + } else { + DeleteTopicsRequest::default() + .with_topic_names(vec![TopicName::from(StrBytes::from_static_str( + "iggy-test-topic", + ))]) + .with_timeout_ms(30_000) + }; + r.encode(&mut buf, version).context("DeleteTopics")?; + } + 21 => { + use kafka_protocol::messages::delete_records_request::*; + let p = DeleteRecordsPartition::default() + .with_partition_index(0) + .with_offset(0); + let t = DeleteRecordsTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + DeleteRecordsRequest::default() + .with_topics(vec![t]) + .with_timeout_ms(30_000) + .encode(&mut buf, version) + .context("DeleteRecords")?; + } + 22 => { + InitProducerIdRequest::default() + .with_transactional_id(None) + .with_transaction_timeout_ms(60_000) + .encode(&mut buf, version) + .context("InitProducerId")?; + } + 24 => { + use kafka_protocol::messages::add_partitions_to_txn_request::*; + let t = AddPartitionsToTxnTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![0i32]); + AddPartitionsToTxnRequest::default() + .with_v3_and_below_transactional_id(TransactionalId(StrBytes::from_static_str( + "test-txn", + ))) + .with_v3_and_below_producer_id(ProducerId(100)) + .with_v3_and_below_producer_epoch(1) + .with_v3_and_below_topics(vec![t]) + .encode(&mut buf, version) + .context("AddPartitionsToTxn")?; + } + 25 => { + AddOffsetsToTxnRequest::default() + .with_transactional_id(TransactionalId(StrBytes::from_static_str("test-txn"))) + .with_producer_id(ProducerId(100)) + .with_producer_epoch(1) + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .encode(&mut buf, version) + .context("AddOffsetsToTxn")?; + } + 26 => { + EndTxnRequest::default() + .with_transactional_id(TransactionalId(StrBytes::from_static_str("test-txn"))) + .with_producer_id(ProducerId(100)) + .with_producer_epoch(1) + .with_committed(true) + .encode(&mut buf, version) + .context("EndTxn")?; + } + 28 => { + use kafka_protocol::messages::txn_offset_commit_request::*; + let p = TxnOffsetCommitRequestPartition::default() + .with_partition_index(0) + .with_committed_offset(42) + .with_committed_metadata(Some(StrBytes::from_static_str(""))); + let t = TxnOffsetCommitRequestTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + TxnOffsetCommitRequest::default() + .with_transactional_id(TransactionalId(StrBytes::from_static_str("test-txn"))) + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_producer_id(ProducerId(100)) + .with_producer_epoch(1) + .with_topics(vec![t]) + .encode(&mut buf, version) + .context("TxnOffsetCommit")?; + } + 32 => { + use kafka_protocol::messages::describe_configs_request::*; + let r = DescribeConfigsResource::default() + .with_resource_type(2) + .with_resource_name(StrBytes::from_static_str("test-topic")); + DescribeConfigsRequest::default() + .with_resources(vec![r]) + .encode(&mut buf, version) + .context("DescribeConfigs")?; + } + 36 => { + SaslAuthenticateRequest::default() + .with_auth_bytes(Bytes::from_static(b"\x00iggy\x00secret")) + .encode(&mut buf, version) + .context("SaslAuthenticate")?; + } + other => { + warn!("api_key={other}: no explicit builder — empty payload (framing test)"); + } + } + Ok(buf.freeze()) +} + +// Build a complete framed Kafka request message ready for TCP transmission. +fn build_framed(api_key: i16, version: i16, corr: i32) -> Result { + let payload = build_payload(api_key, version)?; + let flexible = first_flexible_version(api_key) + .map(|fv| version >= fv) + .unwrap_or(false); + Ok(frame_request( + api_key, + version, + corr, + "kafka-message-gen", + &payload, + flexible, + )) +} + +// ── Commands ────────────────────────────────────────────────────────────────── + +fn cmd_list() { + println!( + "{:<6} {:<42} {:<10} {:<10} {:<8}", + "Key", "Name", "MinVer", "MaxVer", "Count" + ); + println!("{}", "─".repeat(78)); + for &(k, n, min, max) in API_REGISTRY { + println!( + "{:<6} {:<42} {:<10} {:<10} {:<8}", + k, + n, + min, + max, + max - min + 1 + ); + } + let total: i16 = API_REGISTRY + .iter() + .map(|&(_, _, min, max)| max - min + 1) + .sum(); + println!("{}", "─".repeat(78)); + println!( + "Total: {} API keys | {} versioned messages", + API_REGISTRY.len(), + total + ); +} + +async fn cmd_generate( + out: PathBuf, + fk: Option, + fv: Option, + hex_dump: bool, +) -> Result<()> { + tokio::fs::create_dir_all(&out).await?; + let (mut n, mut corr) = (0usize, 1i32); + for &(ak, name, min, max) in API_REGISTRY { + if fk.is_some_and(|k| k != ak) { + continue; + } + for v in min..=max { + if fv.is_some_and(|fv| fv != v) { + continue; + } + match build_framed(ak, v, corr) { + Ok(msg) => { + let fname = format!("{:03}_{}_v{}.bin", ak, name, v); + tokio::fs::write(out.join(&fname), &msg).await?; + if hex_dump { + println!("── {} v{} ({} bytes) ──", name, v, msg.len()); + println!("{}", hex::encode(&msg)); + println!(); + } else { + info!(" {} ({} bytes)", fname, msg.len()); + } + n += 1; + corr += 1; + } + Err(e) => warn!("SKIP {} v{}: {e}", name, v), + } + } + } + println!("\n✓ Generated {n} messages → {}/", out.display()); + println!( + " Quick test: cat {}/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd", + out.display() + ); + Ok(()) +} + +async fn run_send( + host: &str, + fk: Option, + fv: Option, + toms: u64, +) -> Result<(usize, usize)> { + let mut stream = TcpStream::connect(host) + .await + .with_context(|| format!("Cannot connect to {host}"))?; + info!("Connected to {host}"); + let (mut ok, mut fail, mut corr) = (0usize, 0usize, 1i32); + for &(ak, name, min, max) in API_REGISTRY { + if fk.is_some_and(|k| k != ak) { + continue; + } + for v in min..=max { + if fv.is_some_and(|fv| fv != v) { + continue; + } + let msg = match build_framed(ak, v, corr) { + Ok(m) => m, + Err(e) => { + warn!("Build {} v{}: {e}", name, v); + fail += 1; + continue; + } + }; + stream + .write_all(&msg) + .await + .with_context(|| format!("Write {} v{}", name, v))?; + let res = tokio::time::timeout(std::time::Duration::from_millis(toms), async { + let mut lb = [0u8; 4]; + stream.read_exact(&mut lb).await?; + let mut body = vec![0u8; i32::from_be_bytes(lb) as usize]; + stream.read_exact(&mut body).await?; + Ok::, std::io::Error>(body) + }) + .await; + match res { + Ok(Ok(r)) => { + let ec = if r.len() >= 6 { + i16::from_be_bytes(r[4..6].try_into().unwrap()) + } else { + -1 + }; + let sym = if ec <= 0 { "✓" } else { "⚠" }; + println!("{sym} {} v{} → {} bytes ec={ec}", name, v, r.len()); + ok += 1; + } + Ok(Err(e)) => { + println!("✗ {} v{} → IO error: {e}", name, v); + fail += 1; + } + Err(_) => { + println!("✗ {} v{} → timeout ({}ms)", name, v, toms); + fail += 1; + } + } + corr += 1; + } + } + Ok((ok, fail)) +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_target(false) + .with_level(true) + .init(); + let cli = Cli::parse(); + match cli.command { + Command::List => cmd_list(), + Command::Generate { + output, + api_key, + version, + hex, + } => cmd_generate(output, api_key, version, hex).await?, + Command::Send { + host, + api_key, + version, + timeout_ms, + } => { + let (ok, fail) = run_send(&host, api_key, version, timeout_ms).await?; + println!("\nResult: {ok} OK {fail} failed"); + } + Command::Verify { host, .. } => { + let (ok, fail) = run_send(&host, None, None, 5000).await?; + println!("\n=== Verify: {ok} passed {fail} failed ==="); + if fail > 0 { + std::process::exit(1); + } + } + } + Ok(()) +} From b46c6c5acec102c6d593cc81a51a45b2266aae45 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 7 Jun 2026 08:25:43 -0400 Subject: [PATCH 02/15] kafka: Initial version - codec improvements, broker & server Add tokio-util dependency and implement BrokerAdvertise to advertise host/port from server bind address. Replace dynamic supported_api_ranges with a static table and pass BrokerAdvertise into handler so metadata responses include the advertised broker. Harden codec primitives: add MAX_COLLECTION_LEN, checked read_i32_array_count and read_compact_array_count, tagged-field bounds checks, better varint validation, and string-length checks (write_nullable_string returns Result). Update response encoders to take references and use checked conversions for counts. Improve server: use tokio_util::TaskTracker for graceful shutdown, handle transient accept errors, return error-only response for unsupported request header versions, extract correlation id, and add safer frame read/write size checks. Update docs and tests to reflect compact/flexible encoding and new APIs. --- Cargo.lock | 1 + gateways/kafka/Cargo.toml | 1 + gateways/kafka/docs/SCOPE.md | 44 +++- gateways/kafka/src/error.rs | 8 + gateways/kafka/src/lib.rs | 9 - gateways/kafka/src/protocol/api.rs | 219 ++++++++++++------ gateways/kafka/src/protocol/codec.rs | 65 +++++- gateways/kafka/src/protocol/header.rs | 6 + gateways/kafka/src/protocol/requests.rs | 50 ++-- gateways/kafka/src/protocol/responses.rs | 89 ++++--- gateways/kafka/src/server.rs | 107 +++++++-- gateways/kafka/tests/api_handler_tests.rs | 20 +- gateways/kafka/tests/codec_tests.rs | 4 +- .../kafka/tests/decode_validation_tests.rs | 22 +- .../kafka/tests/golden_wire_fixtures_tests.rs | 9 +- gateways/kafka/tests/header_tests.rs | 4 +- .../kafka/tests/server_integration_tests.rs | 2 +- 17 files changed, 463 insertions(+), 197 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d9b4e8efb..ff068265c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7086,6 +7086,7 @@ dependencies = [ "bytes", "thiserror 2.0.18", "tokio", + "tokio-util", "tracing", "tracing-subscriber", ] diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 6bfcb0e34d..93a335d68a 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -36,6 +36,7 @@ path = "src/main.rs" bytes = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time", "sync", "signal"] } +tokio-util = { workspace = true, features = ["rt"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index b56a172730..8d2be24623 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -2,20 +2,46 @@ This gateway iteration implements **wire validation and stub responses only** (no Iggy backend, no real broker semantics). +Source of truth in code: `SUPPORTED_RANGES` in [`src/protocol/api.rs`](../src/protocol/api.rs). + ## Supported API keys and versions -| API key | Name | Min version | Max version | Behavior | -|---------|------|-------------|-------------|----------| -| 18 | ApiVersions | 0 | 3 | Advertise supported ranges; flexible encoding at v3+ | -| 3 | Metadata | 0 | 9 | Decode request; stub broker `127.0.0.1:9093` | -| 0 | Produce | 3 | 9 | Decode request; stub response | -| 1 | Fetch | 4 | 12 | Decode request; stub response | -| 2 | ListOffsets | 1 | 6 | Decode request; stub response | -| 19 | CreateTopics | 2 | 5 | Decode request; stub response | +| API key | Name | Min version | Max version | Valid versions | Behavior | +|---------|------|-------------|-------------|----------------|----------| +| 18 | ApiVersions | 0 | 3 | 0, 1, 2, 3 | Advertise supported ranges; flexible encoding at v3+ | +| 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode request; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | +| 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub response | +| 1 | Fetch | 4 | 12 | 4, 5, 6, 7, 8, 9, 10, 11, 12 | Decode request; stub response | +| 2 | ListOffsets | 1 | 6 | 1, 2, 3, 4, 5, 6 | Decode request; stub response | +| 19 | CreateTopics | 2 | 5 | 2, 3, 4, 5 | Decode request; stub response | + +A request is accepted when `min_version ≤ api_version ≤ max_version` for that API key. Any other version for a listed key, or any unlisted API key, receives `UNSUPPORTED_VERSION` (35). + +## Valid versions reference (by API key) + +Use this table when configuring clients or generating wire fixtures with `kafka-message-gen`. + +| API key | Name | Valid versions (inclusive range) | Flexible wire encoding from | +|---------|------|----------------------------------|----------------------------| +| 0 | Produce | 3–9 | v9 | +| 1 | Fetch | 4–12 | v12 | +| 2 | ListOffsets | 1–6 | v6 | +| 3 | Metadata | 0–9 | v9 | +| 18 | ApiVersions | 0–3 | v3 | +| 19 | CreateTopics | 2–5 | v5 | ## Unsupported API keys -All other API keys receive an error-only response with `UNSUPPORTED_VERSION` (35). +All API keys not listed above receive an error-only response with `UNSUPPORTED_VERSION` (35). Examples not in this foundation scope: + +| API key | Name | Notes | +|---------|------|-------| +| 8 | OffsetCommit | Consumer group — later issue | +| 9 | OffsetFetch | Consumer group — later issue | +| 10 | FindCoordinator | Consumer group — later issue | +| 11–16 | JoinGroup, Heartbeat, LeaveGroup, SyncGroup, DescribeGroups, ListGroups | Consumer group — later issue | +| 17 | SaslHandshake | Auth — later issue | +| 20+ | DeleteTopics, InitProducerId, transactions, ACLs, etc. | Later issues | ## Out of scope (later issues) diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs index 668fc5c442..7eee4b16b8 100644 --- a/gateways/kafka/src/error.rs +++ b/gateways/kafka/src/error.rs @@ -34,6 +34,14 @@ pub enum KafkaProtocolError { InvalidVarint, #[error("unsupported request header version: {0}")] UnsupportedHeaderVersion(i16), + #[error("invalid array length: {0}")] + InvalidArrayLength(i32), + #[error("invalid compact array length: encoded value must be >= 1, got {0}")] + InvalidCompactArrayLength(u64), + #[error("collection length {count} exceeds maximum {max}")] + CollectionTooLarge { count: usize, max: usize }, + #[error("string length {length} exceeds i16::MAX")] + StringTooLong { length: usize }, #[error("io error: {0}")] Io(#[from] std::io::Error), } diff --git a/gateways/kafka/src/lib.rs b/gateways/kafka/src/lib.rs index f0828508c7..c8f0cf9e20 100644 --- a/gateways/kafka/src/lib.rs +++ b/gateways/kafka/src/lib.rs @@ -17,15 +17,6 @@ //! Kafka wire protocol gateway foundation for Apache Iggy. -// Ported wire codec from spike; pedantic clippy cleanup is a follow-up. -#![allow( - clippy::pedantic, - clippy::missing_const_for_fn, - clippy::wildcard_imports, - clippy::match_same_arms, - clippy::needless_pass_by_value -)] - pub mod error; pub mod protocol; pub mod server; diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 8f0367b481..275c542e8a 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::net::SocketAddr; + use bytes::Bytes; use crate::protocol::codec::{Decoder, Encoder}; @@ -61,6 +63,37 @@ pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; pub const ERROR_INVALID_REQUEST: i16 = 42; pub const ERROR_UNSUPPORTED_FOR_MESSAGE_FORMAT: i16 = 43; +#[derive(Debug, Clone)] +pub struct BrokerAdvertise { + pub host: String, + pub port: i32, +} + +impl BrokerAdvertise { + #[must_use] + pub fn from_bind_addr(bind_addr: &str) -> Self { + bind_addr.parse::().map_or_else( + |_| Self { + host: "127.0.0.1".to_string(), + port: 9093, + }, + |addr| Self { + host: addr.ip().to_string(), + port: i32::from(addr.port()), + }, + ) + } +} + +impl Default for BrokerAdvertise { + fn default() -> Self { + Self { + host: "127.0.0.1".to_string(), + port: 9093, + } + } +} + #[derive(Debug, Clone, Copy)] pub struct ApiVersionRange { pub api_key: i16, @@ -68,42 +101,50 @@ pub struct ApiVersionRange { pub max_version: i16, } -pub fn supported_api_ranges() -> Vec { - vec![ - ApiVersionRange { - api_key: API_KEY_PRODUCE, - min_version: 3, - max_version: 9, - }, - ApiVersionRange { - api_key: API_KEY_FETCH, - min_version: 4, - max_version: 12, - }, - ApiVersionRange { - api_key: API_KEY_LIST_OFFSETS, - min_version: 1, - max_version: 6, - }, - ApiVersionRange { - api_key: API_KEY_METADATA, - min_version: 0, - max_version: 9, - }, - ApiVersionRange { - api_key: API_KEY_API_VERSIONS, - min_version: 0, - max_version: 3, - }, - ApiVersionRange { - api_key: API_KEY_CREATE_TOPICS, - min_version: 2, - max_version: 5, - }, - ] +static SUPPORTED_RANGES: &[ApiVersionRange] = &[ + ApiVersionRange { + api_key: API_KEY_PRODUCE, + min_version: 3, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_FETCH, + min_version: 4, + max_version: 12, + }, + ApiVersionRange { + api_key: API_KEY_LIST_OFFSETS, + min_version: 1, + max_version: 6, + }, + ApiVersionRange { + api_key: API_KEY_METADATA, + min_version: 0, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_API_VERSIONS, + min_version: 0, + max_version: 3, + }, + ApiVersionRange { + api_key: API_KEY_CREATE_TOPICS, + min_version: 2, + max_version: 5, + }, +]; + +#[must_use] +pub fn supported_api_ranges() -> &'static [ApiVersionRange] { + SUPPORTED_RANGES } -pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { +pub fn handle_request( + api_key: i16, + api_version: i16, + body: Bytes, + broker: &BrokerAdvertise, +) -> Bytes { match api_key { API_KEY_API_VERSIONS => { if is_supported_version(api_key, api_version) { @@ -114,15 +155,15 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { } API_KEY_METADATA => { if is_supported_version(api_key, api_version) { - encode_metadata_response(api_version, body, ERROR_NONE) + encode_metadata_response(api_version, body, broker, ERROR_NONE) } else { - encode_metadata_response(0, body, ERROR_UNSUPPORTED_VERSION) + encode_metadata_response(0, body, broker, ERROR_UNSUPPORTED_VERSION) } } API_KEY_PRODUCE => { if is_supported_version(api_key, api_version) { match decode_produce_request(api_version, body) { - Ok(req) => encode_produce_response(api_version, req), + Ok(req) => encode_produce_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode Produce request: {:?}", e); encode_error_only_response(ERROR_CORRUPT_MESSAGE) @@ -135,7 +176,7 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { API_KEY_FETCH => { if is_supported_version(api_key, api_version) { match decode_fetch_request(api_version, body) { - Ok(req) => encode_fetch_response(api_version, req), + Ok(req) => encode_fetch_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode Fetch request: {:?}", e); encode_error_only_response(ERROR_CORRUPT_MESSAGE) @@ -148,7 +189,7 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { API_KEY_LIST_OFFSETS => { if is_supported_version(api_key, api_version) { match decode_list_offsets_request(api_version, body) { - Ok(req) => encode_list_offsets_response(api_version, req), + Ok(req) => encode_list_offsets_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode ListOffsets request: {:?}", e); encode_error_only_response(ERROR_CORRUPT_MESSAGE) @@ -161,7 +202,7 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { API_KEY_CREATE_TOPICS => { if is_supported_version(api_key, api_version) { match decode_create_topics_request(api_version, body) { - Ok(req) => encode_create_topics_response(api_version, req), + Ok(req) => encode_create_topics_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode CreateTopics request: {:?}", e); encode_error_only_response(ERROR_CORRUPT_MESSAGE) @@ -175,31 +216,32 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { } } +#[must_use] pub fn is_supported_version(api_key: i16, api_version: i16) -> bool { - supported_api_ranges() - .into_iter() + SUPPORTED_RANGES + .iter() .find(|r| r.api_key == api_key) .is_some_and(|r| api_version >= r.min_version && api_version <= r.max_version) } fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { let flexible = api_version >= 3; - let ranges = supported_api_ranges(); + let ranges = SUPPORTED_RANGES; let mut e = Encoder::with_capacity(128); e.write_i16(error_code); if flexible { e.write_varint((ranges.len() + 1) as u64); - for r in &ranges { + for r in ranges { e.write_i16(r.api_key); e.write_i16(r.min_version); e.write_i16(r.max_version); e.write_empty_tagged_fields(); } } else { - e.write_i32(ranges.len() as i32); - for r in &ranges { + e.write_i32(i32::try_from(ranges.len()).expect("supported range table is small")); + for r in ranges { e.write_i16(r.api_key); e.write_i16(r.min_version); e.write_i16(r.max_version); @@ -217,37 +259,82 @@ fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { e.freeze() } -fn encode_metadata_response(_api_version: i16, body: Bytes, top_level_error_code: i16) -> Bytes { +fn encode_metadata_response( + api_version: i16, + body: Bytes, + broker: &BrokerAdvertise, + top_level_error_code: i16, +) -> Bytes { + let flexible = api_version >= 9; + let topics_count = split_metadata_request_topics(body, api_version); + let topic_error = if top_level_error_code == ERROR_NONE { + ERROR_UNKNOWN_TOPIC_OR_PARTITION + } else { + top_level_error_code + }; + let mut e = Encoder::with_capacity(256); - e.write_i32(1); - e.write_i32(1); - e.write_nullable_string(Some("127.0.0.1")); - e.write_i32(9093); - - let topics_count = split_metadata_request_topics(body); - e.write_i32(topics_count as i32); - for _ in 0..topics_count { - e.write_i16(if top_level_error_code == ERROR_NONE { - ERROR_UNKNOWN_TOPIC_OR_PARTITION - } else { - top_level_error_code - }); - e.write_nullable_string(Some("unknown-topic")); - e.write_i32(0); + if api_version >= 1 { + e.write_i32(0); // throttle_time_ms + } + + if flexible { + e.write_varint(2); // one broker (N+1) + e.write_i32(1); + e.write_compact_nullable_string(Some(&broker.host)); + e.write_i32(broker.port); + e.write_compact_nullable_string(None); // rack + e.write_empty_tagged_fields(); + + if api_version >= 2 { + e.write_compact_nullable_string(None); // cluster_id + } + e.write_i32(1); // controller_id + + e.write_varint((topics_count + 1) as u64); + for _ in 0..topics_count { + e.write_i16(topic_error); + e.write_compact_nullable_string(Some("unknown-topic")); + e.write_varint(1); // empty partitions array + if api_version >= 4 { + e.write_bool(false); // is_internal + } + e.write_empty_tagged_fields(); + } + e.write_empty_tagged_fields(); + } else { + e.write_i32(1); + e.write_i32(1); + let _ = e.write_nullable_string(Some(&broker.host)); + e.write_i32(broker.port); + + e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); + for _ in 0..topics_count { + e.write_i16(topic_error); + let _ = e.write_nullable_string(Some("unknown-topic")); + e.write_i32(0); + } + + e.write_i32(1); // controller_id } - e.write_i32(1); e.freeze() } -fn encode_error_only_response(error_code: i16) -> Bytes { +#[must_use] +pub fn encode_error_only_response(error_code: i16) -> Bytes { let mut e = Encoder::with_capacity(2); e.write_i16(error_code); e.freeze() } -pub fn split_metadata_request_topics(body: Bytes) -> usize { +#[must_use] +pub fn split_metadata_request_topics(body: Bytes, api_version: i16) -> usize { let mut d = Decoder::new(body); - d.read_i32().unwrap_or_default().max(0) as usize + if api_version >= 9 { + d.read_compact_array_count().unwrap_or(0) + } else { + d.read_i32_array_count().unwrap_or(0) + } } diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 1e356b8608..0429beafc1 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -15,10 +15,17 @@ // specific language governing permissions and limitations // under the License. +//! Low-level Kafka primitive encoders/decoders (ported wire codec). + +#![allow(clippy::pedantic, clippy::missing_const_for_fn)] + use bytes::{Buf, BufMut, Bytes, BytesMut}; use crate::error::{KafkaProtocolError, Result}; +/// Upper bound for Kafka array/collection element counts decoded from the wire. +pub const MAX_COLLECTION_LEN: usize = 65_536; + pub struct Decoder { bytes: Bytes, } @@ -69,6 +76,9 @@ impl Decoder { let mut shift = 0u32; loop { let byte = self.read_u8()?; + if shift == 63 && byte & 0x7E != 0 { + return Err(KafkaProtocolError::InvalidVarint); + } result |= ((byte & 0x7F) as u64) << shift; if byte & 0x80 == 0 { return Ok(result); @@ -80,6 +90,41 @@ impl Decoder { } } + /// Legacy array length: signed i32 count (must be non-negative). + pub fn read_i32_array_count(&mut self) -> Result { + let n = self.read_i32()?; + if n < 0 { + return Err(KafkaProtocolError::InvalidArrayLength(n)); + } + let count = usize::try_from(n).map_err(|_| KafkaProtocolError::CollectionTooLarge { + count: n as usize, + max: MAX_COLLECTION_LEN, + })?; + if count > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count, + max: MAX_COLLECTION_LEN, + }); + } + Ok(count) + } + + /// Compact array length: unsigned varint holding `element_count + 1`. + pub fn read_compact_array_count(&mut self) -> Result { + let n = self.read_varint()?; + if n == 0 { + return Err(KafkaProtocolError::InvalidCompactArrayLength(0)); + } + let count = (n - 1) as usize; + if count > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count, + max: MAX_COLLECTION_LEN, + }); + } + Ok(count) + } + /// Legacy nullable string: i16 length prefix (-1 = null). pub fn read_nullable_string(&mut self) -> Result> { let len = self.read_i16()?; @@ -138,7 +183,17 @@ impl Decoder { /// Skip over a tagged-fields section. Each field is: tag (varint) + size (varint) + bytes. /// A count of 0 is the common case (single byte 0x00). pub fn read_tagged_fields(&mut self) -> Result<()> { - let count = self.read_varint()? as usize; + let count = self.read_varint()?; + let count = usize::try_from(count).map_err(|_| KafkaProtocolError::CollectionTooLarge { + count: count as usize, + max: MAX_COLLECTION_LEN, + })?; + if count > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count, + max: MAX_COLLECTION_LEN, + }); + } for _ in 0..count { self.read_varint()?; // tag number let size = self.read_varint()? as usize; @@ -206,14 +261,18 @@ impl Encoder { } /// Legacy nullable string: i16 length prefix, -1 for null. - pub fn write_nullable_string(&mut self, v: Option<&str>) { + pub fn write_nullable_string(&mut self, v: Option<&str>) -> Result<()> { match v { None => self.write_i16(-1), Some(s) => { - self.write_i16(s.len() as i16); + if s.len() > i16::MAX as usize { + return Err(KafkaProtocolError::StringTooLong { length: s.len() }); + } + self.write_i16(i16::try_from(s.len()).expect("checked above")); self.bytes.put_slice(s.as_bytes()); } } + Ok(()) } /// Compact nullable string (flexible versions): varint(len+1), 0 for null. diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index fb137b81e0..93f90e5c5e 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -15,6 +15,12 @@ // specific language governing permissions and limitations // under the License. +#![allow( + clippy::pedantic, + clippy::missing_const_for_fn, + clippy::match_same_arms +)] + use bytes::Bytes; use crate::error::{KafkaProtocolError, Result}; diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index 786fa0b88c..6f4daa8996 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -17,6 +17,8 @@ //! Kafka request decoders for critical API keys +#![allow(clippy::pedantic)] + use crate::error::Result; use crate::protocol::codec::Decoder; use bytes::Bytes; @@ -62,9 +64,9 @@ pub fn decode_produce_request(version: i16, body: Bytes) -> Result Result Result { // topics array let topics_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; let mut topics = Vec::with_capacity(topics_count); @@ -174,9 +176,9 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { }; let partitions_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; let mut partitions = Vec::with_capacity(partitions_count); @@ -219,21 +221,21 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { // forgotten_topics_data (v7+) — skip if version >= 7 { let forgotten_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; for _ in 0..forgotten_count { if flexible { d.read_compact_nullable_string()?; - let partitions_count = (d.read_varint()? - 1) as usize; + let partitions_count = d.read_compact_array_count()?; for _ in 0..partitions_count { d.read_i32()?; } d.read_tagged_fields()?; } else { d.read_nullable_string()?; - let partitions_count = d.read_i32()? as usize; + let partitions_count = d.read_i32_array_count()?; for _ in 0..partitions_count { d.read_i32()?; } @@ -291,9 +293,9 @@ pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result= 2 { d.read_i8()? } else { 0 }; let topics_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; let mut topics = Vec::with_capacity(topics_count); @@ -305,9 +307,9 @@ pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result Result= 5; let topics_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; let mut topics = Vec::with_capacity(topics_count); @@ -388,16 +390,16 @@ pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result Result Bytes { +pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { let flexible = version >= 9; let mut e = Encoder::with_capacity(512); if flexible { e.write_varint((req.topics.len() + 1) as u64); } else { - e.write_i32(req.topics.len() as i32); + e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); } for topic in &req.topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - e.write_nullable_string(Some(&topic.topic)); + let _ = e.write_nullable_string(Some(&topic.topic)); } if flexible { e.write_varint((topic.partitions.len() + 1) as u64); } else { - e.write_i32(topic.partitions.len() as i32); + e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); } for p in &topic.partitions { e.write_i32(p.partition); e.write_i16(ERROR_NONE); - e.write_i64(0); // base_offset — TODO: return real offset from Iggy + e.write_i64(0); if version >= 2 { - e.write_i64(-1); // log_append_time_ms (-1 = not set) + e.write_i64(-1); } if version >= 5 { - e.write_i64(0); // log_start_offset + e.write_i64(0); } - // record_errors[] and error_message added in v8 if version >= 8 { if flexible { - e.write_varint(1); // empty COMPACT_ARRAY - e.write_compact_nullable_string(None); // error_message = null + e.write_varint(1); + e.write_compact_nullable_string(None); } else { - e.write_i32(0); // empty ARRAY - e.write_nullable_string(None); // error_message = null + e.write_i32(0); + let _ = e.write_nullable_string(None); } } if flexible { @@ -76,7 +79,7 @@ pub fn encode_produce_response(version: i16, req: ProduceRequest) -> Bytes { } if version >= 1 { - e.write_i32(0); // throttle_time_ms + e.write_i32(0); } if flexible { e.write_empty_tagged_fields(); @@ -85,49 +88,48 @@ pub fn encode_produce_response(version: i16, req: ProduceRequest) -> Bytes { e.freeze() } -pub fn encode_fetch_response(version: i16, req: FetchRequest) -> Bytes { +pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { let flexible = version >= 12; let mut e = Encoder::with_capacity(512); if version >= 1 { - e.write_i32(0); // throttle_time_ms + e.write_i32(0); } if version >= 7 { - e.write_i16(ERROR_NONE); // error_code - e.write_i32(0); // session_id + e.write_i16(ERROR_NONE); + e.write_i32(0); } if flexible { e.write_varint((req.topics.len() + 1) as u64); } else { - e.write_i32(req.topics.len() as i32); + e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); } for topic in &req.topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - e.write_nullable_string(Some(&topic.topic)); + let _ = e.write_nullable_string(Some(&topic.topic)); } if flexible { e.write_varint((topic.partitions.len() + 1) as u64); } else { - e.write_i32(topic.partitions.len() as i32); + e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); } for partition in &topic.partitions { e.write_i32(partition.partition); e.write_i16(ERROR_NONE); - e.write_i64(0); // high_watermark — TODO: get from Iggy + e.write_i64(0); if version >= 4 { - e.write_i64(0); // last_stable_offset + e.write_i64(0); } if version >= 5 { - e.write_i64(0); // log_start_offset + e.write_i64(0); } if version >= 4 { - // aborted_transactions[] if flexible { e.write_varint(1); } else { @@ -135,9 +137,8 @@ pub fn encode_fetch_response(version: i16, req: FetchRequest) -> Bytes { } } if version >= 11 { - e.write_i32(-1); // preferred_read_replica + e.write_i32(-1); } - // records (empty — TODO: call Iggy poll_messages) if flexible { e.write_compact_nullable_bytes(None); } else { @@ -160,44 +161,42 @@ pub fn encode_fetch_response(version: i16, req: FetchRequest) -> Bytes { e.freeze() } -pub fn encode_list_offsets_response(version: i16, req: ListOffsetsRequest) -> Bytes { +pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> Bytes { let flexible = version >= 6; let mut e = Encoder::with_capacity(256); if version >= 2 { - e.write_i32(0); // throttle_time_ms + e.write_i32(0); } if flexible { e.write_varint((req.topics.len() + 1) as u64); } else { - e.write_i32(req.topics.len() as i32); + e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); } for topic in &req.topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - e.write_nullable_string(Some(&topic.topic)); + let _ = e.write_nullable_string(Some(&topic.topic)); } if flexible { e.write_varint((topic.partitions.len() + 1) as u64); } else { - e.write_i32(topic.partitions.len() as i32); + e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); } for partition in &topic.partitions { e.write_i32(partition.partition); e.write_i16(ERROR_NONE); - // TODO: query Iggy for actual offsets let offset = 0i64; if version >= 1 { - e.write_i64(1_700_000_000_000); // timestamp placeholder + e.write_i64(1_700_000_000_000); } e.write_i64(offset); - // leader_epoch was added in v4, not v1 if version >= 4 { e.write_i32(-1); } @@ -218,25 +217,25 @@ pub fn encode_list_offsets_response(version: i16, req: ListOffsetsRequest) -> By e.freeze() } -pub fn encode_create_topics_response(version: i16, req: CreateTopicsRequest) -> Bytes { +pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> Bytes { let flexible = version >= 5; let mut e = Encoder::with_capacity(256); if version >= 2 { - e.write_i32(0); // throttle_time_ms + e.write_i32(0); } if flexible { e.write_varint((req.topics.len() + 1) as u64); } else { - e.write_i32(req.topics.len() as i32); + e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); } for topic in &req.topics { if flexible { e.write_compact_nullable_string(Some(&topic.name)); } else { - e.write_nullable_string(Some(&topic.name)); + let _ = e.write_nullable_string(Some(&topic.name)); } let error_code = if topic.num_partitions <= 0 { @@ -248,17 +247,17 @@ pub fn encode_create_topics_response(version: i16, req: CreateTopicsRequest) -> if version >= 1 { if flexible { - e.write_compact_nullable_string(None); // error_message + e.write_compact_nullable_string(None); } else { - e.write_nullable_string(None); + let _ = e.write_nullable_string(None); } } if version >= 5 { - e.write_i16(ERROR_NONE); // topic_config_error_code (added in v5) + e.write_i16(ERROR_NONE); e.write_i32(topic.num_partitions); e.write_i16(topic.replication_factor); - e.write_varint(1); // configs[] empty COMPACT_ARRAY + e.write_varint(1); } if flexible { diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index c6465eb0ca..55b3e509bc 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -24,10 +24,13 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::broadcast; use tokio::time::timeout; +use tokio_util::task::TaskTracker; use tracing::{error, info, warn}; use crate::error::{KafkaProtocolError, Result}; -use crate::protocol::api::handle_request; +use crate::protocol::api::{ + BrokerAdvertise, ERROR_INVALID_REQUEST, encode_error_only_response, handle_request, +}; use crate::protocol::codec::Decoder; use crate::protocol::header::{ RequestHeader, ResponseHeader, request_header_version, response_header_version, @@ -57,30 +60,49 @@ pub struct KafkaServer { } impl KafkaServer { + #[must_use] pub fn new(config: ServerConfig) -> Self { Self { config: Arc::new(config), } } + /// Accept Kafka wire connections until `shutdown` fires, then drain in-flight tasks. + /// + /// # Errors + /// + /// Returns an error if binding fails or a non-transient `accept()` error occurs. pub async fn run(self, mut shutdown: broadcast::Receiver<()>) -> Result<()> { let listener = TcpListener::bind(&self.config.bind_addr).await?; info!("kafka listener bound on {}", self.config.bind_addr); + let tracker = TaskTracker::new(); + let broker = BrokerAdvertise::from_bind_addr(&self.config.bind_addr); + loop { tokio::select! { _ = shutdown.recv() => { info!("kafka listener shutdown requested"); + tracker.close(); + tracker.wait().await; break; } accept_result = listener.accept() => { - let (stream, peer) = accept_result?; - let cfg = Arc::clone(&self.config); - tokio::spawn(async move { - if let Err(err) = handle_connection(stream, cfg, peer).await { - warn!(%peer, "connection closed with error: {err}"); + match accept_result { + Ok((stream, peer)) => { + let cfg = Arc::clone(&self.config); + let broker = broker.clone(); + tracker.spawn(async move { + if let Err(err) = handle_connection(stream, cfg, peer, broker).await { + warn!(%peer, "connection closed with error: {err}"); + } + }); } - }); + Err(e) if is_transient_accept_error(&e) => { + warn!(%e, "transient accept error, continuing"); + } + Err(e) => return Err(e.into()), + } } } } @@ -88,10 +110,24 @@ impl KafkaServer { } } +fn is_transient_accept_error(err: &std::io::Error) -> bool { + use std::io::ErrorKind; + + matches!( + err.kind(), + ErrorKind::Interrupted | ErrorKind::ConnectionAborted | ErrorKind::WouldBlock + ) || matches!( + err.raw_os_error(), + // EMFILE / ENFILE are common across Unix platforms when fd limits are hit. + Some(23 | 24) + ) +} + async fn handle_connection( mut stream: TcpStream, config: Arc, peer: SocketAddr, + broker: BrokerAdvertise, ) -> Result<()> { info!(%peer, "connection accepted"); @@ -119,9 +155,26 @@ async fn handle_connection( let api_version = i16::from_be_bytes([frame[2], frame[3]]); let req_hdr_ver = request_header_version(api_key, api_version); let resp_hdr_ver = response_header_version(api_key, api_version); + let correlation_id = correlation_id_from_frame(&frame); let mut decoder = Decoder::new(frame); - let req = RequestHeader::decode_from(&mut decoder, req_hdr_ver)?; + let req = match RequestHeader::decode_from(&mut decoder, req_hdr_ver) { + Ok(req) => req, + Err(KafkaProtocolError::UnsupportedHeaderVersion(_)) => { + warn!(%peer, api_key, api_version, "unsupported request header version"); + let body_response = encode_error_only_response(ERROR_INVALID_REQUEST); + let resp_header = ResponseHeader { correlation_id }; + let encoded_header = resp_header.encode(0); + let mut payload = + BytesMut::with_capacity(encoded_header.len() + body_response.len()); + payload.put_slice(&encoded_header); + payload.put_slice(&body_response); + write_frame(&mut stream, &payload, config.write_timeout).await?; + return Ok(()); + } + Err(e) => return Err(e), + }; + info!( %peer, api_key = req.api_key, @@ -132,7 +185,7 @@ async fn handle_connection( ); let body = decoder.read_bytes(decoder.remaining())?; - let body_response = handle_request(req.api_key, req.api_version, body); + let body_response = handle_request(req.api_key, req.api_version, body, &broker); let resp_header = ResponseHeader { correlation_id: req.correlation_id, @@ -146,6 +199,19 @@ async fn handle_connection( } } +fn correlation_id_from_frame(frame: &bytes::Bytes) -> i32 { + if frame.len() >= 8 { + i32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]) + } else { + 0 + } +} + +/// Read one length-prefixed Kafka frame from `stream`. +/// +/// # Errors +/// +/// Returns an error on timeout, invalid length, or I/O failure. pub async fn read_frame( stream: &mut TcpStream, max_frame_size: usize, @@ -156,12 +222,16 @@ pub async fn read_frame( .await .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; - let frame_len = i32::from_be_bytes(len_buf); - if frame_len <= 0 { - return Err(KafkaProtocolError::InvalidFrameLength(frame_len)); + let frame_len_i32 = i32::from_be_bytes(len_buf); + if frame_len_i32 <= 0 { + return Err(KafkaProtocolError::InvalidFrameLength(frame_len_i32)); } - let frame_len = frame_len as usize; + let frame_len = + usize::try_from(frame_len_i32).map_err(|_| KafkaProtocolError::FrameTooLarge { + max_bytes: max_frame_size, + actual_bytes: u32::MAX as usize, + })?; if frame_len > max_frame_size { return Err(KafkaProtocolError::FrameTooLarge { max_bytes: max_frame_size, @@ -176,6 +246,11 @@ pub async fn read_frame( Ok(bytes::Bytes::from(data)) } +/// Write one length-prefixed Kafka frame to `stream`. +/// +/// # Errors +/// +/// Returns an error on timeout, oversize payload, or I/O failure. pub async fn write_frame( stream: &mut TcpStream, payload: &[u8], @@ -189,7 +264,11 @@ pub async fn write_frame( }); } let mut frame = BytesMut::with_capacity(4 + len); - frame.put_i32(len as i32); + let len_i32 = i32::try_from(len).map_err(|_| KafkaProtocolError::FrameTooLarge { + max_bytes: i32::MAX as usize, + actual_bytes: len, + })?; + frame.put_i32(len_i32); frame.extend_from_slice(payload); timeout(write_timeout, stream.write_all(&frame)) .await diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 5fb8b0c6c2..29d5a5423f 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -18,16 +18,20 @@ use bytes::Bytes; use iggy_gateway_kafka::protocol::api::{ - API_KEY_API_VERSIONS, API_KEY_METADATA, ERROR_UNSUPPORTED_VERSION, handle_request, - is_supported_version, split_metadata_request_topics, supported_api_ranges, + API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, ERROR_UNSUPPORTED_VERSION, + handle_request, is_supported_version, split_metadata_request_topics, supported_api_ranges, }; + +fn test_broker() -> BrokerAdvertise { + BrokerAdvertise::default() +} use iggy_gateway_kafka::protocol::codec::Decoder; // ── ApiVersions ───────────────────────────────────────────────────────────── #[test] fn api_versions_v1_response_non_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new()); + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &test_broker()); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -51,7 +55,7 @@ fn api_versions_v1_response_non_flexible_format() { #[test] fn api_versions_v3_response_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new()); + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &test_broker()); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -81,7 +85,7 @@ fn api_versions_v3_response_flexible_format() { #[test] fn metadata_response_has_broker_array_and_topic_array() { - let body = handle_request(API_KEY_METADATA, 0, Bytes::new()); + let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &test_broker()); let mut d = Decoder::new(body); let broker_count = d.read_i32().unwrap(); @@ -101,7 +105,7 @@ fn metadata_response_has_broker_array_and_topic_array() { fn unsupported_version_returns_protocol_error() { let mut req = Vec::new(); req.extend_from_slice(&1_i32.to_be_bytes()); - let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req)); + let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req), &test_broker()); let mut d = Decoder::new(body); let _broker_count = d.read_i32().unwrap(); let _ = d.read_i32().unwrap(); @@ -123,7 +127,7 @@ fn unsupported_version_returns_protocol_error() { #[test] fn unknown_api_key_returns_error_only_payload() { - let body = handle_request(999, 0, Bytes::new()); + let body = handle_request(999, 0, Bytes::new(), &test_broker()); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } @@ -132,7 +136,7 @@ fn unknown_api_key_returns_error_only_payload() { fn metadata_topic_split_reads_array_count() { let mut raw = Vec::new(); raw.extend_from_slice(&2_i32.to_be_bytes()); - assert_eq!(split_metadata_request_topics(Bytes::from(raw)), 2); + assert_eq!(split_metadata_request_topics(Bytes::from(raw), 0), 2); } #[test] diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs index 093d123451..0b25fe7929 100644 --- a/gateways/kafka/tests/codec_tests.rs +++ b/gateways/kafka/tests/codec_tests.rs @@ -26,8 +26,8 @@ fn codec_round_trip_primitives_and_nullable_fields() { enc.write_i16(42); enc.write_i32(123_456); enc.write_i64(9_999_999); - enc.write_nullable_string(Some("client-a")); - enc.write_nullable_string(None); + enc.write_nullable_string(Some("client-a")).unwrap(); + enc.write_nullable_string(None).unwrap(); enc.write_nullable_bytes(Some(&[1, 2, 3])); enc.write_nullable_bytes(None); let bytes = enc.freeze(); diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index d9f9cab3f9..eaf2cf64c0 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -113,7 +113,7 @@ fn produce_response_encodes_for_all_supported_versions() { let body = load_body(0, "Produce", version); let req = decode_produce_request(version, body) .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); - let resp = encode_produce_response(version, req); + let resp = encode_produce_response(version, &req); assert!( !resp.is_empty(), "Produce v{version}: response must not be empty" @@ -126,7 +126,7 @@ fn produce_response_v3_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(0, "Produce", 3); let req = decode_produce_request(3, body).unwrap(); - let resp = encode_produce_response(3, req); + let resp = encode_produce_response(3, &req); let mut d = Decoder::new(resp); let topic_count = d.read_i32().unwrap(); @@ -153,7 +153,7 @@ fn produce_response_v8_includes_record_errors() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(0, "Produce", 8); let req = decode_produce_request(8, body).unwrap(); - let resp = encode_produce_response(8, req); + let resp = encode_produce_response(8, &req); let mut d = Decoder::new(resp); let topic_count = d.read_i32().unwrap(); @@ -217,7 +217,7 @@ fn fetch_response_encodes_for_all_supported_versions() { let body = load_body(1, "Fetch", version); let req = decode_fetch_request(version, body) .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); - let resp = encode_fetch_response(version, req); + let resp = encode_fetch_response(version, &req); assert!( !resp.is_empty(), "Fetch v{version}: response must not be empty" @@ -230,7 +230,7 @@ fn fetch_response_v7_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(1, "Fetch", 7); let req = decode_fetch_request(7, body).unwrap(); - let resp = encode_fetch_response(7, req); + let resp = encode_fetch_response(7, &req); let mut d = Decoder::new(resp); let throttle_ms = d.read_i32().unwrap(); // v1+ @@ -289,7 +289,7 @@ fn list_offsets_response_encodes_for_all_supported_versions() { let body = load_body(2, "ListOffsets", version); let req = decode_list_offsets_request(version, body) .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); - let resp = encode_list_offsets_response(version, req); + let resp = encode_list_offsets_response(version, &req); assert!( !resp.is_empty(), "ListOffsets v{version}: response must not be empty" @@ -302,7 +302,7 @@ fn list_offsets_response_v1_no_leader_epoch() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(2, "ListOffsets", 1); let req = decode_list_offsets_request(1, body).unwrap(); - let resp = encode_list_offsets_response(1, req); + let resp = encode_list_offsets_response(1, &req); let mut d = Decoder::new(resp); // v1: no throttle_time_ms @@ -329,7 +329,7 @@ fn list_offsets_response_v4_has_leader_epoch() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(2, "ListOffsets", 4); let req = decode_list_offsets_request(4, body).unwrap(); - let resp = encode_list_offsets_response(4, req); + let resp = encode_list_offsets_response(4, &req); let mut d = Decoder::new(resp); let _throttle = d.read_i32().unwrap(); // v2+ @@ -387,7 +387,7 @@ fn create_topics_response_encodes_for_all_supported_versions() { let body = load_body(19, "CreateTopics", version); let req = decode_create_topics_request(version, body) .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); - let resp = encode_create_topics_response(version, req); + let resp = encode_create_topics_response(version, &req); assert!( !resp.is_empty(), "CreateTopics v{version}: response must not be empty" @@ -401,7 +401,7 @@ fn create_topics_response_v2_roundtrip() { let body = load_body(19, "CreateTopics", 2); let req = decode_create_topics_request(2, body).unwrap(); let topic_name = req.topics[0].name.clone(); - let resp = encode_create_topics_response(2, req); + let resp = encode_create_topics_response(2, &req); let mut d = Decoder::new(resp); let _throttle = d.read_i32().unwrap(); // v2+ @@ -421,7 +421,7 @@ fn create_topics_response_v5_has_topic_config_error_code() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(19, "CreateTopics", 5); let req = decode_create_topics_request(5, body).unwrap(); - let resp = encode_create_topics_response(5, req); + let resp = encode_create_topics_response(5, &req); let mut d = Decoder::new(resp); let _throttle = d.read_i32().unwrap(); // v2+ diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index 121074a191..5c19dbbd5f 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -17,12 +17,15 @@ use bytes::Bytes; -use iggy_gateway_kafka::protocol::api::{API_KEY_API_VERSIONS, API_KEY_METADATA, handle_request}; +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, handle_request, +}; use iggy_gateway_kafka::protocol::codec::Encoder; #[test] fn golden_apiversions_v1_response_fixture() { - let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new()); + let broker = BrokerAdvertise::default(); + let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker); // error_code=0, api_count=6 // key 0 (Produce) min=3 max=9 @@ -52,7 +55,7 @@ fn golden_metadata_v0_single_topic_response_fixture() { request.write_i32(1); // one topic let req_bytes = request.freeze(); - let actual = handle_request(API_KEY_METADATA, 0, req_bytes); + let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()); // brokers[1]: node_id=1, host=127.0.0.1, port=9093 // topics[1]: topic_error=3, topic_name=unknown-topic, partitions[0] diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs index d4eef81db6..e88efe4c22 100644 --- a/gateways/kafka/tests/header_tests.rs +++ b/gateways/kafka/tests/header_tests.rs @@ -28,7 +28,7 @@ fn request_header_v1_decodes() { enc.write_i16(18); // api_key: ApiVersions enc.write_i16(2); // api_version enc.write_i32(101); - enc.write_nullable_string(Some("kafka-cli")); + enc.write_nullable_string(Some("kafka-cli")).unwrap(); let bytes = enc.freeze(); let header = RequestHeader::decode(bytes, 1).expect("decode should succeed"); @@ -44,7 +44,7 @@ fn request_header_v1_null_client_id() { enc.write_i16(18); enc.write_i16(1); enc.write_i32(5); - enc.write_nullable_string(None); + enc.write_nullable_string(None).unwrap(); let bytes = enc.freeze(); let header = RequestHeader::decode(bytes, 1).unwrap(); diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs index 7a672348ac..35bc4ff113 100644 --- a/gateways/kafka/tests/server_integration_tests.rs +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -41,7 +41,7 @@ async fn read_frame_reads_valid_payload() { enc.write_i16(18); enc.write_i16(3); enc.write_i32(123); - enc.write_nullable_string(Some("test-client")); + enc.write_nullable_string(Some("test-client")).unwrap(); let payload = enc.freeze(); let mut frame = BytesMut::with_capacity(4 + payload.len()); From 763854667e97d823f28a6898901072359b96ff5c Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 7 Jun 2026 23:02:58 -0400 Subject: [PATCH 03/15] fix(kafka-gw): address all code review findings Critical protocol correctness: - Metadata non-flexible: controller_id now written before topics (v1+); add rack (v1+), cluster_id (v2+), is_internal (v1+) to match spec - Metadata flexible (v9): is_internal now written before partitions array - ListOffsets: replace hardcoded 1_700_000_000_000 stub timestamp with -1 (Kafka "not available" sentinel) - read_compact_array_count: treat varint=0 as Ok(0) (null compact array) instead of Err; fixes Fetch v7+ with null forgotten_topics from librdkafka/kafka-go Performance: - read_frame: replace vec![0u8; frame_len] with BytesMut + read_buf loop (no zero-initialization) - handle_connection: single BytesMut alloc for length+header+body via new send_response() helper and ResponseHeader::encode_into() - BrokerAdvertise: wrap in Arc, clone Arc per connection instead of String-copying the struct Codec hardening: - read_nullable_string / read_compact_nullable_string: single alloc via str::from_utf8 on borrowed slice + to_owned() - read_tagged_fields: remove dead usize::try_from (always succeeds on 64-bit); compare directly as u64 - write_nullable_bytes: add debug_assert for i32::MAX overflow API cleanliness: - Remove 11 out-of-scope API_KEY_* constants (dead code, not referenced) - requests.rs: null topic name now returns Err(NullTopicName) instead of silently mapping to "" - error.rs: add NullTopicName variant - header.rs: add encode_into() and encoded_size() for zero-copy framing Tests: update golden Metadata v0 fixture and api_handler test to match corrected field order (no controller_id in v0). Co-Authored-By: Claude Sonnet 4.6 --- gateways/kafka/src/error.rs | 2 + gateways/kafka/src/protocol/api.rs | 38 ++++----- gateways/kafka/src/protocol/codec.rs | 36 +++++---- gateways/kafka/src/protocol/header.rs | 16 +++- gateways/kafka/src/protocol/requests.rs | 26 +++--- gateways/kafka/src/protocol/responses.rs | 2 +- gateways/kafka/src/server.rs | 81 ++++++++++++++----- gateways/kafka/tests/api_handler_tests.rs | 9 +-- .../kafka/tests/golden_wire_fixtures_tests.rs | 5 +- 9 files changed, 143 insertions(+), 72 deletions(-) diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs index 7eee4b16b8..b2ee237387 100644 --- a/gateways/kafka/src/error.rs +++ b/gateways/kafka/src/error.rs @@ -42,6 +42,8 @@ pub enum KafkaProtocolError { CollectionTooLarge { count: usize, max: usize }, #[error("string length {length} exceeds i16::MAX")] StringTooLong { length: usize }, + #[error("null topic name in request")] + NullTopicName, #[error("io error: {0}")] Io(#[from] std::io::Error), } diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 275c542e8a..3e39698841 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -33,19 +33,8 @@ pub const API_KEY_PRODUCE: i16 = 0; pub const API_KEY_FETCH: i16 = 1; pub const API_KEY_LIST_OFFSETS: i16 = 2; pub const API_KEY_METADATA: i16 = 3; -pub const API_KEY_OFFSET_COMMIT: i16 = 8; -pub const API_KEY_OFFSET_FETCH: i16 = 9; -pub const API_KEY_FIND_COORDINATOR: i16 = 10; -pub const API_KEY_JOIN_GROUP: i16 = 11; -pub const API_KEY_HEARTBEAT: i16 = 12; -pub const API_KEY_LEAVE_GROUP: i16 = 13; -pub const API_KEY_SYNC_GROUP: i16 = 14; -pub const API_KEY_DESCRIBE_GROUPS: i16 = 15; -pub const API_KEY_LIST_GROUPS: i16 = 16; -pub const API_KEY_SASL_HANDSHAKE: i16 = 17; pub const API_KEY_API_VERSIONS: i16 = 18; pub const API_KEY_CREATE_TOPICS: i16 = 19; -pub const API_KEY_DELETE_TOPICS: i16 = 20; pub const ERROR_NONE: i16 = 0; pub const ERROR_OFFSET_OUT_OF_RANGE: i16 = 1; @@ -296,27 +285,38 @@ fn encode_metadata_response( for _ in 0..topics_count { e.write_i16(topic_error); e.write_compact_nullable_string(Some("unknown-topic")); - e.write_varint(1); // empty partitions array - if api_version >= 4 { - e.write_bool(false); // is_internal + if api_version >= 1 { + e.write_bool(false); // is_internal — must come before partitions array } + e.write_varint(1); // empty partitions array e.write_empty_tagged_fields(); } e.write_empty_tagged_fields(); } else { - e.write_i32(1); - e.write_i32(1); + e.write_i32(1); // brokers array length + e.write_i32(1); // node_id let _ = e.write_nullable_string(Some(&broker.host)); e.write_i32(broker.port); + if api_version >= 1 { + let _ = e.write_nullable_string(None); // rack + } + + if api_version >= 2 { + let _ = e.write_nullable_string(None); // cluster_id + } + if api_version >= 1 { + e.write_i32(1); // controller_id — must come before topics array + } e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); for _ in 0..topics_count { e.write_i16(topic_error); let _ = e.write_nullable_string(Some("unknown-topic")); - e.write_i32(0); + if api_version >= 1 { + e.write_bool(false); // is_internal + } + e.write_i32(0); // partitions array (empty) } - - e.write_i32(1); // controller_id } e.freeze() diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 0429beafc1..e1c9ce68ba 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -110,10 +110,12 @@ impl Decoder { } /// Compact array length: unsigned varint holding `element_count + 1`. + /// Per the Kafka spec, varint=0 encodes a null (absent) array; treat as empty (0 elements) + /// so optional fields like `forgotten_topics` are skipped rather than rejected. pub fn read_compact_array_count(&mut self) -> Result { let n = self.read_varint()?; if n == 0 { - return Err(KafkaProtocolError::InvalidCompactArrayLength(0)); + return Ok(0); } let count = (n - 1) as usize; if count > MAX_COLLECTION_LEN { @@ -133,10 +135,11 @@ impl Decoder { } let len = len as usize; self.ensure(len)?; - let chunk = self.bytes.copy_to_bytes(len); - String::from_utf8(chunk.to_vec()) - .map(Some) - .map_err(|_| KafkaProtocolError::InvalidUtf8) + let s = std::str::from_utf8(&self.bytes.chunk()[..len]) + .map_err(|_| KafkaProtocolError::InvalidUtf8)? + .to_owned(); + self.bytes.advance(len); + Ok(Some(s)) } /// Compact nullable string (flexible versions): varint(len+1) prefix, 0 = null. @@ -147,10 +150,11 @@ impl Decoder { } let len = (len_plus_one - 1) as usize; self.ensure(len)?; - let chunk = self.bytes.copy_to_bytes(len); - String::from_utf8(chunk.to_vec()) - .map(Some) - .map_err(|_| KafkaProtocolError::InvalidUtf8) + let s = std::str::from_utf8(&self.bytes.chunk()[..len]) + .map_err(|_| KafkaProtocolError::InvalidUtf8)? + .to_owned(); + self.bytes.advance(len); + Ok(Some(s)) } /// Legacy nullable bytes: i32 length prefix (-1 = null). @@ -184,16 +188,13 @@ impl Decoder { /// A count of 0 is the common case (single byte 0x00). pub fn read_tagged_fields(&mut self) -> Result<()> { let count = self.read_varint()?; - let count = usize::try_from(count).map_err(|_| KafkaProtocolError::CollectionTooLarge { - count: count as usize, - max: MAX_COLLECTION_LEN, - })?; - if count > MAX_COLLECTION_LEN { + if count > MAX_COLLECTION_LEN as u64 { return Err(KafkaProtocolError::CollectionTooLarge { - count, + count: count as usize, max: MAX_COLLECTION_LEN, }); } + let count = count as usize; for _ in 0..count { self.read_varint()?; // tag number let size = self.read_varint()? as usize; @@ -291,6 +292,11 @@ impl Encoder { match v { None => self.write_i32(-1), Some(b) => { + debug_assert!( + b.len() <= i32::MAX as usize, + "byte slice length {} exceeds i32::MAX", + b.len() + ); self.write_i32(b.len() as i32); self.bytes.put_slice(b); } diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index 93f90e5c5e..66ce358fe7 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -21,7 +21,7 @@ clippy::match_same_arms )] -use bytes::Bytes; +use bytes::{BufMut, Bytes}; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::{Decoder, Encoder}; @@ -194,4 +194,18 @@ impl ResponseHeader { } e.freeze() } + + /// Write this header directly into an existing buffer (avoids a separate heap alloc). + pub fn encode_into(&self, buf: &mut bytes::BytesMut, header_version: i16) { + buf.put_i32(self.correlation_id); + if header_version >= 1 { + buf.put_u8(0); // empty tagged fields + } + } + + /// Byte size of the encoded header for a given version. + #[must_use] + pub fn encoded_size(header_version: i16) -> usize { + if header_version >= 1 { 5 } else { 4 } + } } diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index 6f4daa8996..d2bc6b74be 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -19,7 +19,7 @@ #![allow(clippy::pedantic)] -use crate::error::Result; +use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::Decoder; use bytes::Bytes; @@ -72,9 +72,11 @@ pub fn decode_produce_request(version: i16, body: Bytes) -> Result Result { let mut topics = Vec::with_capacity(topics_count); for _ in 0..topics_count { let topic = if flexible { - d.read_compact_nullable_string()?.unwrap_or_default() + d.read_compact_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? } else { - d.read_nullable_string()?.unwrap_or_default() + d.read_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? }; let partitions_count = if flexible { @@ -301,9 +305,11 @@ pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result Result B let offset = 0i64; if version >= 1 { - e.write_i64(1_700_000_000_000); + e.write_i64(-1); // -1 = timestamp not available (Kafka sentinel) } e.write_i64(offset); if version >= 4 { diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 55b3e509bc..02e2bfaf6f 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -35,6 +35,7 @@ use crate::protocol::codec::Decoder; use crate::protocol::header::{ RequestHeader, ResponseHeader, request_header_version, response_header_version, }; +use std::io; #[derive(Debug, Clone)] pub struct ServerConfig { @@ -77,7 +78,7 @@ impl KafkaServer { info!("kafka listener bound on {}", self.config.bind_addr); let tracker = TaskTracker::new(); - let broker = BrokerAdvertise::from_bind_addr(&self.config.bind_addr); + let broker = Arc::new(BrokerAdvertise::from_bind_addr(&self.config.bind_addr)); loop { tokio::select! { @@ -91,7 +92,7 @@ impl KafkaServer { match accept_result { Ok((stream, peer)) => { let cfg = Arc::clone(&self.config); - let broker = broker.clone(); + let broker = Arc::clone(&broker); tracker.spawn(async move { if let Err(err) = handle_connection(stream, cfg, peer, broker).await { warn!(%peer, "connection closed with error: {err}"); @@ -127,7 +128,7 @@ async fn handle_connection( mut stream: TcpStream, config: Arc, peer: SocketAddr, - broker: BrokerAdvertise, + broker: Arc, ) -> Result<()> { info!(%peer, "connection accepted"); @@ -164,12 +165,14 @@ async fn handle_connection( warn!(%peer, api_key, api_version, "unsupported request header version"); let body_response = encode_error_only_response(ERROR_INVALID_REQUEST); let resp_header = ResponseHeader { correlation_id }; - let encoded_header = resp_header.encode(0); - let mut payload = - BytesMut::with_capacity(encoded_header.len() + body_response.len()); - payload.put_slice(&encoded_header); - payload.put_slice(&body_response); - write_frame(&mut stream, &payload, config.write_timeout).await?; + send_response( + &mut stream, + &resp_header, + 0, + &body_response, + config.write_timeout, + ) + .await?; return Ok(()); } Err(e) => return Err(e), @@ -190,13 +193,42 @@ async fn handle_connection( let resp_header = ResponseHeader { correlation_id: req.correlation_id, }; - let encoded_header = resp_header.encode(resp_hdr_ver); - let mut payload = BytesMut::with_capacity(encoded_header.len() + body_response.len()); - payload.put_slice(&encoded_header); - payload.put_slice(&body_response); + send_response( + &mut stream, + &resp_header, + resp_hdr_ver, + &body_response, + config.write_timeout, + ) + .await?; + } +} - write_frame(&mut stream, &payload, config.write_timeout).await?; +/// Write a single length-prefixed Kafka frame using one allocation. +/// Avoids the separate header-encode + payload-concat + length-prefix allocations. +async fn send_response( + stream: &mut TcpStream, + header: &ResponseHeader, + header_version: i16, + body: &[u8], + write_timeout: Duration, +) -> Result<()> { + let header_size = ResponseHeader::encoded_size(header_version); + let payload_size = header_size + body.len(); + if payload_size > i32::MAX as usize { + return Err(KafkaProtocolError::FrameTooLarge { + max_bytes: i32::MAX as usize, + actual_bytes: payload_size, + }); } + let mut frame = BytesMut::with_capacity(4 + payload_size); + frame.put_i32(payload_size as i32); + header.encode_into(&mut frame, header_version); + frame.put_slice(body); + timeout(write_timeout, stream.write_all(&frame)) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "write timeout"))??; + Ok(()) } fn correlation_id_from_frame(frame: &bytes::Bytes) -> i32 { @@ -239,11 +271,22 @@ pub async fn read_frame( }); } - let mut data = vec![0u8; frame_len]; - timeout(read_timeout, stream.read_exact(&mut data)) - .await - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; - Ok(bytes::Bytes::from(data)) + // read_buf fills BytesMut spare capacity without zero-initializing it first. + let mut data = BytesMut::with_capacity(frame_len); + timeout(read_timeout, async { + while data.len() < frame_len { + if stream.read_buf(&mut data).await? == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "connection closed", + )); + } + } + Ok::<_, io::Error>(()) + }) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "read timeout"))??; + Ok(data.freeze()) } /// Write one length-prefixed Kafka frame to `stream`. diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 29d5a5423f..aca2193708 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -107,10 +107,11 @@ fn unsupported_version_returns_protocol_error() { req.extend_from_slice(&1_i32.to_be_bytes()); let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req), &test_broker()); let mut d = Decoder::new(body); + // Metadata v0: brokers[], topics[] — no controller_id (added in v1) let _broker_count = d.read_i32().unwrap(); - let _ = d.read_i32().unwrap(); - let _ = d.read_nullable_string().unwrap(); - let _ = d.read_i32().unwrap(); + let _ = d.read_i32().unwrap(); // node_id + let _ = d.read_nullable_string().unwrap(); // host + let _ = d.read_i32().unwrap(); // port let topic_count = d.read_i32().unwrap(); assert_eq!(topic_count, 1); let topic_error = d.read_i16().unwrap(); @@ -119,8 +120,6 @@ fn unsupported_version_returns_protocol_error() { assert_eq!(topic_name, "unknown-topic"); let partitions_count = d.read_i32().unwrap(); assert_eq!(partitions_count, 0); - let controller_id = d.read_i32().unwrap(); - assert_eq!(controller_id, 1); } // ── Misc ──────────────────────────────────────────────────────────────────── diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index 5c19dbbd5f..62b9e96e60 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -57,10 +57,10 @@ fn golden_metadata_v0_single_topic_response_fixture() { let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()); + // Metadata v0 layout: brokers[], topics[] (no controller_id — added in v1) // brokers[1]: node_id=1, host=127.0.0.1, port=9093 // topics[1]: topic_error=3, topic_name=unknown-topic, partitions[0] - // controller_id=1 (included by this implementation baseline) - let expected: [u8; 52] = [ + let expected: [u8; 48] = [ 0x00, 0x00, 0x00, 0x01, // broker count 0x00, 0x00, 0x00, 0x01, // node id 0x00, 0x09, // host len @@ -72,7 +72,6 @@ fn golden_metadata_v0_single_topic_response_fixture() { 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x2d, 0x74, 0x6f, 0x70, 0x69, 0x63, // unknown-topic 0x00, 0x00, 0x00, 0x00, // partition count - 0x00, 0x00, 0x00, 0x01, // controller id ]; assert_eq!(actual.as_ref(), &expected); } From 0c1e756d0c39cd04ac31ed1a2a27dc33558044f9 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 10 Jun 2026 18:59:50 -0400 Subject: [PATCH 04/15] kafka gateway: docs, tests, and protocol updates Foundation for the Kafka gateway (#3421): add manual testing and test-suite docs, a full API key reference, and update README/SCOPE to reflect bind/config notes and test coverage. Implement various protocol, header, codec and server tweaks (src/protocol/*, server.rs, lib.rs, main.rs, error.rs) and add/extend many tests and test helpers (103 regression tests across new and updated suites). Add kafka-tool response helper and fixtures tooling updates to support decode/validation tests and manual end-to-end checks. These changes bundle documentation, test infra, and protocol-level fixes required to validate wire decoding, version firewalling, and stub responses for the gateway. --- gateways/kafka/README.md | 10 +- gateways/kafka/docs/MANUAL_TESTING.md | 272 ++++++++++++ gateways/kafka/docs/SCOPE.md | 89 +++- gateways/kafka/docs/TEST_SUITE.md | 157 +++++++ .../kafka/docs/kafka_api_keys_reference.md | 304 +++++++++++++ gateways/kafka/src/error.rs | 4 +- gateways/kafka/src/lib.rs | 2 +- gateways/kafka/src/main.rs | 15 +- gateways/kafka/src/protocol/api.rs | 68 ++- gateways/kafka/src/protocol/codec.rs | 43 +- gateways/kafka/src/protocol/header.rs | 4 + gateways/kafka/src/protocol/requests.rs | 2 +- gateways/kafka/src/protocol/responses.rs | 146 ++++-- gateways/kafka/src/server.rs | 165 ++++--- gateways/kafka/tests/api_handler_tests.rs | 22 +- .../kafka/tests/broker_advertise_tests.rs | 112 +++++ gateways/kafka/tests/codec_tests.rs | 16 +- gateways/kafka/tests/common/fixtures.rs | 54 +++ gateways/kafka/tests/common/scope.rs | 35 ++ gateways/kafka/tests/common/server.rs | 52 +++ gateways/kafka/tests/common/tcp.rs | 110 +++++ gateways/kafka/tests/decode_safety_tests.rs | 81 ++++ .../kafka/tests/decode_validation_tests.rs | 46 +- .../kafka/tests/golden_wire_fixtures_tests.rs | 4 +- .../kafka/tests/handler_regression_tests.rs | 171 +++++++ gateways/kafka/tests/header_tests.rs | 11 + .../kafka/tests/metadata_regression_tests.rs | 228 ++++++++++ gateways/kafka/tests/server_e2e_tests.rs | 156 +++++++ .../kafka/tests/server_integration_tests.rs | 33 +- .../kafka/tests/version_firewall_tests.rs | 309 +++++++++++++ gateways/kafka/tools/kafka-tool/src/main.rs | 214 +++++++-- .../kafka/tools/kafka-tool/src/response.rs | 420 ++++++++++++++++++ 32 files changed, 3117 insertions(+), 238 deletions(-) create mode 100644 gateways/kafka/docs/MANUAL_TESTING.md create mode 100644 gateways/kafka/docs/TEST_SUITE.md create mode 100644 gateways/kafka/docs/kafka_api_keys_reference.md create mode 100644 gateways/kafka/tests/broker_advertise_tests.rs create mode 100644 gateways/kafka/tests/common/fixtures.rs create mode 100644 gateways/kafka/tests/common/scope.rs create mode 100644 gateways/kafka/tests/common/server.rs create mode 100644 gateways/kafka/tests/common/tcp.rs create mode 100644 gateways/kafka/tests/decode_safety_tests.rs create mode 100644 gateways/kafka/tests/handler_regression_tests.rs create mode 100644 gateways/kafka/tests/metadata_regression_tests.rs create mode 100644 gateways/kafka/tests/server_e2e_tests.rs create mode 100644 gateways/kafka/tests/version_firewall_tests.rs create mode 100644 gateways/kafka/tools/kafka-tool/src/response.rs diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index b575a0c803..09495b3c3b 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -8,7 +8,7 @@ Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/34 cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway ``` -Default bind: `127.0.0.1:9093`. +Default bind: `127.0.0.1:9093`. Override with `KAFKA_BIND_ADDR` (e.g. `0.0.0.0:9093`). ## Test @@ -16,6 +16,8 @@ Default bind: `127.0.0.1:9093`. cargo test -p iggy_gateway_kafka ``` +103 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. + `decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/`: ```bash @@ -26,9 +28,13 @@ cargo run -p kafka-message-gen -- generate \ (Run from workspace root; adjust paths if needed.) +## Manual testing + +Before check-in, run the procedure in [docs/MANUAL_TESTING.md](docs/MANUAL_TESTING.md) (smoke, version firewall, kcat, adversarial cases). + ## Scoped APIs -See [docs/SCOPE.md](docs/SCOPE.md). +See [docs/SCOPE.md](docs/SCOPE.md) for [#3421](https://github.com/apache/iggy/issues/3421) deliverables, supported API key/version table, and post-foundation TODO backlog. ## Wire fixture tool diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md new file mode 100644 index 0000000000..cd3e8c0e58 --- /dev/null +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -0,0 +1,272 @@ +# Kafka gateway — manual testing procedure + +Manual validation for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421) foundation: TCP listener, wire decode, version firewall, stub responses. **No Iggy backend** — success means correct Kafka wire behavior, not message persistence. + +See also: [SCOPE.md](SCOPE.md) (supported API keys), [TEST_SUITE.md](TEST_SUITE.md) (automated coverage). + +--- + +## 1. Environment setup + +### Requirements + +| Tool | Purpose | Install | +|------|---------|---------| +| Rust toolchain | Build gateway + kafka-tool | [rustup.rs](https://rustup.rs) | +| `kafka-message-gen` | Generate/send wire fixtures | `cargo build -p kafka-message-gen` | +| `kcat` (optional) | Real Kafka client smoke test | `brew install kcat` / `apt install kafkacat` | +| `nc` / `netcat` (optional) | Raw byte injection | Usually preinstalled | +| `xxd` or `hexdump` (optional) | Inspect binary responses | Usually preinstalled | + +### Build and start gateway + +```bash +# From iggy workspace root +cargo build -p iggy_gateway_kafka --bin iggy-kafka-gateway + +# Terminal 1 — start listener (default 127.0.0.1:9093) +RUST_LOG=info cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway +``` + +Expected log: + +``` +kafka listener bound on 127.0.0.1:9093 +``` + +### Generate wire fixtures + +```bash +# Terminal 2 +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 3 --api-key 18 --api-key 19 +``` + +--- + +## 2. Pre-flight automated check + +Run before manual testing to catch regressions: + +```bash +cargo test -p iggy_gateway_kafka +``` + +All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (step above). + +--- + +## 3. Manual test cases + +### Category A — Smoke tests (must pass before check-in) + +| ID | Test | Steps | Expected result | Pass criteria | +|----|------|-------|-----------------|---------------| +| A1 | Gateway starts | Run `iggy-kafka-gateway` | Binds to `:9093`, no panic | Log shows bind address | +| A2 | ApiVersions v1 | `cargo run -p kafka-message-gen -- send --host 127.0.0.1:9093 --api-key 18 --version 1` | Response received | `ec=0`, non-zero byte count | +| A3 | ApiVersions v3 (flexible) | Same with `--version 3` | Response received | `ec=0` | +| A4 | Metadata v0 | `send --api-key 3 --version 0` | Stub broker in response | `ec=0` or topic error 3 (stub) | +| A5 | Produce v3 | `send --api-key 0 --version 3` | Decode + stub ack | `ec=0` | +| A6 | Fetch v4 | `send --api-key 1 --version 4` | Decode + stub response | `ec=0` | +| A7 | ListOffsets v1 | `send --api-key 2 --version 1` | Decode + stub offsets | `ec=0` | +| A8 | CreateTopics v2 | `send --api-key 19 --version 2` | Decode + stub ack | `ec=0` | +| A9 | Verify all scoped keys | `cargo run -p kafka-message-gen -- verify --host 127.0.0.1:9093 --api-key 0 --api-key 1 --api-key 2 --api-key 3 --api-key 18 --api-key 19` | Exit code 0 | No timeouts or I/O errors | + +### Category B — Version firewall (boundary validation) + +For each API key, test **min−1**, **min**, **max**, **max+1** using `kafka-message-gen send` with `--version N`. + +| API key | Name | Min | Max | Test versions | +|---------|------|-----|-----|---------------| +| 18 | ApiVersions | 0 | 3 | −1, 0, 3, 4 | +| 3 | Metadata | 0 | 9 | −1, 0, 9, 10 | +| 0 | Produce | 3 | 9 | 2, 3, 9, 10 | +| 1 | Fetch | 4 | 12 | 3, 4, 12, 13 | +| 2 | ListOffsets | 1 | 6 | 0, 1, 6, 7 | +| 19 | CreateTopics | 2 | 5 | 1, 2, 5, 6 | + +| ID | Test | Expected for in-range | Expected for out-of-range | +|----|------|----------------------|---------------------------| +| B1 | ApiVersions negotiation | `error_code=0`; body lists 6 API keys with correct min/max | `error_code=35` (UNSUPPORTED_VERSION) | +| B2 | Metadata out-of-range | N/A | Topic entries show `error_code=35` | +| B3 | Produce/Fetch/ListOffsets/CreateTopics out-of-range | N/A | Version-aware response with `error_code=35` (top-level or per-topic/partition) | +| B4 | ApiVersions lists only scoped keys | Decode response | Contains keys 0,1,2,3,18,19 only — no consumer-group keys | + +**Validation tip:** Use `--hex` when generating to inspect request bytes: + +```bash +cargo run -p kafka-message-gen -- generate --api-key 18 --version 3 --hex +``` + +### Category C — Unsupported API keys + +| ID | API key | Name | Steps | Expected | +|----|---------|------|-------|----------| +| C1 | 8 | OffsetCommit | `send --api-key 8 --version 2` | `ec=35`, connection stays open | +| C2 | 10 | FindCoordinator | `send --api-key 10` | `ec=35` | +| C3 | 17 | SaslHandshake | `send --api-key 17` | `ec=35` | +| C4 | 20 | DeleteTopics | `send --api-key 20` | `ec=35` | + +Follow C1 with A2 on the **same** `nc` session to confirm the connection is not dropped. + +### Category D — Flexible vs legacy wire encoding + +| ID | API key | Version | Encoding | Validation | +|----|---------|---------|----------|------------| +| D1 | Produce | 8 | Legacy (i32 arrays) | `send` succeeds, `ec=0` | +| D2 | Produce | 9 | Flexible (compact + tagged fields) | `send` succeeds, `ec=0` | +| D3 | Fetch | 11 | Legacy | `send` succeeds | +| D4 | Fetch | 12 | Flexible | `send` succeeds | +| D5 | Metadata | 8 | Legacy | `send` succeeds | +| D6 | Metadata | 9 | Flexible | `send` succeeds | +| D7 | ListOffsets | 5 | Legacy | `send` succeeds | +| D8 | ListOffsets | 6 | Flexible | `send` succeeds | +| D9 | CreateTopics | 4 | Legacy | `send` succeeds | +| D10 | CreateTopics | 5 | Flexible | `send` succeeds | + +### Category E — Metadata stub semantics + +| ID | Test | Steps | Expected | +|----|------|-------|----------| +| E1 | Broker advertise address | Start gateway on `127.0.0.1:9093`; Metadata v0 | Broker host=`127.0.0.1`, port=`9093` | +| E2 | Wildcard bind + advertised host | `KAFKA_BIND_ADDR=0.0.0.0:19093` + `KAFKA_ADVERTISED_HOST=kafka.internal`, restart | Metadata broker host/port match advertised values | +| E3 | Unknown topic stub | Metadata with topic name `my-topic` | Topic error `3` (UNKNOWN_TOPIC_OR_PARTITION), name `unknown-topic` | +| E4 | Multiple topics | Metadata request listing 3 topics | 3 topic entries, each with error 3 | + +### Category F — TCP / connection behavior + +| ID | Test | Steps | Expected | +|----|------|-------|----------| +| F1 | Correlation ID echoed | Send ApiVersions with known correlation_id; decode response header | Response correlation_id matches request | +| F2 | Sequential requests | Send ApiVersions then Metadata on same TCP connection | Both get valid responses | +| F3 | Client disconnect | Connect, send partial frame, close | Gateway logs clean disconnect, no panic | +| F4 | Invalid frame length 0 | `printf '\x00\x00\x00\x00' \| nc 127.0.0.1 9093` | Connection closed, gateway continues serving others | +| F5 | Oversized frame | Send 4-byte length > 8 MiB | Connection rejected/closed, no OOM | +| F6 | Graceful shutdown | Ctrl+C on gateway | Log "shutdown requested", in-flight requests drain | + +### Category G — Real Kafka client (kcat) + +Requires `kcat` installed. Gateway does **not** implement SASL or full broker semantics — expect limited success. + +| ID | Test | Command | Expected (foundation) | +|----|------|---------|---------------------| +| G1 | Broker metadata | `kcat -b 127.0.0.1:9093 -L` | ApiVersions + Metadata handshake; broker appears in metadata | +| G2 | Produce (likely fails later) | `echo "hello" \| kcat -b 127.0.0.1:9093 -t test -P` | May fail at coordinator/group stage — document actual error | +| G3 | Consumer (likely fails later) | `kcat -b 127.0.0.1:9093 -t test -C -o beginning` | May fail without consumer groups — document actual error | + +Record kcat version and exact error strings in your test log. G1 passing is the minimum bar for client compatibility smoke. + +### Category H — Adversarial / negative input + +| ID | Test | Steps | Expected | +|----|------|-------|----------| +| H1 | Truncated Produce body | Send valid header + incomplete body | `error_code=42` (INVALID_REQUEST) or connection error; **no panic** | +| H2 | Random bytes | `dd if=/dev/urandom bs=64 count=1 \| nc 127.0.0.1 9093` | Connection closed or protocol error; gateway stays up | +| H3 | Empty body after header | ApiVersions with valid header, empty body | `ec=0` (ApiVersions accepts empty body) | + +--- + +## 4. Validation reference + +### Kafka error codes used in #3421 + +| Code | Name | When returned | +|------|------|---------------| +| 0 | NONE | Successful stub response | +| 42 | INVALID_REQUEST | Produce/Fetch/ListOffsets/CreateTopics decode failure; unsupported request header | +| 3 | UNKNOWN_TOPIC_OR_PARTITION | Metadata stub per-topic error | +| 35 | UNSUPPORTED_VERSION | Out-of-range version or unlisted API key | +| 42 | INVALID_REQUEST | Unsupported request header version | + +### Response header rules + +| API key | Request flexible? | Response header version | +|---------|--------------------|-------------------------| +| 18 ApiVersions | v3+ | Always v0 (correlation_id only) | +| 3 Metadata | v9+ | v1 (correlation_id + tagged fields) | +| 0 Produce | v9+ | v1 | +| 1 Fetch | v12+ | v1 | +| Others | Per SCOPE.md | See `header.rs` lookup table | + +### Frame layout (for manual hex inspection) + +``` +Request frame: + [length: i32 BE] + [api_key: i16][api_version: i16][correlation_id: i32] + [client_id: NULLABLE_STRING or COMPACT_NULLABLE_STRING] + [tagged_fields: 0x00] ← flexible requests only + [request body] + +Response frame: + [length: i32 BE] + [correlation_id: i32] + [tagged_fields: 0x00] ← flexible responses only (not ApiVersions) + [response body] +``` + +### Raw netcat smoke test + +```bash +# ApiVersions v3 — after generating fixtures +cat gateways/kafka/tools/kafka-tool/kafka_messages/018_ApiVersions_v3.bin \ + | nc -w 2 127.0.0.1 9093 | xxd | head -20 +``` + +First bytes after length prefix should include your correlation_id from the fixture. + +--- + +## 5. Manual test execution checklist + +Copy this checklist into your PR or test log: + +``` +Date: ___________ +Tester: ___________ +Gateway commit: ___________ +kcat version (if used): ___________ + +[ ] A1–A9 Smoke tests +[ ] B1–B4 Version firewall (all 6 keys × 4 boundary versions) +[ ] C1–C4 Unsupported API keys +[ ] D1–D10 Flexible vs legacy encoding +[ ] E1–E4 Metadata stub semantics +[ ] F1–F6 TCP / connection behavior +[ ] G1–G3 kcat client (record errors for G2/G3) +[ ] H1–H3 Adversarial input + +Automated regression: +[ ] cargo test -p iggy_gateway_kafka — ___/103 passed +[ ] cargo clippy -p iggy_gateway_kafka — clean / warnings noted + +Notes / failures: +_________________________________ +``` + +--- + +## 6. Troubleshooting + +| Symptom | Likely cause | Fix | +|---------|--------------|-----| +| `Connection refused` on 9093 | Gateway not running | Start `iggy-kafka-gateway` | +| `decode_validation_tests` panic | Missing fixtures | Run `kafka-message-gen generate` | +| `ec=35` for in-range version | Version not in `SUPPORTED_RANGES` | Check `SCOPE.md` and `api.rs` | +| kcat hangs | Timeout waiting for data | Set `-m 1000`; check gateway logs | +| Buffer underflow on Metadata v9+ | Flexible decode mismatch | File issue; check `api.rs` metadata encoder | +| Port already in use | Another process on 9093 | `lsof -i :9093` / change bind port | + +--- + +## 7. What manual testing does NOT cover (deferred) + +These are documented as TODO in [SCOPE.md](SCOPE.md) — do not fail #3421 validation for these: + +- Message persistence to Iggy +- Consumer group join/sync/heartbeat +- SASL authentication +- Accurate partition leadership / ISR +- Transactional produce +- Real offset commit semantics diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index 8d2be24623..3f20fd411c 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -1,15 +1,37 @@ -# Kafka API scope — issue #3421 foundation +# Kafka gateway scope — [apache/iggy#3421](https://github.com/apache/iggy/issues/3421) -This gateway iteration implements **wire validation and stub responses only** (no Iggy backend, no real broker semantics). +## Issue #3421 — in scope (this iteration) -Source of truth in code: `SUPPORTED_RANGES` in [`src/protocol/api.rs`](../src/protocol/api.rs). +Foundation layer only: a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, validates request wire formats, and returns stub responses. **No Iggy backend integration.** + +| Deliverable | Status | Location | +|-------------|--------|----------| +| TCP listener on `127.0.0.1:9093` (configurable) | Done | `src/server.rs`, `src/main.rs` | +| Length-prefixed frame read/write with `max_frame_size` cap | Done | `src/server.rs` | +| Request header v1/v2 auto-detection | Done | `src/protocol/header.rs` | +| Version negotiation firewall (`SUPPORTED_RANGES`) | Done | `src/protocol/api.rs` | +| Request decode + stub encode for 6 API keys | Done | `src/protocol/requests.rs`, `responses.rs`, `api.rs` | +| Produce hot path: RecordBatch as opaque `Bytes` | Done | `src/protocol/requests.rs` | +| Graceful errors (`UNSUPPORTED_VERSION`, corrupt decode, invalid header) | Done | `src/protocol/api.rs`, `src/server.rs` | +| Adversarial decode safety tests | Done | `tests/decode_safety_tests.rs` | +| Regression test suite (103 tests) | Done | `tests/` — catalog in [`TEST_SUITE.md`](TEST_SUITE.md) | +| Manual testing procedure | Done | [`MANUAL_TESTING.md`](MANUAL_TESTING.md) | +| Wire fixture tool for manual/integration testing | Done | `tools/kafka-tool/` | + +Source of truth for supported ranges: `SUPPORTED_RANGES` in [`src/protocol/api.rs`](../src/protocol/api.rs). + +### Governance model + +Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiVersions advertises exactly what the firewall allows; out-of-range requests receive `UNSUPPORTED_VERSION` (35) without dropping the connection. + +--- ## Supported API keys and versions | API key | Name | Min version | Max version | Valid versions | Behavior | |---------|------|-------------|-------------|----------------|----------| | 18 | ApiVersions | 0 | 3 | 0, 1, 2, 3 | Advertise supported ranges; flexible encoding at v3+ | -| 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode request; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | +| 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode topic list count; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | | 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub response | | 1 | Fetch | 4 | 12 | 4, 5, 6, 7, 8, 9, 10, 11, 12 | Decode request; stub response | | 2 | ListOffsets | 1 | 6 | 1, 2, 3, 4, 5, 6 | Decode request; stub response | @@ -17,7 +39,7 @@ Source of truth in code: `SUPPORTED_RANGES` in [`src/protocol/api.rs`](../src/pr A request is accepted when `min_version ≤ api_version ≤ max_version` for that API key. Any other version for a listed key, or any unlisted API key, receives `UNSUPPORTED_VERSION` (35). -## Valid versions reference (by API key) +### Valid versions reference (by API key) Use this table when configuring clients or generating wire fixtures with `kafka-message-gen`. @@ -30,7 +52,9 @@ Use this table when configuring clients or generating wire fixtures with `kafka- | 18 | ApiVersions | 0–3 | v3 | | 19 | CreateTopics | 2–5 | v5 | -## Unsupported API keys +--- + +## Unsupported API keys (foundation) All API keys not listed above receive an error-only response with `UNSUPPORTED_VERSION` (35). Examples not in this foundation scope: @@ -43,8 +67,53 @@ All API keys not listed above receive an error-only response with `UNSUPPORTED_V | 17 | SaslHandshake | Auth — later issue | | 20+ | DeleteTopics, InitProducerId, transactions, ACLs, etc. | Later issues | -## Out of scope (later issues) +Full reference for future phases: [`kafka_api_keys_reference.md`](kafka_api_keys_reference.md). + +--- + +## Architecture (three layers) + +| Layer | #3421 | Description | +|-------|-------|-------------| +| **1 — Wire framing** | In scope | `server.rs`, `codec.rs`, `header.rs` — keep custom, zero-copy frame I/O | +| **2 — Request/response codecs** | Partial | Custom minimal-parse codecs for 6 hot-path keys; stub responses only | +| **3 — Iggy bridge** | Out of scope | Produce/Fetch → Iggy SDK; deferred to a follow-on issue | + +--- + +## TODO — post-#3421 (architecture review backlog) + +Items from the [hybrid architecture review](https://github.com/apache/iggy/discussions/3252) and maintainer feedback. **Not part of #3421.** + +### Phase 2 — Iggy bridge (new issue) + +- [ ] Add `bridge/` module (`iggy_bridge`): Produce → `send_messages`, Fetch → `poll_messages` +- [ ] Document partition mapping in `docs/BRIDGE_MAPPING.md`: + - Iggy partitions are **0-based** (same as Kafka) — direct `partition_id` mapping, no offset conversion + - Iggy **consumer groups exist** — map Kafka group APIs to Iggy consumer group APIs + - Use `Partitioning::balanced()` only when Kafka sends `partition == -1`; otherwise use request partition ID +- [ ] Idempotent `ensure_stream_and_topic()` (create-if-not-exists) +- [ ] Real Metadata topology (brokers, partitions, leaders) backed by Iggy state + +### Phase 2 — Selective `kafka-protocol` crate (feature-gated) + +- [ ] Add optional `kafka-protocol-cold` feature to `iggy_gateway_kafka` — **not** wholesale replace of `requests.rs`/`responses.rs` +- [ ] Use crate for: RecordBatch decode (compression, CRC), consumer-group API keys (8–14, 10), complex Metadata/FindCoordinator responses +- [ ] **Keep custom codecs** for Produce/Fetch hot paths (opaque RecordBatch bytes) + +### Phase 3 — Consumer groups (~7 API keys) + +- [ ] OffsetCommit (8), OffsetFetch (9), FindCoordinator (10) +- [ ] JoinGroup (11), Heartbeat (12), LeaveGroup (13), SyncGroup (14) +- [ ] DescribeGroups (15), ListGroups (16) as needed by target clients + +### Phase 3+ — Auth, admin, tuning + +- [ ] SASL (17, 36) if required by deployment +- [ ] Tune `max_frame_size` per workload (Kafka defaults: ~1 MiB produce, ~50 MiB fetch; current default 8 MiB) +- [ ] Target **~15–20 API keys** total for a functional bridge — not all 74+ admin keys + +### Open questions (ask maintainers before Phase 2) -- `IggyBridge` / produce-fetch against Iggy streams -- Consumer group APIs (8–14), SASL (17, 36), transactions -- Accurate metadata topology and record batch semantics +- [ ] Repo placement: `gateways/kafka/` in [apache/iggy](https://github.com/apache/iggy) vs separate proxy repo (affects workspace deps and CI) +- [ ] Confirm bridge dependency strategy with spetz/hubcio ([Discussion #3081](https://github.com/apache/iggy/discussions/3081), [#3252](https://github.com/apache/iggy/discussions/3252)) diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md new file mode 100644 index 0000000000..c8515387c2 --- /dev/null +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -0,0 +1,157 @@ +# Kafka gateway — automated regression test suite + +Regression tests live under [`tests/`](../tests/). Run from the workspace root: + +```bash +cargo test -p iggy_gateway_kafka +``` + +**Current count:** 103 tests across 12 suites (as of #3421 foundation). + +## Prerequisites + +### Wire fixtures (required for `decode_validation_tests` and some handler tests) + +```bash +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 19 +``` + +Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that need them skip gracefully when a fixture file is missing (`handler_regression_tests`) or panic with a clear path (`decode_validation_tests`). + +--- + +## Test file catalog + +| File | Suite focus | Test count (approx.) | Depends on fixtures | +|------|-------------|----------------------|---------------------| +| [`codec_tests.rs`](../tests/codec_tests.rs) | Primitive encode/decode round-trips, varint, compact strings, tagged fields | 9 | No | +| [`decode_safety_tests.rs`](../tests/decode_safety_tests.rs) | Adversarial wire input — malformed lengths, truncated bodies | 6 | No | +| [`header_tests.rs`](../tests/header_tests.rs) | Request/response header v1/v2, version lookup table | 10 | No | +| [`api_handler_tests.rs`](../tests/api_handler_tests.rs) | ApiVersions, Metadata stub, unsupported key/version | 7 | No | +| [`golden_wire_fixtures_tests.rs`](../tests/golden_wire_fixtures_tests.rs) | Byte-exact golden responses (ApiVersions v1, Metadata v0) | 2 | No | +| [`decode_validation_tests.rs`](../tests/decode_validation_tests.rs) | kafka-tool fixture decode + response structure per version | 14 | **Yes** | +| [`version_firewall_tests.rs`](../tests/version_firewall_tests.rs) | Version boundary matrix, unsupported keys, corrupt bodies | 17 | Partial | +| [`metadata_regression_tests.rs`](../tests/metadata_regression_tests.rs) | Metadata v0–v9, topic counts, broker advertise | 7 | No | +| [`broker_advertise_tests.rs`](../tests/broker_advertise_tests.rs) | `BrokerAdvertise::from_bind_addr` parsing | 5 | No | +| [`handler_regression_tests.rs`](../tests/handler_regression_tests.rs) | Every scoped key×version via `handle_request`, stub error codes | 5 | Partial | +| [`server_integration_tests.rs`](../tests/server_integration_tests.rs) | `read_frame` / `write_frame` unit-level I/O | 4 | No | +| [`server_e2e_tests.rs`](../tests/server_e2e_tests.rs) | Full `KafkaServer` TCP round-trips | 8 | Partial | +| [`common/mod.rs`](../tests/common/mod.rs) | Shared helpers (not a test binary) | — | — | + +--- + +## Coverage matrix by API key + +### ApiVersions (key 18, v0–v3) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Non-flexible response (v1) | `api_handler_tests` | `api_versions_v1_response_non_flexible_format` | +| Flexible response (v3) | `api_handler_tests` | `api_versions_v3_response_flexible_format` | +| Golden byte fixture (v1) | `golden_wire_fixtures_tests` | `golden_apiversions_v1_response_fixture` | +| Exact advertised ranges (v1, v3) | `version_firewall_tests` | `apiversions_advertises_exact_supported_ranges_*` | +| All versions return `error_code=0` | `version_firewall_tests` | `apiversions_all_versions_return_success` | +| Out-of-range version | `version_firewall_tests` | `apiversions_out_of_range_returns_unsupported_in_body` | +| E2E correlation ID preserved | `server_e2e_tests` | `e2e_apiversions_v1_*`, `e2e_apiversions_v3_*` | + +### Metadata (key 3, v0–v9) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Stub broker (default 127.0.0.1:9093) | `api_handler_tests`, `metadata_regression_tests` | `metadata_response_has_broker_*`, `metadata_v0_empty_*` | +| Unsupported version → topic error 35 | `api_handler_tests`, `version_firewall_tests` | `unsupported_version_returns_protocol_error`, `metadata_*_version_returns_topic_error` | +| Golden byte fixture (v0, 1 topic) | `golden_wire_fixtures_tests` | `golden_metadata_v0_single_topic_response_fixture` | +| v1 controller_id, v2 cluster_id | `metadata_regression_tests` | `metadata_v1_*`, `metadata_v2_*` | +| v9 flexible encoding | `metadata_regression_tests` | `metadata_v9_flexible_encoding` | +| Custom broker advertise | `metadata_regression_tests`, `broker_advertise_tests` | `metadata_uses_custom_*`, `metadata_reflects_parsed_*` | +| E2E round-trip | `server_e2e_tests` | `e2e_metadata_v0_returns_stub_broker` | + +### Produce (key 0, v3–v9) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Decode all versions (fixture) | `decode_validation_tests` | `produce_all_supported_versions_decode` | +| Response encode all versions | `decode_validation_tests` | `produce_response_encodes_for_all_supported_versions` | +| v3 field layout | `decode_validation_tests` | `produce_response_v3_roundtrip` | +| v8 record_errors array | `decode_validation_tests` | `produce_response_v8_includes_record_errors` | +| Unsupported v2 → error 35 | `version_firewall_tests` | `produce_unsupported_version_returns_error_only` | +| Corrupt body → error 42 | `version_firewall_tests` | `corrupt_produce_body_returns_invalid_request_error` | +| Stub partition error 0 | `handler_regression_tests` | `produce_stub_response_has_zero_error_per_partition` | +| E2E round-trip | `server_e2e_tests` | `e2e_produce_v3_round_trip_with_fixture` | + +### Fetch (key 1, v4–v12) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Decode all versions | `decode_validation_tests` | `fetch_all_supported_versions_decode` | +| Response encode all versions | `decode_validation_tests` | `fetch_response_encodes_for_all_supported_versions` | +| v7 session_id / error_code layout | `decode_validation_tests` | `fetch_response_v7_roundtrip` | +| Unsupported v3 | `version_firewall_tests` | `fetch_unsupported_version_returns_error_only` | +| Corrupt body → error 42 | `version_firewall_tests` | `corrupt_fetch_body_returns_invalid_request_error` | +| Stub partition error 0 | `handler_regression_tests` | `fetch_stub_response_has_zero_partition_error` | + +### ListOffsets (key 2, v1–v6) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Decode all versions | `decode_validation_tests` | `list_offsets_all_supported_versions_decode` | +| v1 no leader_epoch | `decode_validation_tests` | `list_offsets_response_v1_no_leader_epoch` | +| v4 has leader_epoch | `decode_validation_tests` | `list_offsets_response_v4_has_leader_epoch` | +| Unsupported v0 | `version_firewall_tests` | `list_offsets_unsupported_version_returns_error_only` | +| Stub error 0 | `handler_regression_tests` | `list_offsets_stub_response_has_zero_error` | + +### CreateTopics (key 19, v2–v5) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Decode all versions | `decode_validation_tests` | `create_topics_all_supported_versions_decode` | +| v2 roundtrip | `decode_validation_tests` | `create_topics_response_v2_roundtrip` | +| v5 flexible roundtrip | `decode_validation_tests` | `create_topics_response_v5_roundtrip` | +| Unsupported v1 | `version_firewall_tests` | `create_topics_unsupported_version_returns_error_only` | +| Stub error 0 | `handler_regression_tests` | `create_topics_stub_response_has_zero_error` | + +--- + +## Cross-cutting scenarios + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Version firewall min/max boundaries | `version_firewall_tests` | `is_supported_version_matches_scope_table` | +| Unknown API keys (8, 9, 10, 17, 20, 999) | `version_firewall_tests`, `api_handler_tests` | `unsupported_api_keys_*`, `unknown_api_key_*` | +| Negative i32 array length | `decode_safety_tests` | `negative_i32_array_length_returns_error_not_panic` | +| Oversized collection count | `decode_safety_tests` | `i32_array_length_above_max_returns_collection_too_large` | +| Compact array varint=0 (null array) | `decode_safety_tests` | `compact_array_varint_zero_decodes_as_empty_without_panic` | +| Malformed varint at shift 63 | `decode_safety_tests` | `varint_terminal_byte_with_extra_bits_at_shift_63_is_rejected` | +| Invalid frame length (0) | `server_integration_tests` | `read_frame_rejects_invalid_lengths` | +| Frame exceeds max_frame_size | `server_integration_tests`, `server_e2e_tests` | `read_frame_rejects_invalid_lengths`, `e2e_oversized_frame_is_rejected` | +| Sequential requests on one TCP connection | `server_e2e_tests` | `e2e_sequential_requests_on_one_connection` | +| Connection survives unsupported API key | `server_e2e_tests` | `e2e_unsupported_api_key_returns_error_without_disconnect` | +| Negative frame length closes connection | `server_e2e_tests` | `e2e_negative_frame_length_closes_connection` | + +--- + +## CI recommendation + +```bash +# 1. Generate fixtures +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 19 + +# 2. Run regression suite +cargo test -p iggy_gateway_kafka + +# 3. Optional lint gate +cargo clippy -p iggy_gateway_kafka -- -D warnings +``` + +--- + +## Adding new tests + +1. **New API key or version range** — update `SUPPORTED_RANGES` in `api.rs`, `SCOPE.md`, and add rows to the coverage matrix above. +2. **New decode path** — add fixture via `kafka-message-gen`, extend `decode_validation_tests.rs`. +3. **New error path** — add to `version_firewall_tests.rs` or `decode_safety_tests.rs`. +4. **New TCP behavior** — add to `server_e2e_tests.rs` using helpers in `tests/common/mod.rs`. diff --git a/gateways/kafka/docs/kafka_api_keys_reference.md b/gateways/kafka/docs/kafka_api_keys_reference.md new file mode 100644 index 0000000000..c31cec2d7c --- /dev/null +++ b/gateways/kafka/docs/kafka_api_keys_reference.md @@ -0,0 +1,304 @@ +# Kafka Protocol API Key Reference — Kafka 4.0.0 + +> **Source**: [`ApiKeys.java` @ Kafka 4.0.0](https://github.com/apache/kafka/blob/4.0.0/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java) +> and the canonical [protocol message schemas](https://github.com/apache/kafka/tree/4.0.0/clients/src/main/resources/common/message). +> +> Generated for: **Iggy Kafka Bridge Gateway** — `gateways/kafka/` +> Branch: `feat(gateways)/kafka_to_iggy_listener` + +--- + +## Legend + +| Symbol | Meaning | +|--------|---------| +| 🔴 Bridge | Core data path — must be fully implemented and forwarded to Iggy | +| 🟠 Required Stub | Client state-machine API — must return a well-formed response or clients will stall/crash | +| 🟡 Optional Stub | Admin/observability — can safely return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` | +| ❌ Reject | Internal broker / KRaft only — return `INVALID_REQUEST` with a well-formed frame; **do not close the connection** | + +> **Header.rs ✓** = The API key is already present in `request_header_version()` / `response_header_version()` with the correct flexible-encoding threshold. + +--- + +## KIP-896 Note — Minimum Version Changes in Kafka 4.0 + +Kafka 4.0 removed all protocol versions older than Kafka 2.1.0 (KIP-896). +Key new minimums: + +| API | Old Min | New Min (4.0) | +|-----|:-------:|:-------------:| +| Produce | 0 | 3 | +| Fetch | 0 | 4 | +| ListOffsets | 0 | 1 | +| OffsetCommit | 0 | 2 | +| OffsetFetch | 0 | 1 | +| CreateTopics | 0 | 2 | +| OffsetForLeaderEpoch | 0 | 1 | + +> ⚠️ **KAFKA-18659 / librdkafka bug**: Even though Produce's actual minimum in Kafka 4.0 is v3, the +> `ApiVersions` response **must advertise min=0** for Produce to avoid breaking librdkafka clients. +> `ApiKeys.java` has a dedicated constant `PRODUCE_API_VERSIONS_RESPONSE_MIN_VERSION = 0` for this. +> Your gateway's `ApiVersions` response encoder must replicate this special case. + +--- + +## Group 1 — Core Data Path + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 0 | **Produce** | 3 †| 12 | v9 | ✅ | 🔴 Bridge | +| 1 | **Fetch** | 4 | 17 | v12 | ✅ | 🔴 Bridge | +| 2 | **ListOffsets** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | +| 3 | **Metadata** | 1 | 12 | v9 | ✅ | 🔴 Bridge | + +† See KAFKA-18659 librdkafka workaround above — advertise min=0 in ApiVersions response. + +--- + +## Group 2 — API Negotiation & Auth + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 17 | **SaslHandshake** | 0 | 1 | never | ✅ | 🔴 Bridge (auth flow) | +| 18 | **ApiVersions** | 0 | 4 | v3 | ✅ | 🔴 Bridge (advertise Iggy caps) | +| 36 | **SaslAuthenticate** | 0 | 2 | v2 | ✅ | 🔴 Bridge (auth flow) | + +> **ApiVersions special case**: The response header is **always v0** (no tagged fields), regardless of +> the request version. This allows clients that don't yet know the server's encoding to parse the +> discovery response. `header.rs` already handles this correctly via the `api_key == 18` guard. + +--- + +## Group 3 — Classic Consumer Group Protocol + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 8 | **OffsetCommit** | 2 | 9 | v8 | ✅ | 🟠 Required Stub | +| 9 | **OffsetFetch** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | +| 10 | **FindCoordinator** | 1 | 6 | v3 | ✅ | 🟠 Required Stub | +| 11 | **JoinGroup** | 2 | 9 | v6 | ✅ | 🟠 Required Stub | +| 12 | **Heartbeat** | 1 | 4 | v4 | ✅ | 🟠 Required Stub | +| 13 | **LeaveGroup** | 1 | 5 | v4 | ✅ | 🟠 Required Stub | +| 14 | **SyncGroup** | 1 | 5 | v4 | ✅ | 🟠 Required Stub | +| 15 | **DescribeGroups** | 0 | 6 | v5 | ✅ | 🟡 Optional Stub | +| 16 | **ListGroups** | 1 | 5 | v3 | ✅ | 🟡 Optional Stub | +| 42 | **DeleteGroups** | 1 | 2 | v2 | ✅ | 🟡 Optional Stub | + +--- + +## Group 4 — New Consumer Group Protocol (KIP-848, Kafka 3.7+) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 68 | **ConsumerGroupHeartbeat** | 0 | 1 | v0 | ✅ | 🟠 Required Stub | +| 69 | **ConsumerGroupDescribe** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | + +> ⚠️ Kafka 4.0 clients use the **new group protocol by default** and will send key 68. +> A gateway that hard-rejects this breaks all modern Kafka consumers. + +--- + +## Group 5 — Topic Administration + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 19 | **CreateTopics** | 2 | 7 | v5 | ✅ (max v5 ⚠️) | 🟠 Required Stub | +| 20 | **DeleteTopics** | 1 | 6 | v4 | ✅ | 🟡 Optional Stub | +| 21 | **DeleteRecords** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 37 | **CreatePartitions** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | + +> ⚠️ `SUPPORTED_RANGES` in `api.rs` currently advertises CreateTopics max=5; actual max is v7. + +--- + +## Group 6 — Transactions (EOS — Exactly Once Semantics) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 22 | **InitProducerId** | 2 | 5 | v2 | ✅ | 🟡 Optional Stub | +| 23 | **OffsetForLeaderEpoch** | 1 | 5 | v4 | ✅ | 🟡 Optional Stub | +| 24 | **AddPartitionsToTxn** | 1 | 5 | v3 | ✅ | 🟡 Optional Stub | +| 25 | **AddOffsetsToTxn** | 1 | 4 | v3 | ✅ | 🟡 Optional Stub | +| 26 | **EndTxn** | 1 | 4 | v3 | ✅ | 🟡 Optional Stub | +| 27 | **WriteTxnMarkers** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | +| 28 | **TxnOffsetCommit** | 2 | 5 | v3 | ✅ | 🟡 Optional Stub | + +--- + +## Group 7 — Security & ACLs + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 29 | **DescribeAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 30 | **CreateAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 31 | **DeleteAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 38 | **CreateDelegationToken** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 39 | **RenewDelegationToken** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 40 | **ExpireDelegationToken** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 41 | **DescribeDelegationToken** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 50 | **DescribeUserScramCredentials** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 51 | **AlterUserScramCredentials** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | + +--- + +## Group 8 — Configuration & Quotas + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 32 | **DescribeConfigs** | 0 | 4 | v4 | ✅ | 🟡 Optional Stub | +| 33 | **AlterConfigs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 44 | **IncrementalAlterConfigs** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | +| 48 | **DescribeClientQuotas** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | +| 49 | **AlterClientQuotas** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | + +--- + +## Group 9 — Log & Partition Admin + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 34 | **AlterReplicaLogDirs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 35 | **DescribeLogDirs** | 0 | 4 | v2 | ✅ | 🟡 Optional Stub | +| 43 | **ElectLeaders** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 45 | **AlterPartitionReassignments** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 46 | **ListPartitionReassignments** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 47 | **OffsetDelete** | 0 | 0 | never | ✅ | 🟡 Optional Stub | +| 57 | **UpdateFeatures** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | + +--- + +## Group 10 — Cluster Introspection + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 55 | **DescribeQuorum** | 0 | 2 | v0 | ✅ | 🟡 Optional Stub | +| 59 | **FetchSnapshot** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | +| 60 | **DescribeCluster** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | +| 61 | **DescribeProducers** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 64 | **UnregisterBroker** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 65 | **DescribeTransactions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 66 | **ListTransactions** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | +| 75 | **DescribeTopicPartitions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | + +--- + +## Group 11 — Observability / Telemetry (KIP-714, Kafka 3.7+) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 71 | **GetTelemetrySubscriptions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 72 | **PushTelemetry** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 76 | **ListClientMetricsResources** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | + +--- + +## Group 12 — Share Groups (NEW in Kafka 4.0, KIP-932) + +> Keys 77–80 use flexible header framing from v0 (added to `header.rs`). Keys 84–88 are internal +> coordinator APIs and can be rejected. + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 77 | **ShareGroupHeartbeat** | 0 | 0 | v0 | ✅ | 🟠 Required Stub | +| 78 | **ShareGroupDescribe** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 79 | **ShareFetch** | 0 | 0 | v0 | ✅ | 🔴 Bridge (share consume) | +| 80 | **ShareAcknowledge** | 0 | 0 | v0 | ✅ | 🟠 Required Stub | + +--- + +## Group 13 — KRaft Raft Voter Management (NEW in Kafka 4.0) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 81 | **AddRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 82 | **RemoveRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 83 | **UpdateRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | + +--- + +## Group 14 — KRaft Internal / Broker-Only (Always Reject at Gateway) + +> These APIs must **never** be handled by a client-facing gateway. +> Return `INVALID_REQUEST` (error code 42) with a properly framed response — **do not drop the connection**. + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 4 | **LeaderAndIsr** | 0 | 7 | v4 | ✅ | ❌ Reject (broker-only) | +| 5 | **StopReplica** | 0 | 4 | v2 | ✅ | ❌ Reject (broker-only) | +| 6 | **UpdateMetadata** | 0 | 8 | v6 | ✅ | ❌ Reject (broker-only) | +| 7 | **ControlledShutdown** | 0 | 3 | v3 | ✅ | ❌ Reject (broker-only) | +| 52 | **Vote** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 53 | **BeginQuorumEpoch** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 54 | **EndQuorumEpoch** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 56 | **AlterPartition** | 0 | 3 | v0 | ✅ | ❌ Reject (KRaft) | +| 58 | **Envelope** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 62 | **BrokerRegistration** | 0 | 4 | v0 | ✅ | ❌ Reject (KRaft) | +| 63 | **BrokerHeartbeat** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 67 | **AllocateProducerIds** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 70 | **ControllerRegistration** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 74 | **AssignReplicasToDirs** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 84 | **InitializeShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 85 | **ReadShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 86 | **WriteShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 87 | **DeleteShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 88 | **ReadShareGroupStateSummary** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | + +--- + +## Summary Counts + +| Category | Count | Notes | +|----------|:-----:|-------| +| 🔴 Bridge (data path) | 6 | Produce, Fetch, Metadata, ApiVersions, SaslHandshake, SaslAuthenticate, ShareFetch | +| 🟠 Required Stub (client state machine) | 14 | Consumer group, CreateTopics, ConsumerGroupHeartbeat (68), ShareGroupHeartbeat (77), ShareAcknowledge (80) | +| 🟡 Optional Stub (admin/observability) | 44 | Can return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` safely | +| ❌ Reject (broker/KRaft internal) | 19 | Return `INVALID_REQUEST` with valid frame — never close the TCP connection | +| **Total API Keys in Kafka 4.0** | **83** | Key IDs 0–88 with gap at 73 | + +--- + +## Current Implementation Gaps in `api.rs` + +### `SUPPORTED_RANGES` is behind the latest Kafka 4.0 max versions + +| API | Declared range | Kafka 4.0 max | Gap | +|-----|:---:|:---:|:---:| +| Produce | v3-v9 | v12 | 3 versions behind | +| Fetch | v4-v12 | v17 | 5 versions behind | +| ListOffsets | v1-v6 | v9 | 3 versions behind | +| Metadata | v0-v9 | v12 | 3 versions behind | +| ApiVersions | v0-v3 | v4 | 1 version behind | +| CreateTopics | v2-v5 | v7 | 2 versions behind | + +### Missing from `SUPPORTED_RANGES` (77 API keys) + +Every key not in the table above falls through to `encode_error_only_response` +(2-byte error frame), including: + +- **Client bootstrap blockers**: OffsetCommit (8), OffsetFetch (9), FindCoordinator (10) +- **Classic consumer group protocol**: JoinGroup (11), Heartbeat (12), LeaveGroup (13), SyncGroup (14) +- **New consumer group protocol**: ConsumerGroupHeartbeat (68) — default in Kafka 4.0 +- **Share groups (KIP-932)**: ShareFetch (79), ShareGroupHeartbeat (77), ShareAcknowledge (80) +- **Auth flow**: SaslHandshake (17), SaslAuthenticate (36) +- **All 19 broker/KRaft-internal keys** (Group 14) — should return `INVALID_REQUEST` with a + valid frame instead of the bare 2-byte fallback, so the connection is never dropped. + +### `header.rs` `request_header_version()` — remaining gaps + +Keys 77-80 (ShareGroupHeartbeat, ShareGroupDescribe, ShareFetch, ShareAcknowledge) are +covered (`flexible_from = 0`). Keys 81-88 — AddRaftVoter, RemoveRaftVoter, UpdateRaftVoter, +and the five ShareGroupState keys (84-88) — are all always-flexible per KIP-932/KRaft but +still fall through to the `_ => i16::MAX` (non-flexible) arm, which would misframe them if +ever dispatched. + +## References + +- [ApiKeys.java @ Kafka 4.0.0](https://github.com/apache/kafka/blob/4.0.0/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java) +- [Kafka Protocol Message Schemas @ 4.0.0](https://github.com/apache/kafka/tree/4.0.0/clients/src/main/resources/common/message) +- [KIP-896: Remove old client protocol API versions in Kafka 4.0](https://cwiki.apache.org/confluence/display/KAFKA/KIP-896%3A+Remove+old+client+protocol+API+versions+in+Kafka+4.0) +- [KIP-848: Apache Kafka Consumer Rebalance Protocol](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol) +- [KIP-932: Queues for Kafka (Share Groups)](https://cwiki.apache.org/confluence/display/KAFKA/KIP-932%3A+Queues+for+Kafka) +- [KIP-714: Client Metrics and Observability](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) +- [Kafka Wire Protocol Documentation](https://kafka.apache.org/protocol.html) +- [kafka-protocol Rust crate](https://crates.io/crates/kafka-protocol) diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs index b2ee237387..16b9de936c 100644 --- a/gateways/kafka/src/error.rs +++ b/gateways/kafka/src/error.rs @@ -19,6 +19,8 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum KafkaProtocolError { + #[error("invalid server configuration: {0}")] + InvalidConfig(String), #[error("buffer underflow: needed {needed} bytes, remaining {remaining}")] BufferUnderflow { needed: usize, remaining: usize }, #[error("invalid frame length: {0}")] @@ -36,8 +38,6 @@ pub enum KafkaProtocolError { UnsupportedHeaderVersion(i16), #[error("invalid array length: {0}")] InvalidArrayLength(i32), - #[error("invalid compact array length: encoded value must be >= 1, got {0}")] - InvalidCompactArrayLength(u64), #[error("collection length {count} exceeds maximum {max}")] CollectionTooLarge { count: usize, max: usize }, #[error("string length {length} exceeds i16::MAX")] diff --git a/gateways/kafka/src/lib.rs b/gateways/kafka/src/lib.rs index c8f0cf9e20..a75b625786 100644 --- a/gateways/kafka/src/lib.rs +++ b/gateways/kafka/src/lib.rs @@ -21,4 +21,4 @@ pub mod error; pub mod protocol; pub mod server; -pub use server::{KafkaServer, ServerConfig, init_tracing}; +pub use server::{KafkaServer, ServerConfig}; diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs index 4a5414b9dd..9cd81a4de2 100644 --- a/gateways/kafka/src/main.rs +++ b/gateways/kafka/src/main.rs @@ -25,7 +25,20 @@ use iggy_gateway_kafka::{KafkaServer, ServerConfig}; async fn main() -> Result<(), Box> { init_tracing(); - let config = ServerConfig::default(); + let mut config = ServerConfig::default(); + if let Ok(bind_addr) = std::env::var("KAFKA_BIND_ADDR") { + config.bind_addr = bind_addr; + } + if let Ok(advertised_host) = std::env::var("KAFKA_ADVERTISED_HOST") { + config.advertised_host = Some(advertised_host); + } + if let Ok(advertised_port) = std::env::var("KAFKA_ADVERTISED_PORT") { + config.advertised_port = Some( + advertised_port + .parse() + .map_err(|e| format!("invalid KAFKA_ADVERTISED_PORT `{advertised_port}`: {e}"))?, + ); + } let server = KafkaServer::new(config); let (tx, rx) = broadcast::channel(1); diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 3e39698841..55cf238f95 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -25,8 +25,9 @@ use crate::protocol::requests::{ decode_produce_request, }; use crate::protocol::responses::{ - encode_create_topics_response, encode_fetch_response, encode_list_offsets_response, - encode_produce_response, + encode_create_topics_error_response, encode_create_topics_response, + encode_fetch_error_response, encode_fetch_response, encode_list_offsets_error_response, + encode_list_offsets_response, encode_produce_error_response, encode_produce_response, }; pub const API_KEY_PRODUCE: i16 = 0; @@ -52,6 +53,9 @@ pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; pub const ERROR_INVALID_REQUEST: i16 = 42; pub const ERROR_UNSUPPORTED_FOR_MESSAGE_FORMAT: i16 = 43; +/// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. +const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; + #[derive(Debug, Clone)] pub struct BrokerAdvertise { pub host: String, @@ -139,7 +143,8 @@ pub fn handle_request( if is_supported_version(api_key, api_version) { encode_api_versions_response(api_version, ERROR_NONE) } else { - encode_api_versions_response(1, ERROR_UNSUPPORTED_VERSION) + // KIP-511: reply with v0 when the requested version is not understood. + encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION) } } API_KEY_METADATA => { @@ -155,11 +160,11 @@ pub fn handle_request( Ok(req) => encode_produce_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode Produce request: {:?}", e); - encode_error_only_response(ERROR_CORRUPT_MESSAGE) + encode_produce_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + encode_produce_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } API_KEY_FETCH => { @@ -168,11 +173,11 @@ pub fn handle_request( Ok(req) => encode_fetch_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode Fetch request: {:?}", e); - encode_error_only_response(ERROR_CORRUPT_MESSAGE) + encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + encode_fetch_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } API_KEY_LIST_OFFSETS => { @@ -181,11 +186,11 @@ pub fn handle_request( Ok(req) => encode_list_offsets_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode ListOffsets request: {:?}", e); - encode_error_only_response(ERROR_CORRUPT_MESSAGE) + encode_list_offsets_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + encode_list_offsets_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } API_KEY_CREATE_TOPICS => { @@ -194,11 +199,11 @@ pub fn handle_request( Ok(req) => encode_create_topics_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode CreateTopics request: {:?}", e); - encode_error_only_response(ERROR_CORRUPT_MESSAGE) + encode_create_topics_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + encode_create_topics_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } _ => encode_error_only_response(ERROR_UNSUPPORTED_VERSION), @@ -213,6 +218,19 @@ pub fn is_supported_version(api_key: i16, api_version: i16) -> bool { .is_some_and(|r| api_version >= r.min_version && api_version <= r.max_version) } +/// Min version advertised in `ApiVersions` (may differ from the firewall min). +/// +/// Produce must advertise min=0 per KAFKA-18659 / `PRODUCE_API_VERSIONS_RESPONSE_MIN_VERSION` +/// even though this gateway only accepts Produce v3+. +#[must_use] +pub const fn advertised_min_version(api_key: i16, firewall_min: i16) -> i16 { + if api_key == API_KEY_PRODUCE { + 0 + } else { + firewall_min + } +} + fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { let flexible = api_version >= 3; let ranges = SUPPORTED_RANGES; @@ -224,7 +242,7 @@ fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { e.write_varint((ranges.len() + 1) as u64); for r in ranges { e.write_i16(r.api_key); - e.write_i16(r.min_version); + e.write_i16(advertised_min_version(r.api_key, r.min_version)); e.write_i16(r.max_version); e.write_empty_tagged_fields(); } @@ -232,7 +250,7 @@ fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { e.write_i32(i32::try_from(ranges.len()).expect("supported range table is small")); for r in ranges { e.write_i16(r.api_key); - e.write_i16(r.min_version); + e.write_i16(advertised_min_version(r.api_key, r.min_version)); e.write_i16(r.max_version); } } @@ -264,8 +282,8 @@ fn encode_metadata_response( let mut e = Encoder::with_capacity(256); - if api_version >= 1 { - e.write_i32(0); // throttle_time_ms + if api_version >= 3 { + e.write_i32(0); // throttle_time_ms (Metadata v3+) } if flexible { @@ -276,21 +294,19 @@ fn encode_metadata_response( e.write_compact_nullable_string(None); // rack e.write_empty_tagged_fields(); - if api_version >= 2 { - e.write_compact_nullable_string(None); // cluster_id - } - e.write_i32(1); // controller_id + e.write_compact_nullable_string(None); // cluster_id (v2+) + e.write_i32(1); // controller_id (v1+) e.write_varint((topics_count + 1) as u64); for _ in 0..topics_count { e.write_i16(topic_error); e.write_compact_nullable_string(Some("unknown-topic")); - if api_version >= 1 { - e.write_bool(false); // is_internal — must come before partitions array - } + e.write_bool(false); // is_internal (v1+) e.write_varint(1); // empty partitions array + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations (v8+) e.write_empty_tagged_fields(); } + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // cluster_authorized_operations (v8+) e.write_empty_tagged_fields(); } else { e.write_i32(1); // brokers array length @@ -316,6 +332,12 @@ fn encode_metadata_response( e.write_bool(false); // is_internal } e.write_i32(0); // partitions array (empty) + if api_version >= 8 { + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations + } + } + if api_version >= 8 { + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // cluster_authorized_operations } } @@ -330,7 +352,7 @@ pub fn encode_error_only_response(error_code: i16) -> Bytes { } #[must_use] -pub fn split_metadata_request_topics(body: Bytes, api_version: i16) -> usize { +pub(crate) fn split_metadata_request_topics(body: Bytes, api_version: i16) -> usize { let mut d = Decoder::new(body); if api_version >= 9 { d.read_compact_array_count().unwrap_or(0) diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index e1c9ce68ba..b46cb6476a 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -24,6 +24,7 @@ use bytes::{Buf, BufMut, Bytes, BytesMut}; use crate::error::{KafkaProtocolError, Result}; /// Upper bound for Kafka array/collection element counts decoded from the wire. +/// Matches typical broker limits and prevents OOM from adversarial length prefixes. pub const MAX_COLLECTION_LEN: usize = 65_536; pub struct Decoder { @@ -117,7 +118,10 @@ impl Decoder { if n == 0 { return Ok(0); } - let count = (n - 1) as usize; + let count = usize::try_from(n - 1).map_err(|_| KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + })?; if count > MAX_COLLECTION_LEN { return Err(KafkaProtocolError::CollectionTooLarge { count, @@ -148,7 +152,12 @@ impl Decoder { if len_plus_one == 0 { return Ok(None); } - let len = (len_plus_one - 1) as usize; + let len = usize::try_from(len_plus_one - 1).map_err(|_| { + KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + } + })?; self.ensure(len)?; let s = std::str::from_utf8(&self.bytes.chunk()[..len]) .map_err(|_| KafkaProtocolError::InvalidUtf8)? @@ -174,7 +183,12 @@ impl Decoder { if len_plus_one == 0 { return Ok(None); } - let len = (len_plus_one - 1) as usize; + let len = usize::try_from(len_plus_one - 1).map_err(|_| { + KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + } + })?; self.ensure(len)?; Ok(Some(self.bytes.copy_to_bytes(len))) } @@ -197,7 +211,12 @@ impl Decoder { let count = count as usize; for _ in 0..count { self.read_varint()?; // tag number - let size = self.read_varint()? as usize; + let size = usize::try_from(self.read_varint()?).map_err(|_| { + KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + } + })?; self.ensure(size)?; self.bytes.advance(size); } @@ -288,19 +307,21 @@ impl Encoder { } /// Legacy nullable bytes: i32 length prefix, -1 for null. - pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) { + pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) -> Result<()> { match v { None => self.write_i32(-1), Some(b) => { - debug_assert!( - b.len() <= i32::MAX as usize, - "byte slice length {} exceeds i32::MAX", - b.len() - ); - self.write_i32(b.len() as i32); + if b.len() > i32::MAX as usize { + return Err(KafkaProtocolError::CollectionTooLarge { + count: b.len(), + max: i32::MAX as usize, + }); + } + self.write_i32(i32::try_from(b.len()).expect("checked above")); self.bytes.put_slice(b); } } + Ok(()) } /// Compact nullable bytes (flexible versions): varint(len+1), 0 for null. diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index 66ce358fe7..7977f4e5c9 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -114,6 +114,10 @@ pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { 74 => 0, // AssignReplicasToDirs — always flexible 75 => 0, // DescribeTopicPartitions — always flexible 76 => 0, // ListClientMetricsResources — always flexible + 77 => 0, // ShareGroupHeartbeat — always flexible (Kafka 4.0) + 78 => 0, // ShareGroupDescribe — always flexible + 79 => 0, // ShareFetch — always flexible + 80 => 0, // ShareAcknowledge — always flexible _ => i16::MAX, // Unknown API — assume non-flexible }; if api_version >= flexible_from { 2 } else { 1 } diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index d2bc6b74be..91e2a40b66 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -156,7 +156,7 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { let isolation_level = if version >= 4 { d.read_i8()? } else { 0 }; - // session_id and session_epoch (v7+) — we'll skip for now + // session_id and session_epoch (v7+) — read and discard (stub path) if version >= 7 { d.read_i32()?; // session_id d.read_i32()?; // session_epoch diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index 01d1c1b13e..8de6868ffd 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -22,21 +22,42 @@ use crate::protocol::api::{ERROR_INVALID_PARTITIONS, ERROR_NONE}; use crate::protocol::codec::Encoder; use crate::protocol::requests::{ - CreateTopicsRequest, FetchRequest, ListOffsetsRequest, ProduceRequest, + CreateTopicsRequest, FetchRequest, ListOffsetsRequest, ProducePartitionData, ProduceRequest, + ProduceTopicData, }; use bytes::Bytes; +/// Well-formed Produce response with a single placeholder topic/partition. +pub fn encode_produce_error_response(version: i16, error_code: i16) -> Bytes { + let topics = vec![ProduceTopicData { + topic: String::new(), + partitions: vec![ProducePartitionData { + partition: 0, + records: None, + }], + }]; + encode_produce_response_inner(version, &topics, error_code) +} + pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { + encode_produce_response_inner(version, &req.topics, ERROR_NONE) +} + +fn encode_produce_response_inner( + version: i16, + topics: &[ProduceTopicData], + partition_error: i16, +) -> Bytes { let flexible = version >= 9; let mut e = Encoder::with_capacity(512); if flexible { - e.write_varint((req.topics.len() + 1) as u64); + e.write_varint((topics.len() + 1) as u64); } else { - e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); } - for topic in &req.topics { + for topic in topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { @@ -51,7 +72,7 @@ pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { for p in &topic.partitions { e.write_i32(p.partition); - e.write_i16(ERROR_NONE); + e.write_i16(partition_error); e.write_i64(0); if version >= 2 { e.write_i64(-1); @@ -88,7 +109,36 @@ pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { e.freeze() } +/// Well-formed Fetch response. Uses top-level `error_code` at v7+, or a single +/// placeholder topic/partition with per-partition `error_code` below v7. +pub fn encode_fetch_error_response(version: i16, error_code: i16) -> Bytes { + use crate::protocol::requests::{FetchPartition, FetchTopic}; + + if version >= 7 { + return encode_fetch_response_inner(version, &[], Some(error_code), error_code); + } + + let topics = vec![FetchTopic { + topic: String::new(), + partitions: vec![FetchPartition { + partition: 0, + fetch_offset: 0, + partition_max_bytes: 1, + }], + }]; + encode_fetch_response_inner(version, &topics, Some(ERROR_NONE), error_code) +} + pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { + encode_fetch_response_inner(version, &req.topics, Some(ERROR_NONE), ERROR_NONE) +} + +fn encode_fetch_response_inner( + version: i16, + topics: &[crate::protocol::requests::FetchTopic], + top_level_error: Option, + partition_error: i16, +) -> Bytes { let flexible = version >= 12; let mut e = Encoder::with_capacity(512); @@ -96,17 +146,17 @@ pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { e.write_i32(0); } if version >= 7 { - e.write_i16(ERROR_NONE); + e.write_i16(top_level_error.unwrap_or(ERROR_NONE)); e.write_i32(0); } if flexible { - e.write_varint((req.topics.len() + 1) as u64); + e.write_varint((topics.len() + 1) as u64); } else { - e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); } - for topic in &req.topics { + for topic in topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { @@ -121,28 +171,29 @@ pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { for partition in &topic.partitions { e.write_i32(partition.partition); - e.write_i16(ERROR_NONE); - e.write_i64(0); + e.write_i16(partition_error); + e.write_i64(0); // high_watermark if version >= 4 { - e.write_i64(0); + e.write_i64(0); // last_stable_offset } if version >= 5 { - e.write_i64(0); + e.write_i64(0); // log_start_offset } if version >= 4 { if flexible { - e.write_varint(1); + e.write_varint(1); // empty aborted_transactions } else { - e.write_i32(0); + e.write_i32(0); // empty aborted_transactions } } if version >= 11 { - e.write_i32(-1); + e.write_i32(-1); // preferred_read_replica } if flexible { e.write_compact_nullable_bytes(None); } else { - e.write_nullable_bytes(None); + e.write_nullable_bytes(None) + .expect("null bytes always encode"); } if flexible { e.write_empty_tagged_fields(); @@ -161,7 +212,29 @@ pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { e.freeze() } +/// Well-formed ListOffsets response with a single placeholder topic/partition. +pub fn encode_list_offsets_error_response(version: i16, error_code: i16) -> Bytes { + use crate::protocol::requests::{ListOffsetsPartition, ListOffsetsTopic}; + + let topics = vec![ListOffsetsTopic { + topic: String::new(), + partitions: vec![ListOffsetsPartition { + partition: 0, + timestamp: -1, + }], + }]; + encode_list_offsets_response_inner(version, &topics, error_code) +} + pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> Bytes { + encode_list_offsets_response_inner(version, &req.topics, ERROR_NONE) +} + +fn encode_list_offsets_response_inner( + version: i16, + topics: &[crate::protocol::requests::ListOffsetsTopic], + partition_error: i16, +) -> Bytes { let flexible = version >= 6; let mut e = Encoder::with_capacity(256); @@ -170,12 +243,12 @@ pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> B } if flexible { - e.write_varint((req.topics.len() + 1) as u64); + e.write_varint((topics.len() + 1) as u64); } else { - e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); } - for topic in &req.topics { + for topic in topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { @@ -190,7 +263,7 @@ pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> B for partition in &topic.partitions { e.write_i32(partition.partition); - e.write_i16(ERROR_NONE); + e.write_i16(partition_error); let offset = 0i64; if version >= 1 { @@ -217,7 +290,27 @@ pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> B e.freeze() } +/// Well-formed CreateTopics response with a single placeholder topic. +pub fn encode_create_topics_error_response(version: i16, error_code: i16) -> Bytes { + use crate::protocol::requests::CreatableTopic; + + let topics = vec![CreatableTopic { + name: String::new(), + num_partitions: 1, + replication_factor: 1, + }]; + encode_create_topics_response_inner(version, &topics, error_code) +} + pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> Bytes { + encode_create_topics_response_inner(version, &req.topics, ERROR_NONE) +} + +fn encode_create_topics_response_inner( + version: i16, + topics: &[crate::protocol::requests::CreatableTopic], + topic_error: i16, +) -> Bytes { let flexible = version >= 5; let mut e = Encoder::with_capacity(256); @@ -226,19 +319,21 @@ pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> } if flexible { - e.write_varint((req.topics.len() + 1) as u64); + e.write_varint((topics.len() + 1) as u64); } else { - e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); } - for topic in &req.topics { + for topic in topics { if flexible { e.write_compact_nullable_string(Some(&topic.name)); } else { let _ = e.write_nullable_string(Some(&topic.name)); } - let error_code = if topic.num_partitions <= 0 { + let error_code = if topic_error != ERROR_NONE { + topic_error + } else if topic.num_partitions <= 0 { ERROR_INVALID_PARTITIONS } else { ERROR_NONE @@ -254,7 +349,6 @@ pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> } if version >= 5 { - e.write_i16(ERROR_NONE); e.write_i32(topic.num_partitions); e.write_i16(topic.replication_factor); e.write_varint(1); diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 02e2bfaf6f..94ef7cb879 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -23,9 +23,9 @@ use bytes::{BufMut, BytesMut}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::broadcast; -use tokio::time::timeout; +use tokio::time::{timeout, timeout_at}; use tokio_util::task::TaskTracker; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{ @@ -40,6 +40,11 @@ use std::io; #[derive(Debug, Clone)] pub struct ServerConfig { pub bind_addr: String, + /// Hostname or IP advertised in Metadata (`KAFKA_ADVERTISED_HOST`). Required when `bind_addr` + /// uses a wildcard address (`0.0.0.0` / `::`). + pub advertised_host: Option, + /// Port advertised in Metadata (`KAFKA_ADVERTISED_PORT`). Defaults to the bind port. + pub advertised_port: Option, pub max_frame_size: usize, pub read_timeout: Duration, pub write_timeout: Duration, @@ -49,6 +54,8 @@ impl Default for ServerConfig { fn default() -> Self { Self { bind_addr: "127.0.0.1:9093".to_string(), + advertised_host: None, + advertised_port: None, max_frame_size: 8 * 1024 * 1024, read_timeout: Duration::from_secs(15), write_timeout: Duration::from_secs(10), @@ -56,6 +63,43 @@ impl Default for ServerConfig { } } +impl BrokerAdvertise { + /// Resolve the broker endpoint advertised in Metadata from listener config. + /// + /// # Errors + /// + /// Returns an error when `bind_addr` is invalid, `advertised_host` is empty, or the listener + /// binds to a wildcard address without an explicit advertised host. + pub fn from_server_config(config: &ServerConfig) -> std::result::Result { + let bind = config + .bind_addr + .parse::() + .map_err(|e| format!("invalid bind address `{}`: {e}", config.bind_addr))?; + + let port = config + .advertised_port + .map_or_else(|| i32::from(bind.port()), i32::from); + + let host = if let Some(ref advertised) = config.advertised_host { + let trimmed = advertised.trim(); + if trimmed.is_empty() { + return Err("KAFKA_ADVERTISED_HOST must not be empty".into()); + } + trimmed.to_string() + } else if bind.ip().is_unspecified() { + return Err( + "binding to a wildcard address (0.0.0.0 or ::) requires KAFKA_ADVERTISED_HOST \ + to be set to a reachable hostname or IP for Metadata broker advertisement" + .into(), + ); + } else { + bind.ip().to_string() + }; + + Ok(Self { host, port }) + } +} + pub struct KafkaServer { config: Arc, } @@ -74,19 +118,42 @@ impl KafkaServer { /// /// Returns an error if binding fails or a non-transient `accept()` error occurs. pub async fn run(self, mut shutdown: broadcast::Receiver<()>) -> Result<()> { + let broker = Arc::new( + BrokerAdvertise::from_server_config(&self.config) + .map_err(KafkaProtocolError::InvalidConfig)?, + ); let listener = TcpListener::bind(&self.config.bind_addr).await?; - info!("kafka listener bound on {}", self.config.bind_addr); + info!( + "kafka listener bound on {} (advertised as {}:{})", + self.config.bind_addr, broker.host, broker.port + ); let tracker = TaskTracker::new(); - let broker = Arc::new(BrokerAdvertise::from_bind_addr(&self.config.bind_addr)); + let broker = Arc::clone(&broker); loop { tokio::select! { - _ = shutdown.recv() => { - info!("kafka listener shutdown requested"); - tracker.close(); - tracker.wait().await; - break; + result = shutdown.recv() => { + match result { + Ok(()) => { + info!("kafka listener shutdown requested"); + tracker.close(); + tracker.wait().await; + break; + } + // Capacity-1 channel: lagged means a signal was sent before we polled — treat as shutdown. + Err(broadcast::error::RecvError::Lagged(_)) => { + info!("kafka listener shutdown requested (lagged)"); + tracker.close(); + tracker.wait().await; + break; + } + Err(broadcast::error::RecvError::Closed) => { + tracker.close(); + tracker.wait().await; + break; + } + } } accept_result = listener.accept() => { match accept_result { @@ -130,7 +197,7 @@ async fn handle_connection( peer: SocketAddr, broker: Arc, ) -> Result<()> { - info!(%peer, "connection accepted"); + debug!(%peer, "connection accepted"); loop { let frame = match read_frame(&mut stream, config.max_frame_size, config.read_timeout).await @@ -146,9 +213,9 @@ async fn handle_connection( Err(e) => return Err(e), }; - if frame.len() < 4 { + if frame.len() < 8 { return Err(KafkaProtocolError::BufferUnderflow { - needed: 4, + needed: 8, remaining: frame.len(), }); } @@ -178,7 +245,7 @@ async fn handle_connection( Err(e) => return Err(e), }; - info!( + debug!( %peer, api_key = req.api_key, api_version = req.api_version, @@ -215,14 +282,13 @@ async fn send_response( ) -> Result<()> { let header_size = ResponseHeader::encoded_size(header_version); let payload_size = header_size + body.len(); - if payload_size > i32::MAX as usize { - return Err(KafkaProtocolError::FrameTooLarge { + let payload_len_i32 = + i32::try_from(payload_size).map_err(|_| KafkaProtocolError::FrameTooLarge { max_bytes: i32::MAX as usize, actual_bytes: payload_size, - }); - } + })?; let mut frame = BytesMut::with_capacity(4 + payload_size); - frame.put_i32(payload_size as i32); + frame.put_i32(payload_len_i32); header.encode_into(&mut frame, header_version); frame.put_slice(body); timeout(write_timeout, stream.write_all(&frame)) @@ -232,11 +298,7 @@ async fn send_response( } fn correlation_id_from_frame(frame: &bytes::Bytes) -> i32 { - if frame.len() >= 8 { - i32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]) - } else { - 0 - } + i32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]) } /// Read one length-prefixed Kafka frame from `stream`. @@ -262,7 +324,8 @@ pub async fn read_frame( let frame_len = usize::try_from(frame_len_i32).map_err(|_| KafkaProtocolError::FrameTooLarge { max_bytes: max_frame_size, - actual_bytes: u32::MAX as usize, + // Positive i32 that does not fit in `usize` (e.g. 16-bit targets). + actual_bytes: usize::MAX, })?; if frame_len > max_frame_size { return Err(KafkaProtocolError::FrameTooLarge { @@ -272,51 +335,23 @@ pub async fn read_frame( } // read_buf fills BytesMut spare capacity without zero-initializing it first. + // Single deadline for the entire body so a slow-drip sender can't stall indefinitely + // by delivering one byte per timeout window. + let deadline = tokio::time::Instant::now() + read_timeout; let mut data = BytesMut::with_capacity(frame_len); - timeout(read_timeout, async { - while data.len() < frame_len { - if stream.read_buf(&mut data).await? == 0 { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "connection closed", - )); + while data.len() < frame_len { + match timeout_at(deadline, stream.read_buf(&mut data)).await { + Err(_) => return Err(io::Error::new(io::ErrorKind::TimedOut, "read timeout").into()), + Ok(Ok(0)) => { + return Err( + io::Error::new(io::ErrorKind::UnexpectedEof, "connection closed").into(), + ); } + Ok(Err(e)) => return Err(e.into()), + Ok(Ok(_)) => {} } - Ok::<_, io::Error>(()) - }) - .await - .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "read timeout"))??; - Ok(data.freeze()) -} - -/// Write one length-prefixed Kafka frame to `stream`. -/// -/// # Errors -/// -/// Returns an error on timeout, oversize payload, or I/O failure. -pub async fn write_frame( - stream: &mut TcpStream, - payload: &[u8], - write_timeout: Duration, -) -> Result<()> { - let len = payload.len(); - if len > i32::MAX as usize { - return Err(KafkaProtocolError::FrameTooLarge { - max_bytes: i32::MAX as usize, - actual_bytes: len, - }); } - let mut frame = BytesMut::with_capacity(4 + len); - let len_i32 = i32::try_from(len).map_err(|_| KafkaProtocolError::FrameTooLarge { - max_bytes: i32::MAX as usize, - actual_bytes: len, - })?; - frame.put_i32(len_i32); - frame.extend_from_slice(payload); - timeout(write_timeout, stream.write_all(&frame)) - .await - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timeout"))??; - Ok(()) + Ok(data.freeze()) } pub fn init_tracing() { diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index aca2193708..2d342cfd8e 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -19,7 +19,7 @@ use bytes::Bytes; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, ERROR_UNSUPPORTED_VERSION, - handle_request, is_supported_version, split_metadata_request_topics, supported_api_ranges, + handle_request, is_supported_version, supported_api_ranges, }; fn test_broker() -> BrokerAdvertise { @@ -63,7 +63,7 @@ fn api_versions_v3_response_flexible_format() { // Flexible: varint(len+1) compact array let count_plus_one = d.read_varint().unwrap(); assert!(count_plus_one >= 3); // at least 2 entries → varint = 3+ - let count = (count_plus_one - 1) as i32; + let count = i32::try_from(count_plus_one - 1).expect("api count fits i32"); let mut keys = Vec::new(); for _ in 0..count { @@ -131,13 +131,6 @@ fn unknown_api_key_returns_error_only_payload() { assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } -#[test] -fn metadata_topic_split_reads_array_count() { - let mut raw = Vec::new(); - raw.extend_from_slice(&2_i32.to_be_bytes()); - assert_eq!(split_metadata_request_topics(Bytes::from(raw), 0), 2); -} - #[test] fn version_support_table_is_applied() { assert!(is_supported_version(API_KEY_API_VERSIONS, 3)); @@ -145,3 +138,14 @@ fn version_support_table_is_applied() { assert!(is_supported_version(API_KEY_METADATA, 1)); assert!(!is_supported_version(API_KEY_METADATA, -1)); } + +#[test] +fn apiversions_unsupported_version_uses_v0_encoding_without_throttle() { + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &test_broker()); + // v0: error_code(2) + api_keys i32 count(4) + 6 entries × 6 bytes = 42 — no throttle_time_ms. + assert_eq!(body.len(), 42); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i32().unwrap(), 6); + assert_eq!(d.remaining(), 36); +} diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs new file mode 100644 index 0000000000..9d60d418e4 --- /dev/null +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `BrokerAdvertise` parsing and metadata reflection. + +use iggy_gateway_kafka::ServerConfig; +use iggy_gateway_kafka::protocol::api::{API_KEY_METADATA, BrokerAdvertise, handle_request}; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +#[test] +fn from_bind_addr_parses_ipv4_and_port() { + let b = BrokerAdvertise::from_bind_addr("192.168.1.10:19092"); + assert_eq!(b.host, "192.168.1.10"); + assert_eq!(b.port, 19092); +} + +#[test] +fn from_bind_addr_parses_ipv6() { + let b = BrokerAdvertise::from_bind_addr("[::1]:9093"); + assert_eq!(b.host, "::1"); + assert_eq!(b.port, 9093); +} + +#[test] +fn from_bind_addr_invalid_falls_back_to_default() { + let b = BrokerAdvertise::from_bind_addr("not-a-socket-addr"); + assert_eq!(b.host, "127.0.0.1"); + assert_eq!(b.port, 9093); +} + +#[test] +fn default_matches_standard_gateway_port() { + let b = BrokerAdvertise::default(); + assert_eq!(b.host, "127.0.0.1"); + assert_eq!(b.port, 9093); +} + +#[test] +fn metadata_reflects_parsed_bind_addr() { + let broker = BrokerAdvertise::from_bind_addr("203.0.113.7:9093"); + let mut req = Encoder::with_capacity(4); + req.write_i32(0); + let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker); + + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + assert_eq!(host, "203.0.113.7"); + assert_eq!(port, 9093); +} + +#[test] +fn from_server_config_uses_explicit_advertised_host_on_wildcard_bind() { + let config = ServerConfig { + bind_addr: "0.0.0.0:9093".to_string(), + advertised_host: Some("kafka.internal".to_string()), + ..ServerConfig::default() + }; + let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + assert_eq!(broker.host, "kafka.internal"); + assert_eq!(broker.port, 9093); +} + +#[test] +fn from_server_config_rejects_wildcard_bind_without_advertised_host() { + let config = ServerConfig { + bind_addr: "0.0.0.0:9093".to_string(), + ..ServerConfig::default() + }; + let err = BrokerAdvertise::from_server_config(&config).unwrap_err(); + assert!(err.contains("KAFKA_ADVERTISED_HOST")); +} + +#[test] +fn from_server_config_uses_bind_ip_for_non_wildcard_listener() { + let config = ServerConfig { + bind_addr: "192.168.1.10:19092".to_string(), + ..ServerConfig::default() + }; + let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + assert_eq!(broker.host, "192.168.1.10"); + assert_eq!(broker.port, 19092); +} + +#[test] +fn from_server_config_honors_advertised_port_override() { + let config = ServerConfig { + bind_addr: "127.0.0.1:9093".to_string(), + advertised_host: Some("broker.example.com".to_string()), + advertised_port: Some(19093), + ..ServerConfig::default() + }; + let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + assert_eq!(broker.host, "broker.example.com"); + assert_eq!(broker.port, 19093); +} diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs index 0b25fe7929..fcaa966477 100644 --- a/gateways/kafka/tests/codec_tests.rs +++ b/gateways/kafka/tests/codec_tests.rs @@ -28,8 +28,8 @@ fn codec_round_trip_primitives_and_nullable_fields() { enc.write_i64(9_999_999); enc.write_nullable_string(Some("client-a")).unwrap(); enc.write_nullable_string(None).unwrap(); - enc.write_nullable_bytes(Some(&[1, 2, 3])); - enc.write_nullable_bytes(None); + enc.write_nullable_bytes(Some(&[1, 2, 3])).unwrap(); + enc.write_nullable_bytes(None).unwrap(); let bytes = enc.freeze(); let mut dec = Decoder::new(bytes); @@ -72,7 +72,17 @@ fn codec_u8_and_bool() { #[test] fn varint_round_trip_small_values() { - for v in [0u64, 1, 127, 128, 255, 300, 16383, 16384, u32::MAX as u64] { + for v in [ + 0u64, + 1, + 127, + 128, + 255, + 300, + 16383, + 16384, + u64::from(u32::MAX), + ] { let mut enc = Encoder::with_capacity(16); enc.write_varint(v); let mut dec = Decoder::new(enc.freeze()); diff --git a/gateways/kafka/tests/common/fixtures.rs b/gateways/kafka/tests/common/fixtures.rs new file mode 100644 index 0000000000..02eb43d369 --- /dev/null +++ b/gateways/kafka/tests/common/fixtures.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Fixture loaders — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use std::path::PathBuf; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::header::{RequestHeader, request_header_version}; + +pub fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") +} + +pub fn fixture_exists(api_key: i16, api_name: &str, version: i16) -> bool { + let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); + fixtures_dir().join(filename).is_file() +} + +/// Load request body bytes from a kafka-tool `.bin` fixture (skips frame header). +pub fn load_fixture_body(api_key: i16, api_name: &str, version: i16) -> Bytes { + let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); + let path = fixtures_dir().join(&filename); + let data = std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {filename}: {e}")); + extract_body_from_framed_message(api_key, version, &data) +} + +/// Strip the 4-byte length prefix and Kafka request header from a framed message. +pub fn extract_body_from_framed_message(api_key: i16, api_version: i16, data: &[u8]) -> Bytes { + let frame = Bytes::copy_from_slice(&data[4..]); + let hdr_ver = request_header_version(api_key, api_version); + let mut decoder = Decoder::new(frame); + RequestHeader::decode_from(&mut decoder, hdr_ver).expect("fixture request header must decode"); + decoder + .read_bytes(decoder.remaining()) + .expect("fixture request body must decode") +} diff --git a/gateways/kafka/tests/common/scope.rs b/gateways/kafka/tests/common/scope.rs new file mode 100644 index 0000000000..61a3d37f51 --- /dev/null +++ b/gateways/kafka/tests/common/scope.rs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared scope constants — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use iggy_gateway_kafka::protocol::api::BrokerAdvertise; + +/// Scoped API keys exercised by the #3421 regression suite. +pub const SCOPED_API_KEYS: &[(i16, &str, i16, i16)] = &[ + (0, "Produce", 3, 9), + (1, "Fetch", 4, 12), + (2, "ListOffsets", 1, 6), + (3, "Metadata", 0, 9), + (18, "ApiVersions", 0, 3), + (19, "CreateTopics", 2, 5), +]; + +pub fn default_broker() -> BrokerAdvertise { + BrokerAdvertise::default() +} diff --git a/gateways/kafka/tests/common/server.rs b/gateways/kafka/tests/common/server.rs new file mode 100644 index 0000000000..178dbce2e5 --- /dev/null +++ b/gateways/kafka/tests/common/server.rs @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Test server spawn helper — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use std::net::SocketAddr; +use std::time::Duration; + +use tokio::sync::broadcast; + +use iggy_gateway_kafka::{KafkaServer, ServerConfig}; + +/// Bind an ephemeral port, start `KafkaServer`, return address + shutdown sender. +pub async fn spawn_test_server() -> (SocketAddr, broadcast::Sender<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("local addr"); + drop(listener); + + let config = ServerConfig { + bind_addr: addr.to_string(), + advertised_host: None, + advertised_port: None, + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(5), + write_timeout: Duration::from_secs(5), + }; + let (shutdown_tx, shutdown_rx) = broadcast::channel(1); + let server = KafkaServer::new(config); + tokio::spawn(async move { + let _ = server.run(shutdown_rx).await; + }); + + tokio::time::sleep(Duration::from_millis(50)).await; + (addr, shutdown_tx) +} diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs new file mode 100644 index 0000000000..6eea90f237 --- /dev/null +++ b/gateways/kafka/tests/common/tcp.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! TCP round-trip helpers — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use std::net::SocketAddr; + +use bytes::{BufMut, Bytes, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::header::{request_header_version, response_header_version}; + +/// Build a complete length-prefixed Kafka request frame (header + body). +pub fn build_request_frame( + api_key: i16, + api_version: i16, + correlation_id: i32, + client_id: Option<&str>, + body: &[u8], +) -> Bytes { + let hdr_ver = request_header_version(api_key, api_version); + let mut enc = iggy_gateway_kafka::protocol::codec::Encoder::with_capacity(64 + body.len()); + enc.write_i16(api_key); + enc.write_i16(api_version); + enc.write_i32(correlation_id); + if hdr_ver >= 2 { + enc.write_compact_nullable_string(client_id); + enc.write_empty_tagged_fields(); + } else { + enc.write_nullable_string(client_id) + .expect("test client_id fits i16"); + } + enc.write_bytes(body); + + let payload = enc.freeze(); + let payload_len = i32::try_from(payload.len()).expect("test payload fits i32"); + let mut frame = BytesMut::with_capacity(4 + payload.len()); + frame.put_i32(payload_len); + frame.extend_from_slice(&payload); + frame.freeze() +} + +/// Parse correlation id and response body from a raw response payload (no length prefix). +pub fn parse_response_payload(api_key: i16, api_version: i16, payload: Bytes) -> (i32, Bytes) { + let resp_hdr_ver = response_header_version(api_key, api_version); + let mut d = Decoder::new(payload); + let correlation_id = d.read_i32().expect("correlation_id"); + if resp_hdr_ver >= 1 { + d.read_tagged_fields().expect("response tagged fields"); + } + let body = d.read_bytes(d.remaining()).expect("response body"); + (correlation_id, body) +} + +/// Read one length-prefixed response frame from the stream. +pub async fn read_response_frame(stream: &mut TcpStream, max_size: usize) -> Bytes { + let mut len_buf = [0u8; 4]; + stream + .read_exact(&mut len_buf) + .await + .expect("response length prefix"); + let frame_len_i32 = i32::from_be_bytes(len_buf); + assert!(frame_len_i32 > 0, "response frame length must be positive"); + let frame_len = usize::try_from(frame_len_i32).expect("positive i32 frame length fits usize"); + assert!( + frame_len <= max_size, + "response frame too large: {frame_len}" + ); + let mut buf = vec![0u8; frame_len]; + stream.read_exact(&mut buf).await.expect("response body"); + Bytes::from(buf) +} + +/// Send one request frame and return parsed `(correlation_id, response_body)`. +pub async fn round_trip( + addr: SocketAddr, + api_key: i16, + api_version: i16, + correlation_id: i32, + body: &[u8], +) -> (i32, Bytes) { + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let frame = build_request_frame( + api_key, + api_version, + correlation_id, + Some("regression-test"), + body, + ); + stream.write_all(&frame).await.expect("write request"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + parse_response_payload(api_key, api_version, payload) +} diff --git a/gateways/kafka/tests/decode_safety_tests.rs b/gateways/kafka/tests/decode_safety_tests.rs new file mode 100644 index 0000000000..f5d8d84a9a --- /dev/null +++ b/gateways/kafka/tests/decode_safety_tests.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Adversarial wire-input tests for #3421 — malformed lengths must return errors, never panic. + +use bytes::Bytes; + +use iggy_gateway_kafka::error::KafkaProtocolError; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder, MAX_COLLECTION_LEN}; +use iggy_gateway_kafka::protocol::requests::decode_produce_request; + +#[test] +fn compact_array_varint_zero_decodes_as_empty_without_panic() { + // Per Kafka spec, compact-array varint=0 means null/absent → 0 elements (not an error). + let mut d = Decoder::new(Bytes::from_static(&[0x00])); + assert_eq!(d.read_compact_array_count().unwrap(), 0); +} + +#[test] +fn negative_i32_array_length_returns_error_not_panic() { + let mut raw = Vec::new(); + raw.extend_from_slice(&(-1_i32).to_be_bytes()); + let mut d = Decoder::new(Bytes::from(raw)); + let err = d.read_i32_array_count().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidArrayLength(-1))); +} + +#[test] +fn i32_array_length_above_max_returns_collection_too_large() { + let mut raw = Vec::new(); + let oversized = i32::try_from(MAX_COLLECTION_LEN + 1).expect("test value fits i32"); + raw.extend_from_slice(&oversized.to_be_bytes()); + let mut d = Decoder::new(Bytes::from(raw)); + let err = d.read_i32_array_count().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::CollectionTooLarge { .. })); +} + +#[test] +fn produce_decoder_rejects_truncated_flexible_body() { + let mut body = Vec::new(); + body.push(0x00); // transactional_id null (compact) + body.extend_from_slice(&1_i16.to_be_bytes()); // acks + body.extend_from_slice(&1000_i32.to_be_bytes()); // timeout + body.push(0x02); // topics compact array: 1 element (varint = count+1) + // truncated before topic name + + let err = decode_produce_request(9, Bytes::from(body)).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn write_nullable_string_rejects_oversized_length() { + let mut enc = Encoder::with_capacity(8); + let long = "x".repeat(i16::MAX as usize + 1); + let err = enc.write_nullable_string(Some(&long)).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::StringTooLong { .. })); +} + +#[test] +fn varint_terminal_byte_with_extra_bits_at_shift_63_is_rejected() { + // Nine continuation bytes then terminal 0x7E at shift 63 (bits 1-6 set, bit 7 clear). + let mut d = Decoder::new(Bytes::from_static(&[ + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x7E, + ])); + let err = d.read_varint().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidVarint)); +} diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index eaf2cf64c0..8a51e45b06 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -20,16 +20,17 @@ //! //! Frame layout written by kafka-tool (all versions): //! [4-byte length prefix] -//! [api_key i16][api_version i16][correlation_id i32] -//! [client_id_len i16][client_id bytes] ← always legacy i16, even for flexible APIs -//! [0x00 tagged-fields byte] ← only for flexible API versions +//! [`api_key` i16][`api_version` i16][`correlation_id` i32] +//! header v1: [`client_id`] `NULLABLE_STRING` +//! header v2: [`client_id`] `COMPACT_NULLABLE_STRING` + request-header tagged fields //! [request body] ← properly encoded per spec (flexible or not) use std::path::PathBuf; use bytes::Bytes; -use iggy_gateway_kafka::protocol::header::request_header_version; +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::header::{RequestHeader, request_header_version}; use iggy_gateway_kafka::protocol::requests::{ decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, decode_produce_request, @@ -45,31 +46,19 @@ fn fixtures_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") } -/// Load a kafka-tool .bin file and return just the request body bytes, correctly -/// skipping the outer Kafka frame header (api_key, api_version, correlation_id, -/// legacy-i16 client_id, and — for flexible versions — the 0x00 tagged-fields byte). +/// Load a kafka-tool `.bin` file and return just the request body bytes. fn load_body(api_key: i16, api_name: &str, version: i16) -> Bytes { - let filename = format!("{:03}_{}_v{}.bin", api_key, api_name, version); + let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); let path = fixtures_dir().join(&filename); let data = std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {filename}: {e}")); - // Skip 4-byte length prefix → frame starts here - let frame = &data[4..]; - - // Bytes 0-7: api_key(2) + api_version(2) + correlation_id(4) - // Bytes 8-9: client_id_len (legacy i16) - let client_id_len = i16::from_be_bytes([frame[8], frame[9]]) as usize; - let body_start_after_client_id = 10 + client_id_len; - - // kafka-tool appends a 0x00 tagged-fields byte for flexible-version APIs - let is_flexible = request_header_version(api_key, version) >= 2; - let body_start = if is_flexible { - body_start_after_client_id + 1 - } else { - body_start_after_client_id - }; - - Bytes::copy_from_slice(&frame[body_start..]) + let frame = Bytes::copy_from_slice(&data[4..]); + let hdr_ver = request_header_version(api_key, version); + let mut decoder = Decoder::new(frame); + RequestHeader::decode_from(&mut decoder, hdr_ver).expect("fixture request header must decode"); + decoder + .read_bytes(decoder.remaining()) + .expect("fixture request body must decode") } // ── Produce (API key 0) ─────────────────────────────────────────────────────── @@ -417,7 +406,7 @@ fn create_topics_response_v2_roundtrip() { } #[test] -fn create_topics_response_v5_has_topic_config_error_code() { +fn create_topics_response_v5_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(19, "CreateTopics", 5); let req = decode_create_topics_request(5, body).unwrap(); @@ -432,11 +421,6 @@ fn create_topics_response_v5_has_topic_config_error_code() { let error_code = d.read_i16().unwrap(); assert_eq!(error_code, 0); let _error_msg = d.read_compact_nullable_string().unwrap(); // v1+ - let topic_config_err = d.read_i16().unwrap(); // v5+: MUST be present - assert_eq!( - topic_config_err, 0, - "v5 must include topic_config_error_code" - ); let num_partitions = d.read_i32().unwrap(); assert_eq!(num_partitions, 1); let replication_factor = d.read_i16().unwrap(); diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index 62b9e96e60..f146215d34 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -28,7 +28,7 @@ fn golden_apiversions_v1_response_fixture() { let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker); // error_code=0, api_count=6 - // key 0 (Produce) min=3 max=9 + // key 0 (Produce) min=0 max=9 (KAFKA-18659 advertise min=0) // key 1 (Fetch) min=4 max=12 // key 2 (ListOffsets) min=1 max=6 // key 3 (Metadata) min=0 max=9 @@ -38,7 +38,7 @@ fn golden_apiversions_v1_response_fixture() { let expected: [u8; 46] = [ 0x00, 0x00, // error_code 0x00, 0x00, 0x00, 0x06, // api count = 6 - 0x00, 0x00, 0x00, 0x03, 0x00, 0x09, // key 0: Produce 3–9 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, // key 0: Produce 0–9 (advertised) 0x00, 0x01, 0x00, 0x04, 0x00, 0x0C, // key 1: Fetch 4–12 0x00, 0x02, 0x00, 0x01, 0x00, 0x06, // key 2: ListOffsets 1–6 0x00, 0x03, 0x00, 0x00, 0x00, 0x09, // key 3: Metadata 0–9 diff --git a/gateways/kafka/tests/handler_regression_tests.rs b/gateways/kafka/tests/handler_regression_tests.rs new file mode 100644 index 0000000000..df569f4c12 --- /dev/null +++ b/gateways/kafka/tests/handler_regression_tests.rs @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Full handler regression — every scoped API key × version through `handle_request`. + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/scope.rs"] +mod scope; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, API_KEY_PRODUCE, ERROR_NONE, + handle_request, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::{fixture_exists, load_fixture_body}; +use scope::{SCOPED_API_KEYS, default_broker}; + +#[test] +fn handle_request_succeeds_for_every_supported_version_with_fixture() { + for &(api_key, name, min_ver, max_ver) in SCOPED_API_KEYS { + if api_key == 3 || api_key == 18 { + // Metadata / ApiVersions: empty body is valid + for version in min_ver..=max_ver { + let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()); + assert!( + !resp.is_empty(), + "{name} v{version} returned empty response" + ); + } + continue; + } + + for version in min_ver..=max_ver { + if !fixture_exists(api_key, name, version) { + continue; + } + let body = load_fixture_body(api_key, name, version); + let resp = handle_request(api_key, version, body, &default_broker()); + assert!( + !resp.is_empty(), + "{name} v{version} returned empty response" + ); + } + } +} + +#[test] +fn produce_stub_response_has_zero_error_per_partition() { + for version in 3i16..=9 { + if !fixture_exists(0, "Produce", version) { + continue; + } + let body = load_fixture_body(0, "Produce", version); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()); + let flexible = version >= 9; + let mut d = Decoder::new(resp); + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "Produce v{version}"); + } +} + +#[test] +fn fetch_stub_response_has_zero_partition_error() { + for version in 4i16..=12 { + if !fixture_exists(1, "Fetch", version) { + continue; + } + let body = load_fixture_body(1, "Fetch", version); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()); + let flexible = version >= 12; + let mut d = Decoder::new(resp); + if version >= 1 { + let _throttle = d.read_i32().unwrap(); + } + if version >= 7 { + assert_eq!(d.read_i16().unwrap(), ERROR_NONE); + let _session = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NONE, + "Fetch v{version} partition error" + ); + } +} + +#[test] +fn list_offsets_stub_response_has_zero_error() { + for version in 1i16..=6 { + if !fixture_exists(2, "ListOffsets", version) { + continue; + } + let body = load_fixture_body(2, "ListOffsets", version); + let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()); + let flexible = version >= 6; + let mut d = Decoder::new(resp); + if version >= 2 { + let _throttle = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "ListOffsets v{version}"); + } +} + +#[test] +fn create_topics_stub_response_has_zero_error() { + for version in 2i16..=5 { + if !fixture_exists(19, "CreateTopics", version) { + continue; + } + let body = load_fixture_body(19, "CreateTopics", version); + let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()); + let flexible = version >= 5; + let mut d = Decoder::new(resp); + if version >= 2 { + let _throttle = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + } + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "CreateTopics v{version}"); + } +} diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs index e88efe4c22..6ea323fdfe 100644 --- a/gateways/kafka/tests/header_tests.rs +++ b/gateways/kafka/tests/header_tests.rs @@ -130,6 +130,17 @@ fn response_header_version_apiversions_always_zero() { assert_eq!(response_header_version(18, 3), 0); // even flexible request → v0 response } +#[test] +fn share_group_api_keys_use_flexible_header_from_v0() { + for key in [77, 78, 79, 80] { + assert_eq!( + request_header_version(key, 0), + 2, + "api_key {key} must use flexible header v2" + ); + } +} + #[test] fn response_header_version_flexible_non_apiversions() { // Metadata v9+ is flexible → response header v1 diff --git a/gateways/kafka/tests/metadata_regression_tests.rs b/gateways/kafka/tests/metadata_regression_tests.rs new file mode 100644 index 0000000000..50c91bb1c4 --- /dev/null +++ b/gateways/kafka/tests/metadata_regression_tests.rs @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Metadata API regression — all supported versions, broker advertise, topic counts. + +#[path = "common/scope.rs"] +mod scope; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_METADATA, BrokerAdvertise, ERROR_UNKNOWN_TOPIC_OR_PARTITION, handle_request, +}; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +use scope::default_broker; + +fn metadata_request_legacy(topic_count: i32) -> Bytes { + let mut enc = Encoder::with_capacity(8); + enc.write_i32(topic_count); + enc.freeze() +} + +fn metadata_request_flexible(topic_count: usize) -> Bytes { + let mut enc = Encoder::with_capacity(8); + enc.write_varint((topic_count + 1) as u64); + enc.freeze() +} + +fn read_broker_legacy(d: &mut Decoder) -> (String, i32) { + let count = d.read_i32().unwrap(); + assert_eq!(count, 1); + let _node = d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + (host, port) +} + +fn read_broker_flexible(d: &mut Decoder) -> (String, i32) { + let count_plus_one = d.read_varint().unwrap(); + assert_eq!(count_plus_one, 2); // one broker + let _node = d.read_i32().unwrap(); + let host = d.read_compact_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + let _rack = d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + (host, port) +} + +#[test] +fn metadata_v0_empty_topics_stub_broker() { + let body = handle_request( + API_KEY_METADATA, + 0, + metadata_request_legacy(0), + &default_broker(), + ); + let mut d = Decoder::new(body); + let (host, port) = read_broker_legacy(&mut d); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 9093); + assert_eq!(d.read_i32().unwrap(), 0); +} + +#[test] +fn metadata_v0_three_topics_each_unknown() { + let body = handle_request( + API_KEY_METADATA, + 0, + metadata_request_legacy(3), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + assert_eq!(d.read_i32().unwrap(), 3); + for _ in 0..3 { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!(d.read_nullable_string().unwrap().unwrap(), "unknown-topic"); + assert_eq!(d.read_i32().unwrap(), 0); + } +} + +#[test] +fn metadata_v1_includes_controller_id() { + let body = handle_request( + API_KEY_METADATA, + 1, + metadata_request_legacy(0), + &default_broker(), + ); + let mut d = Decoder::new(body); + // Metadata v1 has no throttle_time_ms (added in v3). + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let controller = d.read_i32().unwrap(); + assert_eq!(controller, 1); +} + +#[test] +fn metadata_v2_includes_cluster_id_field() { + let body = handle_request( + API_KEY_METADATA, + 2, + metadata_request_legacy(0), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let _cluster_id = d.read_nullable_string().unwrap(); + let _controller = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); +} + +#[test] +fn metadata_all_legacy_versions_produce_valid_response() { + for version in 0i16..=8 { + let body = handle_request( + API_KEY_METADATA, + version, + metadata_request_legacy(1), + &default_broker(), + ); + let mut d = Decoder::new(body); + if version >= 3 { + let _throttle = d.read_i32().unwrap(); + } + let _ = read_broker_legacy(&mut d); + if version >= 1 { + let _rack = d.read_nullable_string().unwrap(); + } + if version >= 2 { + let _cluster = d.read_nullable_string().unwrap(); + } + if version >= 1 { + let _controller = d.read_i32().unwrap(); + } + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + } +} + +#[test] +fn metadata_v9_flexible_encoding() { + let body = handle_request( + API_KEY_METADATA, + 9, + metadata_request_flexible(2), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _throttle = d.read_i32().unwrap(); + let (host, port) = read_broker_flexible(&mut d); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 9093); + let _cluster = d.read_compact_nullable_string().unwrap(); + let controller = d.read_i32().unwrap(); + assert_eq!(controller, 1); + + let topics_plus_one = d.read_varint().unwrap(); + assert_eq!(topics_plus_one, 3); // 2 topics + for _ in 0..2 { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!( + d.read_compact_nullable_string().unwrap().unwrap(), + "unknown-topic" + ); + let _internal = d.read_bool().unwrap(); + let parts_plus_one = d.read_varint().unwrap(); + assert_eq!(parts_plus_one, 1); // empty partitions + assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations (v8+) + d.read_tagged_fields().unwrap(); + } + assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations (v8+) + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v8_includes_authorized_operations_legacy() { + let body = handle_request( + API_KEY_METADATA, + 8, + metadata_request_legacy(1), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _throttle = d.read_i32().unwrap(); + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let _cluster = d.read_nullable_string().unwrap(); + let _controller = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + let _topic_error = d.read_i16().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _internal = d.read_bool().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); // empty partitions + assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations + assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_uses_custom_broker_advertise() { + let broker = BrokerAdvertise { + host: "10.0.0.42".to_string(), + port: 29093, + }; + let body = handle_request(API_KEY_METADATA, 0, metadata_request_legacy(0), &broker); + let mut d = Decoder::new(body); + let (host, port) = read_broker_legacy(&mut d); + assert_eq!(host, "10.0.0.42"); + assert_eq!(port, 29093); +} diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs new file mode 100644 index 0000000000..d0846ba49f --- /dev/null +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end TCP tests through `KafkaServer` (full request/response cycle). + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/server.rs"] +mod server; +#[path = "common/tcp.rs"] +mod tcp; + +use bytes::{BufMut, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_METADATA, API_KEY_PRODUCE, ERROR_UNSUPPORTED_VERSION, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::load_fixture_body; +use server::spawn_test_server; +use tcp::{build_request_frame, parse_response_payload, read_response_frame, round_trip}; + +#[tokio::test] +async fn e2e_apiversions_v1_preserves_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 1, 42_001, &[]).await; + assert_eq!(corr, 42_001); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); +} + +#[tokio::test] +async fn e2e_apiversions_v3_flexible_preserves_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 3, 42_002, &[]).await; + assert_eq!(corr, 42_002); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); + assert_eq!(count, 6); +} + +#[tokio::test] +async fn e2e_metadata_v0_returns_stub_broker() { + let (addr, _shutdown) = spawn_test_server().await; + let mut req = BytesMut::new(); + req.put_i32(0); // empty topics + let (corr, body) = round_trip(addr, API_KEY_METADATA, 0, 77, &req).await; + assert_eq!(corr, 77); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(host, "127.0.0.1"); +} + +#[tokio::test] +async fn e2e_produce_v3_round_trip_with_fixture() { + let (addr, _shutdown) = spawn_test_server().await; + let body = load_fixture_body(0, "Produce", 3); + let (corr, resp_body) = round_trip(addr, API_KEY_PRODUCE, 3, 88, &body).await; + assert_eq!(corr, 88); + assert!(!resp_body.is_empty()); +} + +#[tokio::test] +async fn e2e_unsupported_api_key_returns_error_without_disconnect() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + + let frame1 = build_request_frame(8, 2, 99, Some("e2e-test"), &[]); + stream.write_all(&frame1).await.unwrap(); + let payload1 = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr, body) = parse_response_payload(8, 2, payload1); + assert_eq!(corr, 99); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + + // Second request on same connection must still work. + let frame2 = build_request_frame(API_KEY_API_VERSIONS, 1, 100, Some("e2e-test"), &[]); + stream.write_all(&frame2).await.unwrap(); + let payload2 = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr2, body2) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload2); + assert_eq!(corr2, 100); + let mut d2 = Decoder::new(body2); + assert_eq!(d2.read_i16().unwrap(), 0); +} + +#[tokio::test] +async fn e2e_sequential_requests_on_one_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + + let requests = [(API_KEY_API_VERSIONS, 1i16), (API_KEY_METADATA, 0i16)]; + for (i, (key, ver)) in requests.iter().enumerate() { + let meta_body = { + let mut b = BytesMut::new(); + b.put_i32(0); + b + }; + let body: &[u8] = if *key == API_KEY_METADATA { + &meta_body + } else { + &[] + }; + let correlation_id = 1000 + i32::try_from(i).expect("test index fits i32"); + let frame = build_request_frame(*key, *ver, correlation_id, Some("seq-test"), body); + stream.write_all(&frame).await.unwrap(); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr, _) = parse_response_payload(*key, *ver, payload); + assert_eq!(corr, correlation_id); + } +} + +#[tokio::test] +async fn e2e_negative_frame_length_closes_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream.write_all(&(-1i32).to_be_bytes()).await.unwrap(); + + let mut buf = [0u8; 1]; + let n = stream.read(&mut buf).await.unwrap_or(0); + assert_eq!(n, 0, "server should close after invalid frame length"); +} + +#[tokio::test] +async fn e2e_oversized_frame_is_rejected() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + + let mut frame = BytesMut::new(); + frame.put_i32(10_000_000); // exceeds default 8 MiB cap + frame.resize(4 + 100, 0); + stream.write_all(&frame).await.unwrap(); + + let mut buf = [0u8; 1]; + let n = stream.read(&mut buf).await.unwrap_or(0); + assert_eq!(n, 0, "server should close after oversized frame"); +} diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs index 35bc4ff113..9de8f9b092 100644 --- a/gateways/kafka/tests/server_integration_tests.rs +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -17,12 +17,12 @@ use std::time::Duration; -use bytes::{Buf, BytesMut}; +use bytes::{Buf, BufMut, BytesMut}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use iggy_gateway_kafka::protocol::codec::Encoder; -use iggy_gateway_kafka::server::{read_frame, write_frame}; +use iggy_gateway_kafka::server::read_frame; async fn tcp_pair() -> (TcpStream, TcpStream) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -33,6 +33,23 @@ async fn tcp_pair() -> (TcpStream, TcpStream) { (client, server) } +/// Raw length-prefixed write (no Kafka response header) — mirrors `server::write_frame`. +async fn write_length_prefixed( + stream: &mut TcpStream, + payload: &[u8], + write_timeout: Duration, +) -> Result<(), Box> { + let len = payload.len(); + assert!(i32::try_from(len).is_ok()); + let mut frame = BytesMut::with_capacity(4 + len); + frame.put_i32(i32::try_from(len).expect("len fits i32")); + frame.extend_from_slice(payload); + tokio::time::timeout(write_timeout, stream.write_all(&frame)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timeout"))??; + Ok(()) +} + #[tokio::test] async fn read_frame_reads_valid_payload() { let (mut client, mut server) = tcp_pair().await; @@ -45,7 +62,11 @@ async fn read_frame_reads_valid_payload() { let payload = enc.freeze(); let mut frame = BytesMut::with_capacity(4 + payload.len()); - frame.extend_from_slice(&(payload.len() as i32).to_be_bytes()); + frame.extend_from_slice( + &i32::try_from(payload.len()) + .expect("test payload fits i32") + .to_be_bytes(), + ); frame.extend_from_slice(&payload); client.write_all(&frame).await.unwrap(); @@ -59,13 +80,13 @@ async fn read_frame_reads_valid_payload() { async fn write_frame_writes_length_prefixed_payload() { let (mut client, mut server) = tcp_pair().await; let payload = b"abc123"; - write_frame(&mut server, payload, Duration::from_secs(1)) + write_length_prefixed(&mut server, payload, Duration::from_secs(1)) .await .unwrap(); let mut len = [0u8; 4]; client.read_exact(&mut len).await.unwrap(); - let len = i32::from_be_bytes(len) as usize; + let len = usize::try_from(i32::from_be_bytes(len)).expect("positive frame length"); assert_eq!(len, payload.len()); let mut body = vec![0u8; len]; @@ -97,7 +118,7 @@ async fn read_frame_rejects_invalid_lengths() { #[tokio::test] async fn write_frame_length_prefix_is_big_endian() { let (mut client, mut server) = tcp_pair().await; - write_frame(&mut server, &[1, 2, 3, 4], Duration::from_secs(1)) + write_length_prefixed(&mut server, &[1, 2, 3, 4], Duration::from_secs(1)) .await .unwrap(); diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs new file mode 100644 index 0000000000..de0f96e03b --- /dev/null +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -0,0 +1,309 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Version negotiation firewall — boundary tests for every scoped API key. + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/scope.rs"] +mod scope; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, + API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, ERROR_UNSUPPORTED_VERSION, + advertised_min_version, handle_request, is_supported_version, supported_api_ranges, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::load_fixture_body; +use scope::{SCOPED_API_KEYS, default_broker}; + +#[test] +fn supported_ranges_table_has_six_entries() { + assert_eq!(supported_api_ranges().len(), 6); +} + +#[test] +fn is_supported_version_matches_scope_table() { + for &(api_key, _, min_ver, max_ver) in SCOPED_API_KEYS { + assert!( + !is_supported_version(api_key, min_ver - 1), + "key {api_key} must reject v{}", + min_ver - 1 + ); + assert!( + is_supported_version(api_key, min_ver), + "key {api_key} must accept min v{min_ver}" + ); + assert!( + is_supported_version(api_key, max_ver), + "key {api_key} must accept max v{max_ver}" + ); + assert!( + !is_supported_version(api_key, max_ver + 1), + "key {api_key} must reject v{}", + max_ver + 1 + ); + } +} + +#[test] +fn apiversions_advertises_exact_supported_ranges_v1() { + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_i32().unwrap()).expect("api count fits usize"); + assert_eq!(count, supported_api_ranges().len()); + + for expected in supported_api_ranges() { + let key = d.read_i16().unwrap(); + let min = d.read_i16().unwrap(); + let max = d.read_i16().unwrap(); + assert_eq!(key, expected.api_key); + assert_eq!( + min, + advertised_min_version(expected.api_key, expected.min_version) + ); + assert_eq!(max, expected.max_version); + } + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.remaining(), 0); +} + +#[test] +fn apiversions_advertises_exact_supported_ranges_v3_flexible() { + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); + assert_eq!(count, supported_api_ranges().len()); + + for expected in supported_api_ranges() { + let key = d.read_i16().unwrap(); + let min = d.read_i16().unwrap(); + let max = d.read_i16().unwrap(); + d.read_tagged_fields().unwrap(); + assert_eq!(key, expected.api_key); + assert_eq!( + min, + advertised_min_version(expected.api_key, expected.min_version) + ); + assert_eq!(max, expected.max_version); + } + assert_eq!(d.read_i32().unwrap(), 0); + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn apiversions_advertises_produce_min_zero_while_firewall_stays_three() { + let range = supported_api_ranges() + .iter() + .find(|r| r.api_key == API_KEY_PRODUCE) + .expect("produce range"); + assert_eq!(range.min_version, 3); + assert_eq!( + advertised_min_version(API_KEY_PRODUCE, range.min_version), + 0 + ); + assert!(!is_supported_version(API_KEY_PRODUCE, 0)); +} + +#[test] +fn apiversions_all_versions_return_success() { + for version in 0i16..=3 { + let body = handle_request( + API_KEY_API_VERSIONS, + version, + Bytes::new(), + &default_broker(), + ); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0, "ApiVersions v{version}"); + } +} + +#[test] +fn apiversions_out_of_range_returns_unsupported_in_body() { + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +fn metadata_request_one_topic() -> Bytes { + let mut raw = Vec::new(); + raw.extend_from_slice(&1_i32.to_be_bytes()); + Bytes::from(raw) +} + +#[test] +fn metadata_below_min_version_returns_topic_error() { + let body = handle_request( + API_KEY_METADATA, + -1, + metadata_request_one_topic(), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _brokers = d.read_i32().unwrap(); + let _ = d.read_i32().unwrap(); + let _ = d.read_nullable_string().unwrap(); + let _ = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); // mirrors request topic count + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn metadata_above_max_version_returns_topic_error() { + let body = handle_request( + API_KEY_METADATA, + 10, + metadata_request_one_topic(), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _brokers = d.read_i32().unwrap(); + let _ = d.read_i32().unwrap(); + let _ = d.read_nullable_string().unwrap(); + let _ = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn produce_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + let _ = d.read_i64().unwrap(); + let _ = d.read_i64().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn fetch_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i64().unwrap(), 0); + assert_eq!(d.read_nullable_bytes().unwrap(), None); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn fetch_unsupported_version_above_max_uses_top_level_error() { + let body = handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_varint().unwrap(), 1); + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn list_offsets_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i64().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn create_topics_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_nullable_string().unwrap(), None); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn unsupported_api_keys_return_error_only() { + for key in [8, 9, 10, 11, 17, 20, 42, 999] { + let body = handle_request(key, 0, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "api_key {key}" + ); + } +} + +#[test] +fn supported_produce_versions_accept_valid_fixture() { + for version in 3i16..=9 { + let body = load_fixture_body(0, "Produce", version); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()); + assert!(!resp.is_empty(), "Produce v{version} response empty"); + } +} + +#[test] +fn supported_fetch_versions_accept_valid_fixture() { + for version in 4i16..=12 { + let body = load_fixture_body(1, "Fetch", version); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()); + assert!(!resp.is_empty(), "Fetch v{version} response empty"); + } +} + +#[test] +fn corrupt_produce_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +#[test] +fn corrupt_fetch_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_FETCH, 4, body, &default_broker()); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index e1b29f08b7..b92bf729c1 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -25,6 +25,8 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; use tracing::{info, warn}; +mod response; + #[derive(Parser)] #[command( name = "kafka-message-gen", @@ -65,13 +67,30 @@ enum Command { version: Option, #[arg(long, default_value = "5000")] timeout_ms: u64, + /// Compact one-line output (default is verbose decoded response) + #[arg(long)] + quiet: bool, }, - /// Send all messages and report pass/fail — exit code 1 if any fail Verify { #[arg(long, default_value = "127.0.0.1:9092")] host: String, + /// Limit to these API keys (repeatable). Defaults to all gateway-scoped keys. + #[arg(long, action = clap::ArgAction::Append)] + api_key: Vec, + /// Limit to a single protocol version + #[arg(long)] + version: Option, + #[arg(long, default_value = "5000")] + timeout_ms: u64, + /// Stop on the first failure #[arg(long)] fail_fast: bool, + /// Use the full Kafka 4.1 registry (for real brokers), not the Iggy gateway scope + #[arg(long)] + all_apis: bool, + /// Compact one-line output (default is verbose decoded response) + #[arg(long)] + quiet: bool, }, } @@ -145,6 +164,16 @@ const API_REGISTRY: &[(i16, &str, i16, i16)] = &[ (76, "ListClientMetricsResources", 0, 0), ]; +/// Iggy Kafka gateway #3421 scope — mirrors `SUPPORTED_RANGES` in `iggy_gateway_kafka`. +const GATEWAY_REGISTRY: &[(i16, &str, i16, i16)] = &[ + (0, "Produce", 3, 9), + (1, "Fetch", 4, 12), + (2, "ListOffsets", 1, 6), + (3, "Metadata", 0, 9), + (18, "ApiVersions", 0, 3), + (19, "CreateTopics", 2, 5), +]; + // ── Flexible version table ──────────────────────────────────────────────────── // Source: flexibleVersions field in each Kafka JSON schema. // Returns the first version using compact encoding, or None if never flexible. @@ -224,10 +253,34 @@ fn first_flexible_version(api_key: i16) -> Option { // [api_key: i16] // [api_version: i16] // [correlation_id: i32] -// [client_id_len: i16] -1 = null -// [client_id: bytes] -// [tagged_fields: u8(0)] only present for flexible versions +// header v1: [client_id: NULLABLE_STRING] +// header v2: [client_id: COMPACT_NULLABLE_STRING] [request_header_tagged_fields] // [payload: bytes] + +fn write_unsigned_varint(buf: &mut BytesMut, mut value: u64) { + loop { + let mut byte = (value & 0x7F) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + buf.put_u8(byte); + if value == 0 { + break; + } + } +} + +fn write_compact_nullable_string(buf: &mut BytesMut, value: Option<&str>) { + match value { + None => write_unsigned_varint(buf, 0), + Some(s) => { + write_unsigned_varint(buf, (s.len() + 1) as u64); + buf.put_slice(s.as_bytes()); + } + } +} + fn frame_request( api_key: i16, api_version: i16, @@ -236,19 +289,22 @@ fn frame_request( payload: &[u8], flexible: bool, ) -> Bytes { - let cid = client_id.as_bytes(); - let hlen = 2 + 2 + 4 + 2 + cid.len() + if flexible { 1 } else { 0 }; - let blen = hlen + payload.len(); - let mut buf = BytesMut::with_capacity(4 + blen); - buf.put_i32(blen as i32); - buf.put_i16(api_key); - buf.put_i16(api_version); - buf.put_i32(correlation_id); - buf.put_i16(cid.len() as i16); - buf.put_slice(cid); + let mut header = BytesMut::new(); + header.put_i16(api_key); + header.put_i16(api_version); + header.put_i32(correlation_id); if flexible { - buf.put_u8(0x00); + write_compact_nullable_string(&mut header, Some(client_id)); + header.put_u8(0); // empty request-header tagged fields + } else { + header.put_i16(i16::try_from(client_id.len()).expect("client_id fits i16")); + header.put_slice(client_id.as_bytes()); } + + let blen = header.len() + payload.len(); + let mut buf = BytesMut::with_capacity(4 + blen); + buf.put_i32(i32::try_from(blen).expect("frame fits i32")); + buf.put_slice(&header); buf.put_slice(payload); buf.freeze() } @@ -669,23 +725,53 @@ async fn cmd_generate( Ok(()) } +async fn connect(host: &str) -> Result { + TcpStream::connect(host) + .await + .with_context(|| format!("Cannot connect to {host}")) +} + +async fn read_kafka_response(stream: &mut TcpStream) -> std::io::Result> { + let mut lb = [0u8; 4]; + stream.read_exact(&mut lb).await?; + let frame_len = i32::from_be_bytes(lb); + if frame_len <= 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid response frame length: {frame_len}"), + )); + } + let mut body = vec![ + 0u8; + usize::try_from(frame_len).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "response frame length does not fit usize", + ) + })? + ]; + stream.read_exact(&mut body).await?; + Ok(body) +} + async fn run_send( host: &str, - fk: Option, + registry: &[(i16, &str, i16, i16)], + filter_keys: &[i16], fv: Option, toms: u64, + fail_fast: bool, + quiet: bool, ) -> Result<(usize, usize)> { - let mut stream = TcpStream::connect(host) - .await - .with_context(|| format!("Cannot connect to {host}"))?; + let mut stream = connect(host).await?; info!("Connected to {host}"); let (mut ok, mut fail, mut corr) = (0usize, 0usize, 1i32); - for &(ak, name, min, max) in API_REGISTRY { - if fk.is_some_and(|k| k != ak) { + 'outer: for &(ak, name, min, max) in registry { + if !filter_keys.is_empty() && !filter_keys.contains(&ak) { continue; } for v in min..=max { - if fv.is_some_and(|fv| fv != v) { + if fv.is_some_and(|wanted| wanted != v) { continue; } let msg = match build_framed(ak, v, corr) { @@ -693,39 +779,50 @@ async fn run_send( Err(e) => { warn!("Build {} v{}: {e}", name, v); fail += 1; + if fail_fast { + break 'outer; + } continue; } }; - stream - .write_all(&msg) - .await - .with_context(|| format!("Write {} v{}", name, v))?; - let res = tokio::time::timeout(std::time::Duration::from_millis(toms), async { - let mut lb = [0u8; 4]; - stream.read_exact(&mut lb).await?; - let mut body = vec![0u8; i32::from_be_bytes(lb) as usize]; - stream.read_exact(&mut body).await?; - Ok::, std::io::Error>(body) - }) + if let Err(e) = stream.write_all(&msg).await { + println!("✗ {name} v{v} → write error: {e}"); + fail += 1; + stream = connect(host).await?; + if fail_fast { + break 'outer; + } + corr += 1; + continue; + } + + let res = tokio::time::timeout( + std::time::Duration::from_millis(toms), + read_kafka_response(&mut stream), + ) .await; + match res { Ok(Ok(r)) => { - let ec = if r.len() >= 6 { - i16::from_be_bytes(r[4..6].try_into().unwrap()) - } else { - -1 - }; - let sym = if ec <= 0 { "✓" } else { "⚠" }; - println!("{sym} {} v{} → {} bytes ec={ec}", name, v, r.len()); + let summary = response::analyze_response(ak, v, corr, &r); + summary.print(name, v, quiet); ok += 1; } Ok(Err(e)) => { - println!("✗ {} v{} → IO error: {e}", name, v); + println!("✗ {name} v{v} → IO error: {e}"); fail += 1; + stream = connect(host).await?; + if fail_fast { + break 'outer; + } } Err(_) => { - println!("✗ {} v{} → timeout ({}ms)", name, v, toms); + println!("✗ {name} v{v} → timeout ({toms}ms)"); fail += 1; + stream = connect(host).await?; + if fail_fast { + break 'outer; + } } } corr += 1; @@ -754,12 +851,39 @@ async fn main() -> Result<()> { api_key, version, timeout_ms, + quiet, } => { - let (ok, fail) = run_send(&host, api_key, version, timeout_ms).await?; + let filter_keys: Vec = api_key.into_iter().collect(); + let (ok, fail) = run_send( + &host, + API_REGISTRY, + &filter_keys, + version, + timeout_ms, + false, + quiet, + ) + .await?; println!("\nResult: {ok} OK {fail} failed"); } - Command::Verify { host, .. } => { - let (ok, fail) = run_send(&host, None, None, 5000).await?; + Command::Verify { + host, + api_key, + version, + timeout_ms, + fail_fast, + all_apis, + quiet, + } => { + let registry = if all_apis { + API_REGISTRY + } else { + GATEWAY_REGISTRY + }; + let (ok, fail) = run_send( + &host, registry, &api_key, version, timeout_ms, fail_fast, quiet, + ) + .await?; println!("\n=== Verify: {ok} passed {fail} failed ==="); if fail > 0 { std::process::exit(1); diff --git a/gateways/kafka/tools/kafka-tool/src/response.rs b/gateways/kafka/tools/kafka-tool/src/response.rs new file mode 100644 index 0000000000..a55f5b62a8 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/src/response.rs @@ -0,0 +1,420 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka response frame parsing and human-readable summaries for `send` / `verify`. + +use bytes::Bytes; +use kafka_protocol::messages::{ + ApiVersionsResponse, CreateTopicsResponse, FetchResponse, ListOffsetsResponse, + MetadataResponse, ProduceResponse, +}; +use kafka_protocol::protocol::Decodable; + +/// Parsed view of one length-prefixed Kafka response payload (excluding the 4-byte frame length). +pub struct ResponseSummary { + pub frame_bytes: usize, + pub correlation_id: i32, + pub response_header_version: i16, + pub correlation_match: bool, + /// Highest-severity non-zero error found, or `0` when all decoded codes are zero. + pub primary_error_code: i16, + pub details: Vec, + pub decode_note: Option, +} + +impl ResponseSummary { + #[must_use] + pub fn has_nonzero_error(&self) -> bool { + self.primary_error_code != 0 + } + + pub fn print(&self, api_name: &str, version: i16, quiet: bool) { + let sym = if self.has_nonzero_error() { + "⚠" + } else { + "✓" + }; + let ec_label = format_error_code(self.primary_error_code); + let corr = if self.correlation_match { + format!("{}", self.correlation_id) + } else { + format!("{} (expected correlation mismatch)", self.correlation_id) + }; + + if quiet { + println!( + "{sym} {api_name} v{version} → {}B ec={} ({ec_label})", + self.frame_bytes, self.primary_error_code + ); + return; + } + + println!( + "{sym} {api_name} v{version} frame={}B correlation={corr} resp_hdr=v{} primary_ec={} ({ec_label})", + self.frame_bytes, self.response_header_version, self.primary_error_code + ); + for line in &self.details { + println!(" {line}"); + } + if let Some(note) = &self.decode_note { + println!(" note: {note}"); + } + } +} + +/// Analyze a response payload for the given request `(api_key, api_version)`. +pub fn analyze_response( + api_key: i16, + api_version: i16, + request_correlation_id: i32, + payload: &[u8], +) -> ResponseSummary { + let frame_bytes = payload.len(); + if payload.len() < 4 { + return ResponseSummary { + frame_bytes, + correlation_id: 0, + response_header_version: 0, + correlation_match: false, + primary_error_code: -1, + details: vec!["payload shorter than correlation_id".into()], + decode_note: Some("truncated response".into()), + }; + } + + let correlation_id = i32::from_be_bytes(payload[0..4].try_into().expect("4 bytes")); + let resp_hdr_ver = response_header_version(api_key, api_version); + let body_start = if resp_hdr_ver >= 1 { + 5 // correlation_id + empty tagged fields (0x00) + } else { + 4 + }; + + if payload.len() < body_start { + return ResponseSummary { + frame_bytes, + correlation_id, + response_header_version: resp_hdr_ver, + correlation_match: correlation_id == request_correlation_id, + primary_error_code: -1, + details: vec![format!( + "truncated after correlation (need {body_start} bytes)" + )], + decode_note: None, + }; + } + + let body = &payload[body_start..]; + let mut details = Vec::new(); + let mut codes = Vec::new(); + let mut decode_note = None; + + if body.len() == 2 { + let ec = i16::from_be_bytes(body.try_into().expect("2 bytes")); + codes.push(ec); + details.push(format!( + "error-only body: error_code={ec} ({})", + format_error_code(ec) + )); + } else { + match decode_body(api_key, api_version, body, &mut details, &mut codes) { + Ok(()) => {} + Err(e) => { + decode_note = Some(format!("schema decode failed: {e:#}")); + details.push(format!("raw_body_hex={}", hex::encode(body))); + } + } + } + + let primary_error_code = codes.iter().copied().filter(|&c| c != 0).max().unwrap_or(0); + + ResponseSummary { + frame_bytes, + correlation_id, + response_header_version: resp_hdr_ver, + correlation_match: correlation_id == request_correlation_id, + primary_error_code, + details, + decode_note, + } +} + +fn optional_topic_name(name: &Option) -> String { + name.as_ref() + .map(|n| n.0.as_str().to_string()) + .unwrap_or_else(|| "".into()) +} + +fn topic_name(name: &kafka_protocol::messages::TopicName) -> String { + name.0.as_str().to_string() +} + +fn decode_body( + api_key: i16, + api_version: i16, + body: &[u8], + details: &mut Vec, + codes: &mut Vec, +) -> anyhow::Result<()> { + let mut buf = Bytes::copy_from_slice(body); + match api_key { + 18 => { + let resp = ApiVersionsResponse::decode(&mut buf, api_version)?; + codes.push(resp.error_code); + details.push(format!( + "top_level.error_code={} ({})", + resp.error_code, + format_error_code(resp.error_code) + )); + details.push(format!("api_keys={}", resp.api_keys.len())); + if api_version >= 1 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + for (i, k) in resp.api_keys.iter().enumerate().take(8) { + details.push(format!( + "api_keys[{i}]: key={} min={} max={}", + k.api_key, k.min_version, k.max_version + )); + } + if resp.api_keys.len() > 8 { + details.push(format!("… {} more api_keys", resp.api_keys.len() - 8)); + } + } + 3 => { + let resp = MetadataResponse::decode(&mut buf, api_version)?; + if api_version >= 3 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + details.push(format!("brokers={}", resp.brokers.len())); + if let Some(b) = resp.brokers.first() { + details.push(format!( + "brokers[0]: id={} host={} port={}", + b.node_id.0, b.host, b.port + )); + } + details.push(format!("topics={}", resp.topics.len())); + for (i, t) in resp.topics.iter().enumerate().take(4) { + codes.push(t.error_code); + let name = optional_topic_name(&t.name); + details.push(format!( + "topics[{i}]: name={name} ec={} ({}) partitions={}", + t.error_code, + format_error_code(t.error_code), + t.partitions.len() + )); + } + if resp.topics.len() > 4 { + details.push(format!("… {} more topics", resp.topics.len() - 4)); + } + } + 0 => { + let resp = ProduceResponse::decode(&mut buf, api_version)?; + if api_version >= 1 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + details.push(format!("topics={}", resp.responses.len())); + for (ti, topic) in resp.responses.iter().enumerate().take(4) { + let name = topic_name(&topic.name); + details.push(format!( + "topics[{ti}]: name={name} partitions={}", + topic.partition_responses.len() + )); + for (pi, p) in topic.partition_responses.iter().enumerate().take(4) { + codes.push(p.error_code); + details.push(format!( + " partitions[{pi}]: index={} ec={} ({}) offset={}", + p.index, + p.error_code, + format_error_code(p.error_code), + p.base_offset + )); + } + } + } + 1 => { + let resp = FetchResponse::decode(&mut buf, api_version)?; + if api_version >= 1 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + if api_version >= 7 { + codes.push(resp.error_code); + details.push(format!( + "top_level.error_code={} ({}) session_id={}", + resp.error_code, + format_error_code(resp.error_code), + resp.session_id + )); + } + details.push(format!("topics={}", resp.responses.len())); + for (ti, topic) in resp.responses.iter().enumerate().take(4) { + let name = topic_name(&topic.topic); + details.push(format!( + "topics[{ti}]: name={name} partitions={}", + topic.partitions.len() + )); + for (pi, p) in topic.partitions.iter().enumerate().take(4) { + codes.push(p.error_code); + details.push(format!( + " partitions[{pi}]: index={} ec={} ({}) hw={}", + p.partition_index, + p.error_code, + format_error_code(p.error_code), + p.high_watermark + )); + } + } + } + 2 => { + let resp = ListOffsetsResponse::decode(&mut buf, api_version)?; + if api_version >= 2 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + details.push(format!("topics={}", resp.topics.len())); + for (ti, topic) in resp.topics.iter().enumerate().take(4) { + let name = topic_name(&topic.name); + details.push(format!( + "topics[{ti}]: name={name} partitions={}", + topic.partitions.len() + )); + for (pi, p) in topic.partitions.iter().enumerate().take(4) { + codes.push(p.error_code); + details.push(format!( + " partitions[{pi}]: index={} ec={} ({}) offset={}", + p.partition_index, + p.error_code, + format_error_code(p.error_code), + p.offset + )); + } + } + } + 19 => { + let resp = CreateTopicsResponse::decode(&mut buf, api_version)?; + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + details.push(format!("topics={}", resp.topics.len())); + for (i, t) in resp.topics.iter().enumerate().take(4) { + codes.push(t.error_code); + let name = topic_name(&t.name); + details.push(format!( + "topics[{i}]: name={name} ec={} ({})", + t.error_code, + format_error_code(t.error_code) + )); + } + } + other => { + details.push(format!("no schema decoder for api_key={other}")); + if body.len() >= 2 { + let ec = i16::from_be_bytes(body[0..2].try_into().expect("2 bytes")); + codes.push(ec); + details.push(format!( + "body[0..2] as i16={ec} ({}) — may not be top-level error_code", + format_error_code(ec) + )); + } + } + } + Ok(()) +} + +fn format_error_code(code: i16) -> &'static str { + match code { + 0 => "NONE", + 1 => "OFFSET_OUT_OF_RANGE", + 2 => "CORRUPT_MESSAGE", + 3 => "UNKNOWN_TOPIC_OR_PARTITION", + 35 => "UNSUPPORTED_VERSION", + 36 => "TOPIC_ALREADY_EXISTS", + 37 => "INVALID_PARTITIONS", + 42 => "INVALID_REQUEST", + -1 => "UNKNOWN", + _ => "OTHER", + } +} + +fn request_header_version(api_key: i16, api_version: i16) -> i16 { + let flex_from = first_flexible_version(api_key); + match flex_from { + Some(fv) if api_version >= fv => 2, + _ => 1, + } +} + +fn response_header_version(api_key: i16, api_version: i16) -> i16 { + if api_key == 18 { + return 0; + } + if request_header_version(api_key, api_version) >= 2 { + 1 + } else { + 0 + } +} + +/// First flexible protocol version per API key (matches gateway `header.rs` / kafka-tool framing). +fn first_flexible_version(api_key: i16) -> Option { + match api_key { + 0 => Some(9), + 1 => Some(12), + 2 => Some(6), + 3 => Some(9), + 8 => Some(8), + 9 => Some(6), + 10 => Some(3), + 11 => Some(6), + 12 => Some(4), + 13 => Some(4), + 14 => Some(4), + 15 => Some(5), + 16 => Some(3), + 17 => None, + 18 => Some(3), + 19 => Some(5), + 20 => Some(4), + 21 => Some(2), + 22 => Some(2), + 23 => Some(4), + 24 => Some(3), + 25 => Some(3), + 26 => Some(3), + 27 => Some(1), + 28 => Some(3), + 29 => Some(2), + 30 => Some(2), + 31 => Some(2), + 32 => Some(4), + 33 => Some(2), + 34 => Some(2), + 35 => Some(2), + 36 => Some(2), + 37 => Some(2), + 38 => Some(2), + 39 => Some(2), + 40 => Some(2), + 41 => Some(2), + 42 => Some(2), + 43 => Some(2), + 44 => Some(1), + 45 | 46 => Some(0), + 47 => None, + 48 | 49 => Some(1), + 50 | 51 | 55 | 56 => Some(0), + 57 => Some(1), + 60 | 61 | 64 | 65 | 66 | 67 | 68 | 69 | 71 | 72 | 74 | 75 | 76 => Some(0), + _ => None, + } +} From fa03e5d10735908269b8dc472df5d1544e1bcba0 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 12 Jun 2026 23:43:49 -0400 Subject: [PATCH 05/15] Adjust clippy lints, usize casts, and freeze() Add/relax several clippy allow attributes in the Kafka codec module; simplify i32->usize conversion for array length by using a direct cast with a safety comment (avoids unnecessary try_from and map_err), and add #[must_use] to Encoder::freeze to prevent accidental discards. Also add a TODO comment in the Produce response placeholder for populating the topic name. --- gateways/kafka/src/protocol/codec.rs | 14 ++++++++------ gateways/kafka/src/protocol/responses.rs | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index b46cb6476a..5f001ac209 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -17,7 +17,9 @@ //! Low-level Kafka primitive encoders/decoders (ported wire codec). -#![allow(clippy::pedantic, clippy::missing_const_for_fn)] +#![allow(clippy::missing_const_for_fn, clippy::bool_to_int_with_if)] +#![allow(clippy::missing_errors_doc, clippy::cast_sign_loss, clippy::must_use_candidate,clippy::missing_panics_doc, clippy::cast_possible_truncation,clippy::cast_lossless)] + use bytes::{Buf, BufMut, Bytes, BytesMut}; @@ -97,10 +99,9 @@ impl Decoder { if n < 0 { return Err(KafkaProtocolError::InvalidArrayLength(n)); } - let count = usize::try_from(n).map_err(|_| KafkaProtocolError::CollectionTooLarge { - count: n as usize, - max: MAX_COLLECTION_LEN, - })?; + // Safe: n is in [0, i32::MAX]; i32::MAX (2_147_483_647) fits in usize + // on all 32-bit and 64-bit platforms this crate targets. + let count = n as usize; if count > MAX_COLLECTION_LEN { return Err(KafkaProtocolError::CollectionTooLarge { count, @@ -209,6 +210,7 @@ impl Decoder { }); } let count = count as usize; + for _ in 0..count { self.read_varint()?; // tag number let size = usize::try_from(self.read_varint()?).map_err(|_| { @@ -343,7 +345,7 @@ impl Encoder { pub fn write_empty_tagged_fields(&mut self) { self.write_varint(0); } - + #[must_use] pub fn freeze(self) -> Bytes { self.bytes.freeze() } diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index 8de6868ffd..6a84be1402 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -30,7 +30,7 @@ use bytes::Bytes; /// Well-formed Produce response with a single placeholder topic/partition. pub fn encode_produce_error_response(version: i16, error_code: i16) -> Bytes { let topics = vec![ProduceTopicData { - topic: String::new(), + topic: String::new(), // TODO topic name will be populated in the end to end functional completion partitions: vec![ProducePartitionData { partition: 0, records: None, From 882bad43bab9f9581e4bd764ef55f0287ade7230 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 15:27:26 -0400 Subject: [PATCH 06/15] kafka gateway: rename crate and refactor server Rename the crate/binary to iggy-gateway-kafka and update README and Cargo.toml accordingly. Introduce DEFAULT_KAFKA_PORT and refactor the server to accept a pre-bound TcpListener (eliminates bind TOCTOU) and use listener.local_addr() to compute advertised broker information, returning clearer config errors. Harden protocol handling: add Encoder helpers (unchecked nullable strings, write_null_bytes), handle metadata parsing errors safely, downgrade some decode errors to warnings, and adjust metadata encoding for flexible versions. Improve read_frame (single deadline for length+body, chunked reservation to avoid large upfront allocs, truncate extra bytes, and transient accept backoff). Update tests and CI: add rust-gateway component and include gateways in PR-title workflow. --- .github/config/components.yml | 18 ++++ .github/workflows/pr-title.yml | 1 + Cargo.lock | 24 +++--- gateways/kafka/Cargo.toml | 6 +- gateways/kafka/README.md | 6 +- gateways/kafka/src/main.rs | 6 +- gateways/kafka/src/protocol/api.rs | 69 ++++++--------- gateways/kafka/src/protocol/codec.rs | 28 ++++++- gateways/kafka/src/protocol/responses.rs | 15 ++-- gateways/kafka/src/server.rs | 83 ++++++++++++------- gateways/kafka/tests/api_handler_tests.rs | 41 ++++++--- .../kafka/tests/broker_advertise_tests.rs | 44 ++++------ gateways/kafka/tests/common/server.rs | 4 +- .../kafka/tests/version_firewall_tests.rs | 26 ++++-- 14 files changed, 217 insertions(+), 154 deletions(-) diff --git a/.github/config/components.yml b/.github/config/components.yml index 79b11a255c..d529ff4bbe 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -146,6 +146,7 @@ components: - "rust-connectors" - "rust-mcp" - "rust-integration" + - "rust-gateway" - "ci-infrastructure" paths: - "Dockerfile*" @@ -484,3 +485,20 @@ components: - ".github/actions/**/*.yml" - ".github/ci/**/*.yml" tasks: ["validate"] # Could run workflow validation + + # gateways are not Rust components, but we want to run them in CI + rust-gateway: + depends_on: + - "rust-sdk" + - "rust-workspace" + - "ci-infrastructure" + paths: + - "gateways/**" + tasks: + - "check" + - "fmt" + - "clippy" + - "sort" + - "test-1" + - "test-2" + - "machete" \ No newline at end of file diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 57a6bbc2e4..f485734caa 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -95,3 +95,4 @@ jobs: storage simulator configs + gateways diff --git a/Cargo.lock b/Cargo.lock index ff068265c6..7bd874ff7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6668,6 +6668,18 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "iggy-gateway-kafka" +version = "0.1.0" +dependencies = [ + "bytes", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", +] + [[package]] name = "iggy-mcp" version = "0.4.1-edge.1" @@ -7079,18 +7091,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "iggy_gateway_kafka" -version = "0.1.0" -dependencies = [ - "bytes", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "tracing-subscriber", -] - [[package]] name = "ignore" version = "0.4.25" diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 93a335d68a..88c0e8a2dc 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -16,7 +16,7 @@ # under the License. [package] -name = "iggy_gateway_kafka" +name = "iggy-gateway-kafka" version = "0.1.0" description = "Kafka wire protocol gateway foundation for Apache Iggy" edition = "2024" @@ -29,7 +29,7 @@ readme = "README.md" publish = false [[bin]] -name = "iggy-kafka-gateway" +name = "iggy-gateway-kafka" path = "src/main.rs" [dependencies] @@ -45,6 +45,6 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io- [lints.clippy] enum_glob_use = "deny" -# Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. +#Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. pedantic = "warn" nursery = "warn" diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index 09495b3c3b..7994cf0178 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -1,11 +1,11 @@ -# Kafka gateway (`iggy_gateway_kafka`) +# Kafka gateway (`iggy-gateway-kafka`) Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421): a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, and returns stub responses. ## Run ```bash -cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway +cargo run -p iggy-gateway-kafka ``` Default bind: `127.0.0.1:9093`. Override with `KAFKA_BIND_ADDR` (e.g. `0.0.0.0:9093`). @@ -13,7 +13,7 @@ Default bind: `127.0.0.1:9093`. Override with `KAFKA_BIND_ADDR` (e.g. `0.0.0.0:9 ## Test ```bash -cargo test -p iggy_gateway_kafka +cargo test -p iggy-gateway-kafka ``` 103 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs index 9cd81a4de2..0f48ea943e 100644 --- a/gateways/kafka/src/main.rs +++ b/gateways/kafka/src/main.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use tokio::net::TcpListener; use tokio::signal; use tokio::sync::broadcast; @@ -39,10 +40,13 @@ async fn main() -> Result<(), Box> { .map_err(|e| format!("invalid KAFKA_ADVERTISED_PORT `{advertised_port}`: {e}"))?, ); } + let listener = TcpListener::bind(&config.bind_addr) + .await + .map_err(|e| format!("failed to bind {}: {e}", config.bind_addr))?; let server = KafkaServer::new(config); let (tx, rx) = broadcast::channel(1); - let mut server_task = tokio::spawn(async move { server.run(rx).await }); + let mut server_task = tokio::spawn(async move { server.run(listener, rx).await }); tokio::select! { result = &mut server_task => { diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 55cf238f95..b49419eb79 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -use std::net::SocketAddr; - use bytes::Bytes; +use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::{Decoder, Encoder}; use crate::protocol::requests::{ decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, @@ -37,21 +36,13 @@ pub const API_KEY_METADATA: i16 = 3; pub const API_KEY_API_VERSIONS: i16 = 18; pub const API_KEY_CREATE_TOPICS: i16 = 19; +pub const DEFAULT_KAFKA_PORT: u16 = 9093; + pub const ERROR_NONE: i16 = 0; -pub const ERROR_OFFSET_OUT_OF_RANGE: i16 = 1; -pub const ERROR_CORRUPT_MESSAGE: i16 = 2; pub const ERROR_UNKNOWN_TOPIC_OR_PARTITION: i16 = 3; -pub const ERROR_INVALID_FETCH_SIZE: i16 = 4; -pub const ERROR_LEADER_NOT_AVAILABLE: i16 = 5; -pub const ERROR_NOT_LEADER_OR_FOLLOWER: i16 = 6; -pub const ERROR_REQUEST_TIMED_OUT: i16 = 7; -pub const ERROR_UNKNOWN_SERVER_ERROR: i16 = -1; pub const ERROR_UNSUPPORTED_VERSION: i16 = 35; -pub const ERROR_TOPIC_ALREADY_EXISTS: i16 = 36; pub const ERROR_INVALID_PARTITIONS: i16 = 37; -pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; pub const ERROR_INVALID_REQUEST: i16 = 42; -pub const ERROR_UNSUPPORTED_FOR_MESSAGE_FORMAT: i16 = 43; /// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; @@ -62,27 +53,11 @@ pub struct BrokerAdvertise { pub port: i32, } -impl BrokerAdvertise { - #[must_use] - pub fn from_bind_addr(bind_addr: &str) -> Self { - bind_addr.parse::().map_or_else( - |_| Self { - host: "127.0.0.1".to_string(), - port: 9093, - }, - |addr| Self { - host: addr.ip().to_string(), - port: i32::from(addr.port()), - }, - ) - } -} - impl Default for BrokerAdvertise { fn default() -> Self { Self { host: "127.0.0.1".to_string(), - port: 9093, + port: i32::from(DEFAULT_KAFKA_PORT), } } } @@ -151,7 +126,7 @@ pub fn handle_request( if is_supported_version(api_key, api_version) { encode_metadata_response(api_version, body, broker, ERROR_NONE) } else { - encode_metadata_response(0, body, broker, ERROR_UNSUPPORTED_VERSION) + encode_metadata_response(api_version, body, broker, ERROR_UNSUPPORTED_VERSION) } } API_KEY_PRODUCE => { @@ -159,7 +134,7 @@ pub fn handle_request( match decode_produce_request(api_version, body) { Ok(req) => encode_produce_response(api_version, &req), Err(e) => { - tracing::error!("Failed to decode Produce request: {:?}", e); + tracing::warn!("Failed to decode Produce request: {:?}", e); encode_produce_error_response(api_version, ERROR_INVALID_REQUEST) } } @@ -172,7 +147,7 @@ pub fn handle_request( match decode_fetch_request(api_version, body) { Ok(req) => encode_fetch_response(api_version, &req), Err(e) => { - tracing::error!("Failed to decode Fetch request: {:?}", e); + tracing::warn!("Failed to decode Fetch request: {:?}", e); encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST) } } @@ -185,7 +160,7 @@ pub fn handle_request( match decode_list_offsets_request(api_version, body) { Ok(req) => encode_list_offsets_response(api_version, &req), Err(e) => { - tracing::error!("Failed to decode ListOffsets request: {:?}", e); + tracing::warn!("Failed to decode ListOffsets request: {:?}", e); encode_list_offsets_error_response(api_version, ERROR_INVALID_REQUEST) } } @@ -198,7 +173,7 @@ pub fn handle_request( match decode_create_topics_request(api_version, body) { Ok(req) => encode_create_topics_response(api_version, &req), Err(e) => { - tracing::error!("Failed to decode CreateTopics request: {:?}", e); + tracing::warn!("Failed to decode CreateTopics request: {:?}", e); encode_create_topics_error_response(api_version, ERROR_INVALID_REQUEST) } } @@ -273,7 +248,12 @@ fn encode_metadata_response( top_level_error_code: i16, ) -> Bytes { let flexible = api_version >= 9; - let topics_count = split_metadata_request_topics(body, api_version); + // BufferUnderflow (empty body) → treat as 0 topics; other decode errors are truly invalid. + let topics_count = match split_metadata_request_topics(body, api_version) { + Ok(n) => n, + Err(KafkaProtocolError::BufferUnderflow { .. }) => 0, + Err(_) => return encode_error_only_response(ERROR_INVALID_REQUEST), + }; let topic_error = if top_level_error_code == ERROR_NONE { ERROR_UNKNOWN_TOPIC_OR_PARTITION } else { @@ -311,14 +291,18 @@ fn encode_metadata_response( } else { e.write_i32(1); // brokers array length e.write_i32(1); // node_id - let _ = e.write_nullable_string(Some(&broker.host)); + // broker.host is config-derived (KAFKA_ADVERTISED_HOST), not request-decoded — use + // the checked variant so an overly long hostname returns an error instead of panicking. + if e.write_nullable_string(Some(&broker.host)).is_err() { + return encode_error_only_response(ERROR_INVALID_REQUEST); + } e.write_i32(broker.port); if api_version >= 1 { - let _ = e.write_nullable_string(None); // rack + e.write_nullable_string_unchecked(None); // rack } if api_version >= 2 { - let _ = e.write_nullable_string(None); // cluster_id + e.write_nullable_string_unchecked(None); // cluster_id } if api_version >= 1 { e.write_i32(1); // controller_id — must come before topics array @@ -327,7 +311,7 @@ fn encode_metadata_response( e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); for _ in 0..topics_count { e.write_i16(topic_error); - let _ = e.write_nullable_string(Some("unknown-topic")); + e.write_nullable_string_unchecked(Some("unknown-topic")); if api_version >= 1 { e.write_bool(false); // is_internal } @@ -351,12 +335,11 @@ pub fn encode_error_only_response(error_code: i16) -> Bytes { e.freeze() } -#[must_use] -pub(crate) fn split_metadata_request_topics(body: Bytes, api_version: i16) -> usize { +pub(crate) fn split_metadata_request_topics(body: Bytes, api_version: i16) -> Result { let mut d = Decoder::new(body); if api_version >= 9 { - d.read_compact_array_count().unwrap_or(0) + d.read_compact_array_count() } else { - d.read_i32_array_count().unwrap_or(0) + d.read_i32_array_count() } } diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 5f001ac209..214c79af9e 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -18,8 +18,14 @@ //! Low-level Kafka primitive encoders/decoders (ported wire codec). #![allow(clippy::missing_const_for_fn, clippy::bool_to_int_with_if)] -#![allow(clippy::missing_errors_doc, clippy::cast_sign_loss, clippy::must_use_candidate,clippy::missing_panics_doc, clippy::cast_possible_truncation,clippy::cast_lossless)] - +#![allow( + clippy::missing_errors_doc, + clippy::cast_sign_loss, + clippy::must_use_candidate, + clippy::missing_panics_doc, + clippy::cast_possible_truncation, + clippy::cast_lossless +)] use bytes::{Buf, BufMut, Bytes, BytesMut}; @@ -297,6 +303,19 @@ impl Encoder { Ok(()) } + /// Infallible variant for response-encoding paths where the string originated from a decoded + /// Kafka request and is therefore already bounded to `i16::MAX` bytes. + pub fn write_nullable_string_unchecked(&mut self, v: Option<&str>) { + match v { + None => self.write_i16(-1), + Some(s) => { + debug_assert!(i16::try_from(s.len()).is_ok()); + self.write_i16(i16::try_from(s.len()).expect("caller guarantees len <= i16::MAX")); + self.bytes.put_slice(s.as_bytes()); + } + } + } + /// Compact nullable string (flexible versions): varint(len+1), 0 for null. pub fn write_compact_nullable_string(&mut self, v: Option<&str>) { match v { @@ -308,6 +327,11 @@ impl Encoder { } } + /// Write a null bytes field (i32 -1). Infallible; use instead of `write_nullable_bytes(None)`. + pub fn write_null_bytes(&mut self) { + self.write_i32(-1); + } + /// Legacy nullable bytes: i32 length prefix, -1 for null. pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) -> Result<()> { match v { diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index 6a84be1402..8c44692393 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -61,7 +61,7 @@ fn encode_produce_response_inner( if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - let _ = e.write_nullable_string(Some(&topic.topic)); + e.write_nullable_string_unchecked(Some(&topic.topic)); } if flexible { @@ -86,7 +86,7 @@ fn encode_produce_response_inner( e.write_compact_nullable_string(None); } else { e.write_i32(0); - let _ = e.write_nullable_string(None); + e.write_nullable_string_unchecked(None); } } if flexible { @@ -160,7 +160,7 @@ fn encode_fetch_response_inner( if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - let _ = e.write_nullable_string(Some(&topic.topic)); + e.write_nullable_string_unchecked(Some(&topic.topic)); } if flexible { @@ -192,8 +192,7 @@ fn encode_fetch_response_inner( if flexible { e.write_compact_nullable_bytes(None); } else { - e.write_nullable_bytes(None) - .expect("null bytes always encode"); + e.write_null_bytes(); } if flexible { e.write_empty_tagged_fields(); @@ -252,7 +251,7 @@ fn encode_list_offsets_response_inner( if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - let _ = e.write_nullable_string(Some(&topic.topic)); + e.write_nullable_string_unchecked(Some(&topic.topic)); } if flexible { @@ -328,7 +327,7 @@ fn encode_create_topics_response_inner( if flexible { e.write_compact_nullable_string(Some(&topic.name)); } else { - let _ = e.write_nullable_string(Some(&topic.name)); + e.write_nullable_string_unchecked(Some(&topic.name)); } let error_code = if topic_error != ERROR_NONE { @@ -344,7 +343,7 @@ fn encode_create_topics_response_inner( if flexible { e.write_compact_nullable_string(None); } else { - let _ = e.write_nullable_string(None); + e.write_nullable_string_unchecked(None); } } diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 94ef7cb879..6b313a67e8 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -29,7 +29,8 @@ use tracing::{debug, error, info, warn}; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{ - BrokerAdvertise, ERROR_INVALID_REQUEST, encode_error_only_response, handle_request, + BrokerAdvertise, DEFAULT_KAFKA_PORT, ERROR_INVALID_REQUEST, encode_error_only_response, + handle_request, }; use crate::protocol::codec::Decoder; use crate::protocol::header::{ @@ -37,6 +38,8 @@ use crate::protocol::header::{ }; use std::io; +const READ_CHUNK: usize = 65536; + #[derive(Debug, Clone)] pub struct ServerConfig { pub bind_addr: String, @@ -53,7 +56,7 @@ pub struct ServerConfig { impl Default for ServerConfig { fn default() -> Self { Self { - bind_addr: "127.0.0.1:9093".to_string(), + bind_addr: format!("127.0.0.1:{DEFAULT_KAFKA_PORT}"), advertised_host: None, advertised_port: None, max_frame_size: 8 * 1024 * 1024, @@ -64,36 +67,35 @@ impl Default for ServerConfig { } impl BrokerAdvertise { - /// Resolve the broker endpoint advertised in Metadata from listener config. + /// Resolve the broker endpoint advertised in Metadata. + /// + /// `local_addr` is the address the listener is actually bound to (from `listener.local_addr()`). /// /// # Errors /// - /// Returns an error when `bind_addr` is invalid, `advertised_host` is empty, or the listener - /// binds to a wildcard address without an explicit advertised host. - pub fn from_server_config(config: &ServerConfig) -> std::result::Result { - let bind = config - .bind_addr - .parse::() - .map_err(|e| format!("invalid bind address `{}`: {e}", config.bind_addr))?; - + /// Returns `InvalidConfig` when `advertised_host` is empty or the listener binds to a wildcard + /// without an explicit advertised host. + pub fn from_server_config(config: &ServerConfig, local_addr: SocketAddr) -> Result { let port = config .advertised_port - .map_or_else(|| i32::from(bind.port()), i32::from); + .map_or_else(|| i32::from(local_addr.port()), i32::from); let host = if let Some(ref advertised) = config.advertised_host { let trimmed = advertised.trim(); if trimmed.is_empty() { - return Err("KAFKA_ADVERTISED_HOST must not be empty".into()); + return Err(KafkaProtocolError::InvalidConfig( + "KAFKA_ADVERTISED_HOST must not be empty".into(), + )); } trimmed.to_string() - } else if bind.ip().is_unspecified() { - return Err( + } else if local_addr.ip().is_unspecified() { + return Err(KafkaProtocolError::InvalidConfig( "binding to a wildcard address (0.0.0.0 or ::) requires KAFKA_ADVERTISED_HOST \ to be set to a reachable hostname or IP for Metadata broker advertisement" .into(), - ); + )); } else { - bind.ip().to_string() + local_addr.ip().to_string() }; Ok(Self { host, port }) @@ -114,18 +116,25 @@ impl KafkaServer { /// Accept Kafka wire connections until `shutdown` fires, then drain in-flight tasks. /// + /// `listener` must already be bound by the caller. This lets tests and `main` bind + /// the port before spawning the task, eliminating the TOCTOU race of bind-drop-rebind. + /// /// # Errors /// - /// Returns an error if binding fails or a non-transient `accept()` error occurs. - pub async fn run(self, mut shutdown: broadcast::Receiver<()>) -> Result<()> { - let broker = Arc::new( - BrokerAdvertise::from_server_config(&self.config) - .map_err(KafkaProtocolError::InvalidConfig)?, - ); - let listener = TcpListener::bind(&self.config.bind_addr).await?; + /// Returns an error on invalid config or a non-transient `accept()` error. + pub async fn run( + self, + listener: TcpListener, + mut shutdown: broadcast::Receiver<()>, + ) -> Result<()> { + let local_addr = listener.local_addr()?; + let broker = Arc::new(BrokerAdvertise::from_server_config( + &self.config, + local_addr, + )?); info!( "kafka listener bound on {} (advertised as {}:{})", - self.config.bind_addr, broker.host, broker.port + local_addr, broker.host, broker.port ); let tracker = TaskTracker::new(); @@ -167,6 +176,10 @@ impl KafkaServer { }); } Err(e) if is_transient_accept_error(&e) => { + // Brief backoff on fd exhaustion to avoid busy-spinning. + if matches!(e.raw_os_error(), Some(23 | 24)) { + tokio::time::sleep(Duration::from_millis(10)).await; + } warn!(%e, "transient accept error, continuing"); } Err(e) => return Err(e.into()), @@ -311,8 +324,12 @@ pub async fn read_frame( max_frame_size: usize, read_timeout: Duration, ) -> Result { + // Single deadline for both the length-prefix read and the body read. Without this, a + // slow-drip sender could hold a connection open for 2x read_timeout by sending one byte + // per timeout window. + let deadline = tokio::time::Instant::now() + read_timeout; let mut len_buf = [0u8; 4]; - timeout(read_timeout, stream.read_exact(&mut len_buf)) + timeout_at(deadline, stream.read_exact(&mut len_buf)) .await .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; @@ -324,7 +341,6 @@ pub async fn read_frame( let frame_len = usize::try_from(frame_len_i32).map_err(|_| KafkaProtocolError::FrameTooLarge { max_bytes: max_frame_size, - // Positive i32 that does not fit in `usize` (e.g. 16-bit targets). actual_bytes: usize::MAX, })?; if frame_len > max_frame_size { @@ -334,12 +350,11 @@ pub async fn read_frame( }); } - // read_buf fills BytesMut spare capacity without zero-initializing it first. - // Single deadline for the entire body so a slow-drip sender can't stall indefinitely - // by delivering one byte per timeout window. - let deadline = tokio::time::Instant::now() + read_timeout; - let mut data = BytesMut::with_capacity(frame_len); + // Reserve in 64 KB increments so a max-size frame (8 MB by default) does not trigger a + // single large upfront allocation before any payload bytes have arrived. + let mut data = BytesMut::new(); while data.len() < frame_len { + data.reserve((frame_len - data.len()).min(READ_CHUNK)); match timeout_at(deadline, stream.read_buf(&mut data)).await { Err(_) => return Err(io::Error::new(io::ErrorKind::TimedOut, "read timeout").into()), Ok(Ok(0)) => { @@ -351,6 +366,10 @@ pub async fn read_frame( Ok(Ok(_)) => {} } } + // read_buf may have written past frame_len if the OS returned more bytes than we + // reserved (capacity can round up). Truncate so pipelined frames don't bleed into + // decoder.read_bytes(remaining()) at the call site. + data.truncate(frame_len); Ok(data.freeze()) } diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 2d342cfd8e..ab3e762332 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -103,23 +103,38 @@ fn metadata_response_has_broker_array_and_topic_array() { #[test] fn unsupported_version_returns_protocol_error() { - let mut req = Vec::new(); - req.extend_from_slice(&1_i32.to_be_bytes()); - let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req), &test_broker()); + // v99 client sends a compact-array body (count+1 = 2 = 0x02 for 1 topic). + // The gateway caps at v9 (highest supported Metadata version) for both parsing + // and encoding, so the response uses the flexible (v9) wire format. + let body = handle_request( + API_KEY_METADATA, + 99, + Bytes::from_static(&[0x02]), + &test_broker(), + ); let mut d = Decoder::new(body); - // Metadata v0: brokers[], topics[] — no controller_id (added in v1) - let _broker_count = d.read_i32().unwrap(); - let _ = d.read_i32().unwrap(); // node_id - let _ = d.read_nullable_string().unwrap(); // host - let _ = d.read_i32().unwrap(); // port - let topic_count = d.read_i32().unwrap(); + // v9 flexible response layout: + d.read_i32().unwrap(); // throttle_time_ms (v3+) + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); // node_id + d.read_compact_nullable_string().unwrap(); // host + d.read_i32().unwrap(); // port + d.read_compact_nullable_string().unwrap(); // rack + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); // cluster_id (v2+) + d.read_i32().unwrap(); // controller_id (v1+) + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); assert_eq!(topic_count, 1); let topic_error = d.read_i16().unwrap(); assert_eq!(topic_error, ERROR_UNSUPPORTED_VERSION); - let topic_name = d.read_nullable_string().unwrap().unwrap(); - assert_eq!(topic_name, "unknown-topic"); - let partitions_count = d.read_i32().unwrap(); - assert_eq!(partitions_count, 0); + let topic_name = d.read_compact_nullable_string().unwrap(); + assert_eq!(topic_name, Some("unknown-topic".to_string())); } // ── Misc ──────────────────────────────────────────────────────────────────── diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs index 9d60d418e4..8f482019c4 100644 --- a/gateways/kafka/tests/broker_advertise_tests.rs +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -17,31 +17,12 @@ //! `BrokerAdvertise` parsing and metadata reflection. +use std::net::SocketAddr; + use iggy_gateway_kafka::ServerConfig; use iggy_gateway_kafka::protocol::api::{API_KEY_METADATA, BrokerAdvertise, handle_request}; use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; -#[test] -fn from_bind_addr_parses_ipv4_and_port() { - let b = BrokerAdvertise::from_bind_addr("192.168.1.10:19092"); - assert_eq!(b.host, "192.168.1.10"); - assert_eq!(b.port, 19092); -} - -#[test] -fn from_bind_addr_parses_ipv6() { - let b = BrokerAdvertise::from_bind_addr("[::1]:9093"); - assert_eq!(b.host, "::1"); - assert_eq!(b.port, 9093); -} - -#[test] -fn from_bind_addr_invalid_falls_back_to_default() { - let b = BrokerAdvertise::from_bind_addr("not-a-socket-addr"); - assert_eq!(b.host, "127.0.0.1"); - assert_eq!(b.port, 9093); -} - #[test] fn default_matches_standard_gateway_port() { let b = BrokerAdvertise::default(); @@ -50,8 +31,11 @@ fn default_matches_standard_gateway_port() { } #[test] -fn metadata_reflects_parsed_bind_addr() { - let broker = BrokerAdvertise::from_bind_addr("203.0.113.7:9093"); +fn metadata_reflects_broker_addr() { + let broker = BrokerAdvertise { + host: "203.0.113.7".to_string(), + port: 9093, + }; let mut req = Encoder::with_capacity(4); req.write_i32(0); let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker); @@ -72,7 +56,8 @@ fn from_server_config_uses_explicit_advertised_host_on_wildcard_bind() { advertised_host: Some("kafka.internal".to_string()), ..ServerConfig::default() }; - let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + let local_addr: SocketAddr = "0.0.0.0:9093".parse().unwrap(); + let broker = BrokerAdvertise::from_server_config(&config, local_addr).expect("valid config"); assert_eq!(broker.host, "kafka.internal"); assert_eq!(broker.port, 9093); } @@ -83,8 +68,9 @@ fn from_server_config_rejects_wildcard_bind_without_advertised_host() { bind_addr: "0.0.0.0:9093".to_string(), ..ServerConfig::default() }; - let err = BrokerAdvertise::from_server_config(&config).unwrap_err(); - assert!(err.contains("KAFKA_ADVERTISED_HOST")); + let local_addr: SocketAddr = "0.0.0.0:9093".parse().unwrap(); + let err = BrokerAdvertise::from_server_config(&config, local_addr).unwrap_err(); + assert!(err.to_string().contains("KAFKA_ADVERTISED_HOST")); } #[test] @@ -93,7 +79,8 @@ fn from_server_config_uses_bind_ip_for_non_wildcard_listener() { bind_addr: "192.168.1.10:19092".to_string(), ..ServerConfig::default() }; - let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + let local_addr: SocketAddr = "192.168.1.10:19092".parse().unwrap(); + let broker = BrokerAdvertise::from_server_config(&config, local_addr).expect("valid config"); assert_eq!(broker.host, "192.168.1.10"); assert_eq!(broker.port, 19092); } @@ -106,7 +93,8 @@ fn from_server_config_honors_advertised_port_override() { advertised_port: Some(19093), ..ServerConfig::default() }; - let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + let local_addr: SocketAddr = "127.0.0.1:9093".parse().unwrap(); + let broker = BrokerAdvertise::from_server_config(&config, local_addr).expect("valid config"); assert_eq!(broker.host, "broker.example.com"); assert_eq!(broker.port, 19093); } diff --git a/gateways/kafka/tests/common/server.rs b/gateways/kafka/tests/common/server.rs index 178dbce2e5..a009b46423 100644 --- a/gateways/kafka/tests/common/server.rs +++ b/gateways/kafka/tests/common/server.rs @@ -31,7 +31,6 @@ pub async fn spawn_test_server() -> (SocketAddr, broadcast::Sender<()>) { .await .expect("bind ephemeral port"); let addr = listener.local_addr().expect("local addr"); - drop(listener); let config = ServerConfig { bind_addr: addr.to_string(), @@ -44,9 +43,8 @@ pub async fn spawn_test_server() -> (SocketAddr, broadcast::Sender<()>) { let (shutdown_tx, shutdown_rx) = broadcast::channel(1); let server = KafkaServer::new(config); tokio::spawn(async move { - let _ = server.run(shutdown_rx).await; + let _ = server.run(listener, shutdown_rx).await; }); - tokio::time::sleep(Duration::from_millis(50)).await; (addr, shutdown_tx) } diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index de0f96e03b..77c4e768f8 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -171,18 +171,32 @@ fn metadata_below_min_version_returns_topic_error() { #[test] fn metadata_above_max_version_returns_topic_error() { + // v10 uses flexible encoding; compact array varint(2) = 1 topic. let body = handle_request( API_KEY_METADATA, 10, - metadata_request_one_topic(), + Bytes::from_static(&[0x02]), &default_broker(), ); + // Response is in v9 flexible format (highest supported). let mut d = Decoder::new(body); - let _brokers = d.read_i32().unwrap(); - let _ = d.read_i32().unwrap(); - let _ = d.read_nullable_string().unwrap(); - let _ = d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 1); + d.read_i32().unwrap(); // throttle_time_ms (v3+) + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); // cluster_id + d.read_i32().unwrap(); // controller_id + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!(topic_count, 1); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } From 470c319de15df0667e1541db9ebb849ee3b9ef34 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:11:32 -0400 Subject: [PATCH 07/15] kafka gateway: metadata, server fixes and docs Multiple fixes and improvements to the Kafka gateway: - Docs: document KAFKA_* env vars in README and normalize package name references from `iggy_gateway_kafka` to `iggy-gateway-kafka` across MANUAL_TESTING and TEST_SUITE. - Protocol: add MAX_SUPPORTED_METADATA_VERSION and clamp metadata responses to the highest implemented version; treat empty or malformed metadata request bodies as a 0-topic response with appropriate per-topic error instead of failing; remove unused KafkaProtocolError import. - Server: validate KAFKA_ADVERTISED_HOST length against Kafka nullable string limit and return a config error if exceeded; log a warn when setting TCP_NODELAY fails. - I/O: rewrite read_frame to use bounded read() slices (avoid allocator over-reads via read_buf) so pipelined frames are not consumed accidentally. - Tests: add tests for advertised-host length, corrupt metadata partial-body behavior returning zero topics, and that read_frame does not consume pipelined bytes; adjust existing tests/docs to match crate name changes. These changes fix metadata decoding semantics, prevent frame bleed between pipelined requests, harden config validation, and align documentation/test commands with the package name. --- gateways/kafka/README.md | 8 ++++- gateways/kafka/docs/MANUAL_TESTING.md | 14 ++++---- gateways/kafka/docs/TEST_SUITE.md | 8 ++--- gateways/kafka/src/protocol/api.rs | 30 ++++++++++++----- gateways/kafka/src/server.rs | 33 ++++++++++++------- .../kafka/tests/broker_advertise_tests.rs | 12 +++++++ .../kafka/tests/metadata_regression_tests.rs | 14 ++++++++ .../kafka/tests/server_integration_tests.rs | 25 ++++++++++++++ 8 files changed, 112 insertions(+), 32 deletions(-) diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index 7994cf0178..b336887cd0 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -8,7 +8,13 @@ Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/34 cargo run -p iggy-gateway-kafka ``` -Default bind: `127.0.0.1:9093`. Override with `KAFKA_BIND_ADDR` (e.g. `0.0.0.0:9093`). +Default bind: `127.0.0.1:9093`. Environment variables: + +| Variable | Default | Description | +|---|---|---| +| `KAFKA_BIND_ADDR` | `127.0.0.1:9093` | TCP address to listen on | +| `KAFKA_ADVERTISED_HOST` | bind IP | Hostname/IP clients use to reach this broker (required when binding to `0.0.0.0`/`::`) | +| `KAFKA_ADVERTISED_PORT` | bind port | Port advertised in Metadata responses | ## Test diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index cd3e8c0e58..30f37783c6 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -22,10 +22,10 @@ See also: [SCOPE.md](SCOPE.md) (supported API keys), [TEST_SUITE.md](TEST_SUITE. ```bash # From iggy workspace root -cargo build -p iggy_gateway_kafka --bin iggy-kafka-gateway +cargo build -p iggy-gateway-kafka # Terminal 1 — start listener (default 127.0.0.1:9093) -RUST_LOG=info cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway +RUST_LOG=info cargo run -p iggy-gateway-kafka ``` Expected log: @@ -50,7 +50,7 @@ cargo run -p kafka-message-gen -- generate \ Run before manual testing to catch regressions: ```bash -cargo test -p iggy_gateway_kafka +cargo test -p iggy-gateway-kafka ``` All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (step above). @@ -63,7 +63,7 @@ All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (ste | ID | Test | Steps | Expected result | Pass criteria | |----|------|-------|-----------------|---------------| -| A1 | Gateway starts | Run `iggy-kafka-gateway` | Binds to `:9093`, no panic | Log shows bind address | +| A1 | Gateway starts | Run `iggy-gateway-kafka` | Binds to `:9093`, no panic | Log shows bind address | | A2 | ApiVersions v1 | `cargo run -p kafka-message-gen -- send --host 127.0.0.1:9093 --api-key 18 --version 1` | Response received | `ec=0`, non-zero byte count | | A3 | ApiVersions v3 (flexible) | Same with `--version 3` | Response received | `ec=0` | | A4 | Metadata v0 | `send --api-key 3 --version 0` | Stub broker in response | `ec=0` or topic error 3 (stub) | @@ -238,8 +238,8 @@ kcat version (if used): ___________ [ ] H1–H3 Adversarial input Automated regression: -[ ] cargo test -p iggy_gateway_kafka — ___/103 passed -[ ] cargo clippy -p iggy_gateway_kafka — clean / warnings noted +[ ] cargo test -p iggy-gateway-kafka — ___/103 passed +[ ] cargo clippy -p iggy-gateway-kafka — clean / warnings noted Notes / failures: _________________________________ @@ -251,7 +251,7 @@ _________________________________ | Symptom | Likely cause | Fix | |---------|--------------|-----| -| `Connection refused` on 9093 | Gateway not running | Start `iggy-kafka-gateway` | +| `Connection refused` on 9093 | Gateway not running | Start `iggy-gateway-kafka` | | `decode_validation_tests` panic | Missing fixtures | Run `kafka-message-gen generate` | | `ec=35` for in-range version | Version not in `SUPPORTED_RANGES` | Check `SCOPE.md` and `api.rs` | | kcat hangs | Timeout waiting for data | Set `-m 1000`; check gateway logs | diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index c8515387c2..c0f6df8c1e 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -3,7 +3,7 @@ Regression tests live under [`tests/`](../tests/). Run from the workspace root: ```bash -cargo test -p iggy_gateway_kafka +cargo test -p iggy-gateway-kafka ``` **Current count:** 103 tests across 12 suites (as of #3421 foundation). @@ -34,7 +34,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee | [`decode_validation_tests.rs`](../tests/decode_validation_tests.rs) | kafka-tool fixture decode + response structure per version | 14 | **Yes** | | [`version_firewall_tests.rs`](../tests/version_firewall_tests.rs) | Version boundary matrix, unsupported keys, corrupt bodies | 17 | Partial | | [`metadata_regression_tests.rs`](../tests/metadata_regression_tests.rs) | Metadata v0–v9, topic counts, broker advertise | 7 | No | -| [`broker_advertise_tests.rs`](../tests/broker_advertise_tests.rs) | `BrokerAdvertise::from_bind_addr` parsing | 5 | No | +| [`broker_advertise_tests.rs`](../tests/broker_advertise_tests.rs) | `BrokerAdvertise::from_server_config` parsing | 5 | No | | [`handler_regression_tests.rs`](../tests/handler_regression_tests.rs) | Every scoped key×version via `handle_request`, stub error codes | 5 | Partial | | [`server_integration_tests.rs`](../tests/server_integration_tests.rs) | `read_frame` / `write_frame` unit-level I/O | 4 | No | | [`server_e2e_tests.rs`](../tests/server_e2e_tests.rs) | Full `KafkaServer` TCP round-trips | 8 | Partial | @@ -141,10 +141,10 @@ cargo run -p kafka-message-gen -- generate \ --api-key 0 --api-key 1 --api-key 2 --api-key 19 # 2. Run regression suite -cargo test -p iggy_gateway_kafka +cargo test -p iggy-gateway-kafka # 3. Optional lint gate -cargo clippy -p iggy_gateway_kafka -- -D warnings +cargo clippy -p iggy-gateway-kafka -- -D warnings ``` --- diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index b49419eb79..2c1d3b39b2 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -17,7 +17,7 @@ use bytes::Bytes; -use crate::error::{KafkaProtocolError, Result}; +use crate::error::Result; use crate::protocol::codec::{Decoder, Encoder}; use crate::protocol::requests::{ decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, @@ -44,6 +44,8 @@ pub const ERROR_UNSUPPORTED_VERSION: i16 = 35; pub const ERROR_INVALID_PARTITIONS: i16 = 37; pub const ERROR_INVALID_REQUEST: i16 = 42; +const MAX_SUPPORTED_METADATA_VERSION: i16 = 9; + /// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; @@ -126,7 +128,13 @@ pub fn handle_request( if is_supported_version(api_key, api_version) { encode_metadata_response(api_version, body, broker, ERROR_NONE) } else { - encode_metadata_response(api_version, body, broker, ERROR_UNSUPPORTED_VERSION) + // Encode at the highest version we implement, not the client's unknown version. + encode_metadata_response( + api_version.clamp(0, MAX_SUPPORTED_METADATA_VERSION), + body, + broker, + ERROR_UNSUPPORTED_VERSION, + ) } } API_KEY_PRODUCE => { @@ -248,16 +256,20 @@ fn encode_metadata_response( top_level_error_code: i16, ) -> Bytes { let flexible = api_version >= 9; - // BufferUnderflow (empty body) → treat as 0 topics; other decode errors are truly invalid. - let topics_count = match split_metadata_request_topics(body, api_version) { - Ok(n) => n, - Err(KafkaProtocolError::BufferUnderflow { .. }) => 0, - Err(_) => return encode_error_only_response(ERROR_INVALID_REQUEST), + // Empty body = all-topics request; 0 topics is correct for this stub. + // Non-empty body that fails to decode = malformed request; return 0 topics. + // Kafka Metadata response has no top-level error code field: errors are per-topic only. + // 0 topics is spec-correct and unambiguous for a decode failure. + let (topics_count, effective_error) = if body.is_empty() { + (0usize, top_level_error_code) + } else { + split_metadata_request_topics(body, api_version) + .map_or((0, ERROR_INVALID_REQUEST), |n| (n, top_level_error_code)) }; - let topic_error = if top_level_error_code == ERROR_NONE { + let topic_error = if effective_error == ERROR_NONE { ERROR_UNKNOWN_TOPIC_OR_PARTITION } else { - top_level_error_code + effective_error }; let mut e = Encoder::with_capacity(256); diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 6b313a67e8..a41d5fa9bd 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -87,6 +87,12 @@ impl BrokerAdvertise { "KAFKA_ADVERTISED_HOST must not be empty".into(), )); } + if trimmed.len() > i16::MAX as usize { + return Err(KafkaProtocolError::InvalidConfig( + "KAFKA_ADVERTISED_HOST exceeds Kafka nullable string limit (32767 bytes)" + .into(), + )); + } trimmed.to_string() } else if local_addr.ip().is_unspecified() { return Err(KafkaProtocolError::InvalidConfig( @@ -167,6 +173,9 @@ impl KafkaServer { accept_result = listener.accept() => { match accept_result { Ok((stream, peer)) => { + if let Err(e) = stream.set_nodelay(true) { + warn!(%peer, "TCP_NODELAY failed: {e}"); + } let cfg = Arc::clone(&self.config); let broker = Arc::clone(&broker); tracker.spawn(async move { @@ -350,12 +359,17 @@ pub async fn read_frame( }); } - // Reserve in 64 KB increments so a max-size frame (8 MB by default) does not trigger a - // single large upfront allocation before any payload bytes have arrived. - let mut data = BytesMut::new(); + // read_buf() exposes all BytesMut spare capacity to the OS; after reserve(n) the + // allocator may give more than n bytes, so the OS can fill past frame_len and silently + // consume bytes belonging to the next pipelined frame. Use read() with a bounded slice + // so each OS call is limited to exactly the remaining bytes needed. + let mut data = BytesMut::with_capacity(frame_len); while data.len() < frame_len { - data.reserve((frame_len - data.len()).min(READ_CHUNK)); - match timeout_at(deadline, stream.read_buf(&mut data)).await { + let remaining = frame_len - data.len(); + let chunk = remaining.min(READ_CHUNK); + let prev = data.len(); + data.resize(prev + chunk, 0); + let n = match timeout_at(deadline, stream.read(&mut data[prev..prev + chunk])).await { Err(_) => return Err(io::Error::new(io::ErrorKind::TimedOut, "read timeout").into()), Ok(Ok(0)) => { return Err( @@ -363,13 +377,10 @@ pub async fn read_frame( ); } Ok(Err(e)) => return Err(e.into()), - Ok(Ok(_)) => {} - } + Ok(Ok(n)) => n, + }; + data.truncate(prev + n); } - // read_buf may have written past frame_len if the OS returned more bytes than we - // reserved (capacity can round up). Truncate so pipelined frames don't bleed into - // decoder.read_bytes(remaining()) at the call site. - data.truncate(frame_len); Ok(data.freeze()) } diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs index 8f482019c4..a445540dd7 100644 --- a/gateways/kafka/tests/broker_advertise_tests.rs +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -85,6 +85,18 @@ fn from_server_config_uses_bind_ip_for_non_wildcard_listener() { assert_eq!(broker.port, 19092); } +#[test] +fn from_server_config_rejects_advertised_host_exceeding_kafka_string_limit() { + let config = ServerConfig { + bind_addr: "127.0.0.1:9093".to_string(), + advertised_host: Some("x".repeat(i16::MAX as usize + 1)), + ..ServerConfig::default() + }; + let local_addr: SocketAddr = "127.0.0.1:9093".parse().unwrap(); + let err = BrokerAdvertise::from_server_config(&config, local_addr).unwrap_err(); + assert!(err.to_string().contains("KAFKA_ADVERTISED_HOST")); +} + #[test] fn from_server_config_honors_advertised_port_override() { let config = ServerConfig { diff --git a/gateways/kafka/tests/metadata_regression_tests.rs b/gateways/kafka/tests/metadata_regression_tests.rs index 50c91bb1c4..1c33e38b7a 100644 --- a/gateways/kafka/tests/metadata_regression_tests.rs +++ b/gateways/kafka/tests/metadata_regression_tests.rs @@ -61,6 +61,20 @@ fn read_broker_flexible(d: &mut Decoder) -> (String, i32) { (host, port) } +#[test] +fn metadata_corrupt_partial_body_returns_zero_topics() { + let body = handle_request( + API_KEY_METADATA, + 0, + Bytes::from_static(&[0x00, 0x00]), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + #[test] fn metadata_v0_empty_topics_stub_broker() { let body = handle_request( diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs index 9de8f9b092..a8d6d921bc 100644 --- a/gateways/kafka/tests/server_integration_tests.rs +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -129,3 +129,28 @@ async fn write_frame_length_prefix_is_big_endian() { assert_eq!(len, 4); assert_eq!(&len_and_data[4..], &[1, 2, 3, 4]); } + +#[tokio::test] +async fn read_frame_does_not_consume_pipelined_frame_bytes() { + let (mut client, mut server) = tcp_pair().await; + + let payload1 = b"first-request-body-data"; + let payload2 = b"second-pipelined-request"; + + // Write both frames in a single syscall so the OS delivers them together. + // With the old read_buf approach, allocator rounding causes the first read_frame + // call to consume bytes from payload2, which truncate() then silently discards. + let mut both = BytesMut::with_capacity(4 + payload1.len() + 4 + payload2.len()); + both.put_i32(i32::try_from(payload1.len()).unwrap()); + both.extend_from_slice(payload1); + both.put_i32(i32::try_from(payload2.len()).unwrap()); + both.extend_from_slice(payload2); + client.write_all(&both).await.unwrap(); + + let timeout = Duration::from_secs(1); + let frame1 = read_frame(&mut server, 4096, timeout).await.unwrap(); + let frame2 = read_frame(&mut server, 4096, timeout).await.unwrap(); + + assert_eq!(&frame1[..], payload1); + assert_eq!(&frame2[..], payload2); +} From f89499c3b685fb4e8799c5a8cfeabb54e958d766 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:17:52 -0400 Subject: [PATCH 08/15] Normalize trailing newlines in GitHub configs Adjust trailing newline/whitespace in .github/config/publish.yml and .github/dependabot.yml. These are non-functional formatting fixes to normalize end-of-file newlines and do not change any configuration values. --- .github/config/publish.yml | 2 +- .github/dependabot.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/config/publish.yml b/.github/config/publish.yml index 36376d34c6..f549fe4376 100644 --- a/.github/config/publish.yml +++ b/.github/config/publish.yml @@ -146,4 +146,4 @@ components: tag_pattern: "^foreign/go/v([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" registry: none version_file: "foreign/go/contracts/version.go" - version_regex: 'const\s+Version\s*=\s*"([^"]+)"' + version_regex: 'const\s+Version\s*=\s*"([^"]+)"' \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b61effac1f..78c7f2e122 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -300,4 +300,4 @@ updates: groups: cpp: patterns: - - "*" + - "*" \ No newline at end of file From aa9b78a06848f12c81ae661d3c9bff9c7aefc525 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:46:03 -0400 Subject: [PATCH 09/15] Fixing the pre checks failures --- .github/config/components.yml | 2 +- .github/config/publish.yml | 2 +- .github/dependabot.yml | 2 +- gateways/README.md | 2 +- gateways/kafka/README.md | 2 +- gateways/kafka/docs/MANUAL_TESTING.md | 26 +++++++------- gateways/kafka/docs/SCOPE.md | 10 +++--- gateways/kafka/docs/TEST_SUITE.md | 16 ++++----- .../kafka/docs/kafka_api_keys_reference.md | 36 +++++++++---------- gateways/kafka/tools/kafka-tool/README.md | 8 +++-- 10 files changed, 54 insertions(+), 52 deletions(-) diff --git a/.github/config/components.yml b/.github/config/components.yml index d529ff4bbe..f1830562cd 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -501,4 +501,4 @@ components: - "sort" - "test-1" - "test-2" - - "machete" \ No newline at end of file + - "machete" diff --git a/.github/config/publish.yml b/.github/config/publish.yml index f549fe4376..36376d34c6 100644 --- a/.github/config/publish.yml +++ b/.github/config/publish.yml @@ -146,4 +146,4 @@ components: tag_pattern: "^foreign/go/v([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" registry: none version_file: "foreign/go/contracts/version.go" - version_regex: 'const\s+Version\s*=\s*"([^"]+)"' \ No newline at end of file + version_regex: 'const\s+Version\s*=\s*"([^"]+)"' diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 78c7f2e122..b61effac1f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -300,4 +300,4 @@ updates: groups: cpp: patterns: - - "*" \ No newline at end of file + - "*" diff --git a/gateways/README.md b/gateways/README.md index 7b0c550f5e..7fd825b4ae 100644 --- a/gateways/README.md +++ b/gateways/README.md @@ -3,7 +3,7 @@ Protocol gateways that let existing clients talk to Iggy without changing the core server wire surface. | Gateway | Issue | Description | -|---------|-------|-------------| +| --------- | ------- | ------------- | | [kafka](kafka/) | [#3421](https://github.com/apache/iggy/issues/3421) | Kafka wire protocol TCP listener (port 9093) | Each gateway is a separate workspace crate under `gateways//`. diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index b336887cd0..6c1e2d608f 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -11,7 +11,7 @@ cargo run -p iggy-gateway-kafka Default bind: `127.0.0.1:9093`. Environment variables: | Variable | Default | Description | -|---|---|---| +| --- | --- | --- | | `KAFKA_BIND_ADDR` | `127.0.0.1:9093` | TCP address to listen on | | `KAFKA_ADVERTISED_HOST` | bind IP | Hostname/IP clients use to reach this broker (required when binding to `0.0.0.0`/`::`) | | `KAFKA_ADVERTISED_PORT` | bind port | Port advertised in Metadata responses | diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index 30f37783c6..fecafb219d 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -11,7 +11,7 @@ See also: [SCOPE.md](SCOPE.md) (supported API keys), [TEST_SUITE.md](TEST_SUITE. ### Requirements | Tool | Purpose | Install | -|------|---------|---------| +| ------ | --------- | --------- | | Rust toolchain | Build gateway + kafka-tool | [rustup.rs](https://rustup.rs) | | `kafka-message-gen` | Generate/send wire fixtures | `cargo build -p kafka-message-gen` | | `kcat` (optional) | Real Kafka client smoke test | `brew install kcat` / `apt install kafkacat` | @@ -62,7 +62,7 @@ All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (ste ### Category A — Smoke tests (must pass before check-in) | ID | Test | Steps | Expected result | Pass criteria | -|----|------|-------|-----------------|---------------| +| ---- | ------ | ------- | ----------------- | --------------- | | A1 | Gateway starts | Run `iggy-gateway-kafka` | Binds to `:9093`, no panic | Log shows bind address | | A2 | ApiVersions v1 | `cargo run -p kafka-message-gen -- send --host 127.0.0.1:9093 --api-key 18 --version 1` | Response received | `ec=0`, non-zero byte count | | A3 | ApiVersions v3 (flexible) | Same with `--version 3` | Response received | `ec=0` | @@ -78,7 +78,7 @@ All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (ste For each API key, test **min−1**, **min**, **max**, **max+1** using `kafka-message-gen send` with `--version N`. | API key | Name | Min | Max | Test versions | -|---------|------|-----|-----|---------------| +| --------- | ------ | ----- | ----- | --------------- | | 18 | ApiVersions | 0 | 3 | −1, 0, 3, 4 | | 3 | Metadata | 0 | 9 | −1, 0, 9, 10 | | 0 | Produce | 3 | 9 | 2, 3, 9, 10 | @@ -87,7 +87,7 @@ For each API key, test **min−1**, **min**, **max**, **max+1** using `kafka-mes | 19 | CreateTopics | 2 | 5 | 1, 2, 5, 6 | | ID | Test | Expected for in-range | Expected for out-of-range | -|----|------|----------------------|---------------------------| +| ---- | ------ | ---------------------- | --------------------------- | | B1 | ApiVersions negotiation | `error_code=0`; body lists 6 API keys with correct min/max | `error_code=35` (UNSUPPORTED_VERSION) | | B2 | Metadata out-of-range | N/A | Topic entries show `error_code=35` | | B3 | Produce/Fetch/ListOffsets/CreateTopics out-of-range | N/A | Version-aware response with `error_code=35` (top-level or per-topic/partition) | @@ -102,7 +102,7 @@ cargo run -p kafka-message-gen -- generate --api-key 18 --version 3 --hex ### Category C — Unsupported API keys | ID | API key | Name | Steps | Expected | -|----|---------|------|-------|----------| +| ---- | --------- | ------ | ------- | ---------- | | C1 | 8 | OffsetCommit | `send --api-key 8 --version 2` | `ec=35`, connection stays open | | C2 | 10 | FindCoordinator | `send --api-key 10` | `ec=35` | | C3 | 17 | SaslHandshake | `send --api-key 17` | `ec=35` | @@ -113,7 +113,7 @@ Follow C1 with A2 on the **same** `nc` session to confirm the connection is not ### Category D — Flexible vs legacy wire encoding | ID | API key | Version | Encoding | Validation | -|----|---------|---------|----------|------------| +| ---- | --------- | --------- | ---------- | ------------ | | D1 | Produce | 8 | Legacy (i32 arrays) | `send` succeeds, `ec=0` | | D2 | Produce | 9 | Flexible (compact + tagged fields) | `send` succeeds, `ec=0` | | D3 | Fetch | 11 | Legacy | `send` succeeds | @@ -128,7 +128,7 @@ Follow C1 with A2 on the **same** `nc` session to confirm the connection is not ### Category E — Metadata stub semantics | ID | Test | Steps | Expected | -|----|------|-------|----------| +| ---- | ------ | ------- | ---------- | | E1 | Broker advertise address | Start gateway on `127.0.0.1:9093`; Metadata v0 | Broker host=`127.0.0.1`, port=`9093` | | E2 | Wildcard bind + advertised host | `KAFKA_BIND_ADDR=0.0.0.0:19093` + `KAFKA_ADVERTISED_HOST=kafka.internal`, restart | Metadata broker host/port match advertised values | | E3 | Unknown topic stub | Metadata with topic name `my-topic` | Topic error `3` (UNKNOWN_TOPIC_OR_PARTITION), name `unknown-topic` | @@ -137,7 +137,7 @@ Follow C1 with A2 on the **same** `nc` session to confirm the connection is not ### Category F — TCP / connection behavior | ID | Test | Steps | Expected | -|----|------|-------|----------| +| ---- | ------ | ------- | ---------- | | F1 | Correlation ID echoed | Send ApiVersions with known correlation_id; decode response header | Response correlation_id matches request | | F2 | Sequential requests | Send ApiVersions then Metadata on same TCP connection | Both get valid responses | | F3 | Client disconnect | Connect, send partial frame, close | Gateway logs clean disconnect, no panic | @@ -150,7 +150,7 @@ Follow C1 with A2 on the **same** `nc` session to confirm the connection is not Requires `kcat` installed. Gateway does **not** implement SASL or full broker semantics — expect limited success. | ID | Test | Command | Expected (foundation) | -|----|------|---------|---------------------| +| ---- | ------ | --------- | --------------------- | | G1 | Broker metadata | `kcat -b 127.0.0.1:9093 -L` | ApiVersions + Metadata handshake; broker appears in metadata | | G2 | Produce (likely fails later) | `echo "hello" \| kcat -b 127.0.0.1:9093 -t test -P` | May fail at coordinator/group stage — document actual error | | G3 | Consumer (likely fails later) | `kcat -b 127.0.0.1:9093 -t test -C -o beginning` | May fail without consumer groups — document actual error | @@ -160,7 +160,7 @@ Record kcat version and exact error strings in your test log. G1 passing is the ### Category H — Adversarial / negative input | ID | Test | Steps | Expected | -|----|------|-------|----------| +| ---- | ------ | ------- | ---------- | | H1 | Truncated Produce body | Send valid header + incomplete body | `error_code=42` (INVALID_REQUEST) or connection error; **no panic** | | H2 | Random bytes | `dd if=/dev/urandom bs=64 count=1 \| nc 127.0.0.1 9093` | Connection closed or protocol error; gateway stays up | | H3 | Empty body after header | ApiVersions with valid header, empty body | `ec=0` (ApiVersions accepts empty body) | @@ -172,7 +172,7 @@ Record kcat version and exact error strings in your test log. G1 passing is the ### Kafka error codes used in #3421 | Code | Name | When returned | -|------|------|---------------| +| ------ | ------ | --------------- | | 0 | NONE | Successful stub response | | 42 | INVALID_REQUEST | Produce/Fetch/ListOffsets/CreateTopics decode failure; unsupported request header | | 3 | UNKNOWN_TOPIC_OR_PARTITION | Metadata stub per-topic error | @@ -182,7 +182,7 @@ Record kcat version and exact error strings in your test log. G1 passing is the ### Response header rules | API key | Request flexible? | Response header version | -|---------|--------------------|-------------------------| +| --------- | -------------------- | ------------------------- | | 18 ApiVersions | v3+ | Always v0 (correlation_id only) | | 3 Metadata | v9+ | v1 (correlation_id + tagged fields) | | 0 Produce | v9+ | v1 | @@ -250,7 +250,7 @@ _________________________________ ## 6. Troubleshooting | Symptom | Likely cause | Fix | -|---------|--------------|-----| +| --------- | -------------- | ----- | | `Connection refused` on 9093 | Gateway not running | Start `iggy-gateway-kafka` | | `decode_validation_tests` panic | Missing fixtures | Run `kafka-message-gen generate` | | `ec=35` for in-range version | Version not in `SUPPORTED_RANGES` | Check `SCOPE.md` and `api.rs` | diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index 3f20fd411c..984b812470 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -5,7 +5,7 @@ Foundation layer only: a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, validates request wire formats, and returns stub responses. **No Iggy backend integration.** | Deliverable | Status | Location | -|-------------|--------|----------| +| ------------- | -------- | ---------- | | TCP listener on `127.0.0.1:9093` (configurable) | Done | `src/server.rs`, `src/main.rs` | | Length-prefixed frame read/write with `max_frame_size` cap | Done | `src/server.rs` | | Request header v1/v2 auto-detection | Done | `src/protocol/header.rs` | @@ -29,7 +29,7 @@ Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiV ## Supported API keys and versions | API key | Name | Min version | Max version | Valid versions | Behavior | -|---------|------|-------------|-------------|----------------|----------| +| --------- | ------ | ------------- | ------------- | ---------------- | ---------- | | 18 | ApiVersions | 0 | 3 | 0, 1, 2, 3 | Advertise supported ranges; flexible encoding at v3+ | | 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode topic list count; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | | 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub response | @@ -44,7 +44,7 @@ A request is accepted when `min_version ≤ api_version ≤ max_version` for tha Use this table when configuring clients or generating wire fixtures with `kafka-message-gen`. | API key | Name | Valid versions (inclusive range) | Flexible wire encoding from | -|---------|------|----------------------------------|----------------------------| +| --------- | ------ | ---------------------------------- | ---------------------------- | | 0 | Produce | 3–9 | v9 | | 1 | Fetch | 4–12 | v12 | | 2 | ListOffsets | 1–6 | v6 | @@ -59,7 +59,7 @@ Use this table when configuring clients or generating wire fixtures with `kafka- All API keys not listed above receive an error-only response with `UNSUPPORTED_VERSION` (35). Examples not in this foundation scope: | API key | Name | Notes | -|---------|------|-------| +| --------- | ------ | ------- | | 8 | OffsetCommit | Consumer group — later issue | | 9 | OffsetFetch | Consumer group — later issue | | 10 | FindCoordinator | Consumer group — later issue | @@ -74,7 +74,7 @@ Full reference for future phases: [`kafka_api_keys_reference.md`](kafka_api_keys ## Architecture (three layers) | Layer | #3421 | Description | -|-------|-------|-------------| +| ------- | ------- | ------------- | | **1 — Wire framing** | In scope | `server.rs`, `codec.rs`, `header.rs` — keep custom, zero-copy frame I/O | | **2 — Request/response codecs** | Partial | Custom minimal-parse codecs for 6 hot-path keys; stub responses only | | **3 — Iggy bridge** | Out of scope | Produce/Fetch → Iggy SDK; deferred to a follow-on issue | diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index c0f6df8c1e..27ed05a6ba 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -25,7 +25,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ## Test file catalog | File | Suite focus | Test count (approx.) | Depends on fixtures | -|------|-------------|----------------------|---------------------| +| ------ | ------------- | ---------------------- | --------------------- | | [`codec_tests.rs`](../tests/codec_tests.rs) | Primitive encode/decode round-trips, varint, compact strings, tagged fields | 9 | No | | [`decode_safety_tests.rs`](../tests/decode_safety_tests.rs) | Adversarial wire input — malformed lengths, truncated bodies | 6 | No | | [`header_tests.rs`](../tests/header_tests.rs) | Request/response header v1/v2, version lookup table | 10 | No | @@ -47,7 +47,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### ApiVersions (key 18, v0–v3) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Non-flexible response (v1) | `api_handler_tests` | `api_versions_v1_response_non_flexible_format` | | Flexible response (v3) | `api_handler_tests` | `api_versions_v3_response_flexible_format` | | Golden byte fixture (v1) | `golden_wire_fixtures_tests` | `golden_apiversions_v1_response_fixture` | @@ -59,7 +59,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### Metadata (key 3, v0–v9) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Stub broker (default 127.0.0.1:9093) | `api_handler_tests`, `metadata_regression_tests` | `metadata_response_has_broker_*`, `metadata_v0_empty_*` | | Unsupported version → topic error 35 | `api_handler_tests`, `version_firewall_tests` | `unsupported_version_returns_protocol_error`, `metadata_*_version_returns_topic_error` | | Golden byte fixture (v0, 1 topic) | `golden_wire_fixtures_tests` | `golden_metadata_v0_single_topic_response_fixture` | @@ -71,7 +71,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### Produce (key 0, v3–v9) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Decode all versions (fixture) | `decode_validation_tests` | `produce_all_supported_versions_decode` | | Response encode all versions | `decode_validation_tests` | `produce_response_encodes_for_all_supported_versions` | | v3 field layout | `decode_validation_tests` | `produce_response_v3_roundtrip` | @@ -84,7 +84,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### Fetch (key 1, v4–v12) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Decode all versions | `decode_validation_tests` | `fetch_all_supported_versions_decode` | | Response encode all versions | `decode_validation_tests` | `fetch_response_encodes_for_all_supported_versions` | | v7 session_id / error_code layout | `decode_validation_tests` | `fetch_response_v7_roundtrip` | @@ -95,7 +95,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### ListOffsets (key 2, v1–v6) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Decode all versions | `decode_validation_tests` | `list_offsets_all_supported_versions_decode` | | v1 no leader_epoch | `decode_validation_tests` | `list_offsets_response_v1_no_leader_epoch` | | v4 has leader_epoch | `decode_validation_tests` | `list_offsets_response_v4_has_leader_epoch` | @@ -105,7 +105,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### CreateTopics (key 19, v2–v5) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Decode all versions | `decode_validation_tests` | `create_topics_all_supported_versions_decode` | | v2 roundtrip | `decode_validation_tests` | `create_topics_response_v2_roundtrip` | | v5 flexible roundtrip | `decode_validation_tests` | `create_topics_response_v5_roundtrip` | @@ -117,7 +117,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ## Cross-cutting scenarios | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Version firewall min/max boundaries | `version_firewall_tests` | `is_supported_version_matches_scope_table` | | Unknown API keys (8, 9, 10, 17, 20, 999) | `version_firewall_tests`, `api_handler_tests` | `unsupported_api_keys_*`, `unknown_api_key_*` | | Negative i32 array length | `decode_safety_tests` | `negative_i32_array_length_returns_error_not_panic` | diff --git a/gateways/kafka/docs/kafka_api_keys_reference.md b/gateways/kafka/docs/kafka_api_keys_reference.md index c31cec2d7c..0cec397e46 100644 --- a/gateways/kafka/docs/kafka_api_keys_reference.md +++ b/gateways/kafka/docs/kafka_api_keys_reference.md @@ -11,7 +11,7 @@ ## Legend | Symbol | Meaning | -|--------|---------| +| -------- | --------- | | 🔴 Bridge | Core data path — must be fully implemented and forwarded to Iggy | | 🟠 Required Stub | Client state-machine API — must return a well-formed response or clients will stall/crash | | 🟡 Optional Stub | Admin/observability — can safely return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` | @@ -27,7 +27,7 @@ Kafka 4.0 removed all protocol versions older than Kafka 2.1.0 (KIP-896). Key new minimums: | API | Old Min | New Min (4.0) | -|-----|:-------:|:-------------:| +| ----- | :-------: | :-------------: | | Produce | 0 | 3 | | Fetch | 0 | 4 | | ListOffsets | 0 | 1 | @@ -46,7 +46,7 @@ Key new minimums: ## Group 1 — Core Data Path | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 0 | **Produce** | 3 †| 12 | v9 | ✅ | 🔴 Bridge | | 1 | **Fetch** | 4 | 17 | v12 | ✅ | 🔴 Bridge | | 2 | **ListOffsets** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | @@ -59,7 +59,7 @@ Key new minimums: ## Group 2 — API Negotiation & Auth | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 17 | **SaslHandshake** | 0 | 1 | never | ✅ | 🔴 Bridge (auth flow) | | 18 | **ApiVersions** | 0 | 4 | v3 | ✅ | 🔴 Bridge (advertise Iggy caps) | | 36 | **SaslAuthenticate** | 0 | 2 | v2 | ✅ | 🔴 Bridge (auth flow) | @@ -73,7 +73,7 @@ Key new minimums: ## Group 3 — Classic Consumer Group Protocol | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 8 | **OffsetCommit** | 2 | 9 | v8 | ✅ | 🟠 Required Stub | | 9 | **OffsetFetch** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | | 10 | **FindCoordinator** | 1 | 6 | v3 | ✅ | 🟠 Required Stub | @@ -90,7 +90,7 @@ Key new minimums: ## Group 4 — New Consumer Group Protocol (KIP-848, Kafka 3.7+) | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 68 | **ConsumerGroupHeartbeat** | 0 | 1 | v0 | ✅ | 🟠 Required Stub | | 69 | **ConsumerGroupDescribe** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | @@ -102,7 +102,7 @@ Key new minimums: ## Group 5 — Topic Administration | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 19 | **CreateTopics** | 2 | 7 | v5 | ✅ (max v5 ⚠️) | 🟠 Required Stub | | 20 | **DeleteTopics** | 1 | 6 | v4 | ✅ | 🟡 Optional Stub | | 21 | **DeleteRecords** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | @@ -115,7 +115,7 @@ Key new minimums: ## Group 6 — Transactions (EOS — Exactly Once Semantics) | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 22 | **InitProducerId** | 2 | 5 | v2 | ✅ | 🟡 Optional Stub | | 23 | **OffsetForLeaderEpoch** | 1 | 5 | v4 | ✅ | 🟡 Optional Stub | | 24 | **AddPartitionsToTxn** | 1 | 5 | v3 | ✅ | 🟡 Optional Stub | @@ -129,7 +129,7 @@ Key new minimums: ## Group 7 — Security & ACLs | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 29 | **DescribeAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | | 30 | **CreateAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | | 31 | **DeleteAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | @@ -145,7 +145,7 @@ Key new minimums: ## Group 8 — Configuration & Quotas | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 32 | **DescribeConfigs** | 0 | 4 | v4 | ✅ | 🟡 Optional Stub | | 33 | **AlterConfigs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | | 44 | **IncrementalAlterConfigs** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | @@ -157,7 +157,7 @@ Key new minimums: ## Group 9 — Log & Partition Admin | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 34 | **AlterReplicaLogDirs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | | 35 | **DescribeLogDirs** | 0 | 4 | v2 | ✅ | 🟡 Optional Stub | | 43 | **ElectLeaders** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | @@ -171,7 +171,7 @@ Key new minimums: ## Group 10 — Cluster Introspection | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 55 | **DescribeQuorum** | 0 | 2 | v0 | ✅ | 🟡 Optional Stub | | 59 | **FetchSnapshot** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | | 60 | **DescribeCluster** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | @@ -186,7 +186,7 @@ Key new minimums: ## Group 11 — Observability / Telemetry (KIP-714, Kafka 3.7+) | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 71 | **GetTelemetrySubscriptions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | | 72 | **PushTelemetry** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | | 76 | **ListClientMetricsResources** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | @@ -199,7 +199,7 @@ Key new minimums: > coordinator APIs and can be rejected. | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 77 | **ShareGroupHeartbeat** | 0 | 0 | v0 | ✅ | 🟠 Required Stub | | 78 | **ShareGroupDescribe** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | | 79 | **ShareFetch** | 0 | 0 | v0 | ✅ | 🔴 Bridge (share consume) | @@ -210,7 +210,7 @@ Key new minimums: ## Group 13 — KRaft Raft Voter Management (NEW in Kafka 4.0) | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 81 | **AddRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | | 82 | **RemoveRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | | 83 | **UpdateRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | @@ -223,7 +223,7 @@ Key new minimums: > Return `INVALID_REQUEST` (error code 42) with a properly framed response — **do not drop the connection**. | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 4 | **LeaderAndIsr** | 0 | 7 | v4 | ✅ | ❌ Reject (broker-only) | | 5 | **StopReplica** | 0 | 4 | v2 | ✅ | ❌ Reject (broker-only) | | 6 | **UpdateMetadata** | 0 | 8 | v6 | ✅ | ❌ Reject (broker-only) | @@ -249,7 +249,7 @@ Key new minimums: ## Summary Counts | Category | Count | Notes | -|----------|:-----:|-------| +| ---------- | :-----: | ------- | | 🔴 Bridge (data path) | 6 | Produce, Fetch, Metadata, ApiVersions, SaslHandshake, SaslAuthenticate, ShareFetch | | 🟠 Required Stub (client state machine) | 14 | Consumer group, CreateTopics, ConsumerGroupHeartbeat (68), ShareGroupHeartbeat (77), ShareAcknowledge (80) | | 🟡 Optional Stub (admin/observability) | 44 | Can return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` safely | @@ -263,7 +263,7 @@ Key new minimums: ### `SUPPORTED_RANGES` is behind the latest Kafka 4.0 max versions | API | Declared range | Kafka 4.0 max | Gap | -|-----|:---:|:---:|:---:| +| ----- | :---: | :---: | :---: | | Produce | v3-v9 | v12 | 3 versions behind | | Fetch | v4-v12 | v17 | 5 versions behind | | ListOffsets | v1-v6 | v9 | 3 versions behind | diff --git a/gateways/kafka/tools/kafka-tool/README.md b/gateways/kafka/tools/kafka-tool/README.md index a65eb2a005..678b43f2a6 100644 --- a/gateways/kafka/tools/kafka-tool/README.md +++ b/gateways/kafka/tools/kafka-tool/README.md @@ -56,6 +56,7 @@ cargo run -- list ``` Output: + ``` Key Name MinVer MaxVer Count ────────────────────────────────────────────────────────────────────────────── @@ -97,7 +98,7 @@ kafka_messages/ #### Options | Flag | Description | Default | -|------|-------------|---------| +| ------ | ------------- | --------- | | `--output` | Output directory | `kafka_messages/` | | `--api-key N` | Generate only for API key N | all | | `--version N` | Generate only for version N | all | @@ -121,6 +122,7 @@ cargo run -- send --host 127.0.0.1:9092 ``` Output (one line per API key × version): + ``` ✓ ApiVersions v3 → 32 bytes ec=0 ✓ Metadata v12 → 148 bytes ec=0 @@ -133,7 +135,7 @@ Result: 243 OK 37 failed #### Options | Flag | Description | Default | -|------|-------------|---------| +| ------ | ------------- | --------- | | `--host` | Server address | `127.0.0.1:9092` | | `--api-key N` | Test only API key N | all | | `--version N` | Test only version N | all | @@ -170,7 +172,7 @@ cat kafka_messages/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd | head ## Supported API Keys (Kafka 4.1.0) | Key | Name | Versions | Phase 1 Priority | -|-----|------|----------|-----------------| +| ----- | ------ | ---------- | ----------------- | | 0 | Produce | v3–v13 | ✅ Critical | | 1 | Fetch | v4–v18 | ✅ Critical | | 2 | ListOffsets | v1–v11 | ✅ Critical | From 831c3a26fc1c722ab84e523b64427dad57f62a03 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:48:48 -0400 Subject: [PATCH 10/15] Fixing the formatting of md files --- gateways/kafka/docs/MANUAL_TESTING.md | 6 +- .../kafka/docs/kafka_api_keys_reference.md | 2 +- gateways/kafka/tools/kafka-tool/README.md | 60 +++++++++---------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index fecafb219d..5d933a231a 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -30,7 +30,7 @@ RUST_LOG=info cargo run -p iggy-gateway-kafka Expected log: -``` +```text kafka listener bound on 127.0.0.1:9093 ``` @@ -191,7 +191,7 @@ Record kcat version and exact error strings in your test log. G1 passing is the ### Frame layout (for manual hex inspection) -``` +```text Request frame: [length: i32 BE] [api_key: i16][api_version: i16][correlation_id: i32] @@ -222,7 +222,7 @@ First bytes after length prefix should include your correlation_id from the fixt Copy this checklist into your PR or test log: -``` +```text Date: ___________ Tester: ___________ Gateway commit: ___________ diff --git a/gateways/kafka/docs/kafka_api_keys_reference.md b/gateways/kafka/docs/kafka_api_keys_reference.md index 0cec397e46..2555bf96af 100644 --- a/gateways/kafka/docs/kafka_api_keys_reference.md +++ b/gateways/kafka/docs/kafka_api_keys_reference.md @@ -47,7 +47,7 @@ Key new minimums: | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | | :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | -| 0 | **Produce** | 3 †| 12 | v9 | ✅ | 🔴 Bridge | +| 0 | **Produce** | 3 † | 12 | v9 | ✅ | 🔴 Bridge | | 1 | **Fetch** | 4 | 17 | v12 | ✅ | 🔴 Bridge | | 2 | **ListOffsets** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | | 3 | **Metadata** | 1 | 12 | v9 | ✅ | 🔴 Bridge | diff --git a/gateways/kafka/tools/kafka-tool/README.md b/gateways/kafka/tools/kafka-tool/README.md index 678b43f2a6..eb44ee3759 100644 --- a/gateways/kafka/tools/kafka-tool/README.md +++ b/gateways/kafka/tools/kafka-tool/README.md @@ -6,7 +6,7 @@ A Rust CLI tool that generates correct, fully-framed Kafka binary wire protocol Each output `.bin` file is a complete, TCP-ready Kafka request: -``` +```text [total_length: i32][api_key: i16][api_version: i16] [correlation_id: i32][client_id: NULLABLE_STRING] [tagged_fields: 0x00] ← only for flexible versions @@ -57,7 +57,7 @@ cargo run -- list Output: -``` +```text Key Name MinVer MaxVer Count ────────────────────────────────────────────────────────────────────────────── 0 Produce 3 13 11 @@ -80,7 +80,7 @@ cargo run -- generate --output ./kafka_messages/ Creates one `.bin` file per API key × version: -``` +```text kafka_messages/ 000_Produce_v3.bin 000_Produce_v4.bin @@ -123,7 +123,7 @@ cargo run -- send --host 127.0.0.1:9092 Output (one line per API key × version): -``` +```text ✓ ApiVersions v3 → 32 bytes ec=0 ✓ Metadata v12 → 148 bytes ec=0 ⚠ Produce v9 → 24 bytes ec=3 ← ec=3 = UnknownTopicOrPartition (expected) @@ -173,39 +173,39 @@ cat kafka_messages/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd | head | Key | Name | Versions | Phase 1 Priority | | ----- | ------ | ---------- | ----------------- | -| 0 | Produce | v3–v13 | ✅ Critical | -| 1 | Fetch | v4–v18 | ✅ Critical | -| 2 | ListOffsets | v1–v11 | ✅ Critical | -| 3 | Metadata | v0–v13 | ✅ Critical | -| 8 | OffsetCommit | v2–v10 | ✅ Critical | -| 9 | OffsetFetch | v1–v10 | ✅ Critical | -| 10 | FindCoordinator | v0–v6 | ✅ Critical | -| 11 | JoinGroup | v0–v9 | ✅ Critical | -| 12 | Heartbeat | v0–v4 | ✅ Critical | -| 13 | LeaveGroup | v0–v5 | ✅ Critical | -| 14 | SyncGroup | v0–v5 | ✅ Critical | -| 15 | DescribeGroups | v0–v6 | 🟡 Important | -| 16 | ListGroups | v0–v5 | 🟡 Important | -| 17 | SaslHandshake | v0–v1 | 🟡 Important | -| 18 | ApiVersions | v0–v5 | ✅ Critical | -| 19 | CreateTopics | v2–v7 | ✅ Critical | -| 20 | DeleteTopics | v1–v6 | 🟡 Important | -| 21 | DeleteRecords | v0–v2 | 🔵 Phase 2 | -| 22 | InitProducerId | v0–v6 | 🔵 Phase 2 | -| 24 | AddPartitionsToTxn | v0–v5 | 🔵 Phase 2 | -| 25 | AddOffsetsToTxn | v0–v4 | 🔵 Phase 2 | -| 26 | EndTxn | v0–v5 | 🔵 Phase 2 | -| 28 | TxnOffsetCommit | v0–v5 | 🔵 Phase 2 | +| 0 | Produce | v3–v13 | ✅ Critical | +| 1 | Fetch | v4–v18 | ✅ Critical | +| 2 | ListOffsets | v1–v11 | ✅ Critical | +| 3 | Metadata | v0–v13 | ✅ Critical | +| 8 | OffsetCommit | v2–v10 | ✅ Critical | +| 9 | OffsetFetch | v1–v10 | ✅ Critical | +| 10 | FindCoordinator | v0–v6 | ✅ Critical | +| 11 | JoinGroup | v0–v9 | ✅ Critical | +| 12 | Heartbeat | v0–v4 | ✅ Critical | +| 13 | LeaveGroup | v0–v5 | ✅ Critical | +| 14 | SyncGroup | v0–v5 | ✅ Critical | +| 15 | DescribeGroups | v0–v6 | 🟡 Important | +| 16 | ListGroups | v0–v5 | 🟡 Important | +| 17 | SaslHandshake | v0–v1 | 🟡 Important | +| 18 | ApiVersions | v0–v5 | ✅ Critical | +| 19 | CreateTopics | v2–v7 | ✅ Critical | +| 20 | DeleteTopics | v1–v6 | 🟡 Important | +| 21 | DeleteRecords | v0–v2 | 🔵 Phase 2 | +| 22 | InitProducerId | v0–v6 | 🔵 Phase 2 | +| 24 | AddPartitionsToTxn | v0–v5 | 🔵 Phase 2 | +| 25 | AddOffsetsToTxn | v0–v4 | 🔵 Phase 2 | +| 26 | EndTxn | v0–v5 | 🔵 Phase 2 | +| 28 | TxnOffsetCommit | v0–v5 | 🔵 Phase 2 | | 29–31 | ACL APIs | v1–v3 | 🔵 Phase 2 | -| 32 | DescribeConfigs | v1–v4 | 🟡 Important | -| 36 | SaslAuthenticate | v0–v2 | 🟡 Important | +| 32 | DescribeConfigs | v1–v4 | 🟡 Important | +| 36 | SaslAuthenticate | v0–v2 | 🟡 Important | | ... | 40+ more | various | 🔵 Phase 3 | --- ## Project Structure -``` +```text tools/kafka-tool/ ├── Cargo.toml ← package manifest and dependencies ├── src/ From 5ac045718fcd0041f12e37b86d0d268d35d3b884 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:53:56 -0400 Subject: [PATCH 11/15] Update Cargo.toml --- gateways/kafka/Cargo.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 88c0e8a2dc..20618711e2 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -35,7 +35,15 @@ path = "src/main.rs" [dependencies] bytes = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time", "sync", "signal"] } +tokio = { workspace = true, features = [ + "rt-multi-thread", + "macros", + "net", + "io-util", + "time", + "sync", + "signal", +] } tokio-util = { workspace = true, features = ["rt"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } From 0661ec9099bd1f43904ad9d9a63f8c70e0fc870f Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 23:06:49 -0400 Subject: [PATCH 12/15] Fixing pre checks issues --- Cargo.lock | 2 -- gateways/kafka/tools/kafka-tool/Cargo.toml | 2 -- 2 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7bd874ff7f..bf7442c1c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7588,8 +7588,6 @@ dependencies = [ "hex", "indexmap 2.14.0", "kafka-protocol", - "serde", - "serde_json", "tokio", "tracing", "tracing-subscriber", diff --git a/gateways/kafka/tools/kafka-tool/Cargo.toml b/gateways/kafka/tools/kafka-tool/Cargo.toml index a73d8addd9..41473e434a 100644 --- a/gateways/kafka/tools/kafka-tool/Cargo.toml +++ b/gateways/kafka/tools/kafka-tool/Cargo.toml @@ -36,8 +36,6 @@ clap = { workspace = true } hex = "0.4" indexmap = "2" kafka-protocol = "0.17" -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } From f3588935b2d80c43c129cc1dd5e512cdc6dbb72e Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 23:37:51 -0400 Subject: [PATCH 13/15] Fixing the test fixtures as part of pre merge flow and updating the documentation. --- .github/actions/rust/pre-merge/action.yml | 12 ++++- gateways/kafka/README.md | 18 ++++++-- gateways/kafka/docs/TEST_SUITE.md | 6 +-- gateways/kafka/scripts/ci-wire-fixtures.sh | 53 ++++++++++++++++++++++ 4 files changed, 79 insertions(+), 10 deletions(-) create mode 100755 gateways/kafka/scripts/ci-wire-fixtures.sh diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 43175bbb0c..dc9beed5a1 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -44,7 +44,7 @@ runs: # Light legs fit in the runner's ~89 GiB default headroom; skip the # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers # images on test-*, cross-builds, miri, verify-publish) keep it. - free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }} + free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'machete' || inputs.task == 'doctest' || (startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway')) && 'false' || 'true' }} - name: Install cargo-sort if: inputs.task == 'sort' @@ -179,6 +179,11 @@ runs: with: tool: cargo-llvm-cov + - name: Generate Kafka gateway wire fixtures + if: startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway' + run: ./gateways/kafka/scripts/ci-wire-fixtures.sh generate + shell: bash + - name: Build and test with coverage if: startsWith(inputs.task, 'test-') run: | @@ -308,6 +313,11 @@ runs: ls -la codecov.json shell: bash + - name: Remove Kafka gateway wire fixtures + if: always() && startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway' + run: ./gateways/kafka/scripts/ci-wire-fixtures.sh cleanup + shell: bash + - name: Backwards compatibility check if: inputs.task == 'compat' && (github.event_name != 'pull_request' || !contains(join(github.event.pull_request.labels.*.name, ','), 'breaking:storage')) run: | diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index 6c1e2d608f..3843706888 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -24,15 +24,23 @@ cargo test -p iggy-gateway-kafka 103 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. -`decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/`: +`decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/` (gitignored locally; CI generates them via `scripts/ci-wire-fixtures.sh`): ```bash -cargo run -p kafka-message-gen -- generate \ - --output gateways/kafka/tools/kafka-tool/kafka_messages \ - --api-key 0 --api-key 1 --api-key 2 --api-key 19 +./gateways/kafka/scripts/ci-wire-fixtures.sh generate +cargo test -p iggy-gateway-kafka +./gateways/kafka/scripts/ci-wire-fixtures.sh cleanup # optional ``` -(Run from workspace root; adjust paths if needed.) +Or generate only the keys the tests need: + +```bash +for key in 0 1 2 19; do + cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key "$key" +done +``` ## Manual testing diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index 27ed05a6ba..74dd995268 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -13,12 +13,10 @@ cargo test -p iggy-gateway-kafka ### Wire fixtures (required for `decode_validation_tests` and some handler tests) ```bash -cargo run -p kafka-message-gen -- generate \ - --output gateways/kafka/tools/kafka-tool/kafka_messages \ - --api-key 0 --api-key 1 --api-key 2 --api-key 19 +./gateways/kafka/scripts/ci-wire-fixtures.sh generate ``` -Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that need them skip gracefully when a fixture file is missing (`handler_regression_tests`) or panic with a clear path (`decode_validation_tests`). +Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the same script before `rust-gateway` test jobs and removes the directory afterward. Tests that need fixtures skip gracefully when a file is missing (`handler_regression_tests`) or panic with a clear path (`decode_validation_tests`). --- diff --git a/gateways/kafka/scripts/ci-wire-fixtures.sh b/gateways/kafka/scripts/ci-wire-fixtures.sh new file mode 100755 index 0000000000..0cdb87aefd --- /dev/null +++ b/gateways/kafka/scripts/ci-wire-fixtures.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generate or remove gitignored kafka-tool wire fixtures for CI. +# Run from the iggy workspace root. + +set -euo pipefail + +FIXTURES_DIR="gateways/kafka/tools/kafka-tool/kafka_messages" + +# API keys exercised by decode_validation_tests and handler_regression_tests. +FIXTURE_API_KEYS=(0 1 2 19) + +usage() { + echo "Usage: $0 {generate|cleanup}" >&2 + exit 2 +} + +generate() { + mkdir -p "$FIXTURES_DIR" + for key in "${FIXTURE_API_KEYS[@]}"; do + cargo run --locked -p kafka-message-gen -- generate \ + --output "$FIXTURES_DIR" \ + --api-key "$key" + done + echo "Generated wire fixtures under ${FIXTURES_DIR}/" +} + +cleanup() { + rm -rf "$FIXTURES_DIR" + echo "Removed ${FIXTURES_DIR}/" +} + +case "${1:-}" in + generate) generate ;; + cleanup) cleanup ;; + *) usage ;; +esac From d09e017f2b088e942f0f1d774c415d4004f24d43 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 23:48:13 -0400 Subject: [PATCH 14/15] Update action.yml --- .github/actions/rust/pre-merge/action.yml | 26 +++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index dc9beed5a1..088748a71d 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -44,7 +44,7 @@ runs: # Light legs fit in the runner's ~89 GiB default headroom; skip the # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers # images on test-*, cross-builds, miri, verify-publish) keep it. - free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'machete' || inputs.task == 'doctest' || (startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway')) && 'false' || 'true' }} + free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }} - name: Install cargo-sort if: inputs.task == 'sort' @@ -179,11 +179,6 @@ runs: with: tool: cargo-llvm-cov - - name: Generate Kafka gateway wire fixtures - if: startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway' - run: ./gateways/kafka/scripts/ci-wire-fixtures.sh generate - shell: bash - - name: Build and test with coverage if: startsWith(inputs.task, 'test-') run: | @@ -254,6 +249,20 @@ runs: compile_duration=$((compile_end - compile_start)) echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" + # decode_validation_tests need gitignored wire fixtures. Generate when + # iggy-gateway-kafka is in the DAG test scope (rust-gateway job or parent + # rust job — both run gateway tests when gateways/** changes). + NEEDS_KAFKA_FIXTURES=false + if [[ -z "$NEXTEST_FILTER" ]]; then + NEEDS_KAFKA_FIXTURES=true + elif grep -q 'package(iggy-gateway-kafka)' <<< "$NEXTEST_FILTER"; then + NEEDS_KAFKA_FIXTURES=true + fi + if [[ "$NEEDS_KAFKA_FIXTURES" == true ]]; then + ./gateways/kafka/scripts/ci-wire-fixtures.sh generate + trap './gateways/kafka/scripts/ci-wire-fixtures.sh cleanup' EXIT + fi + # Start D-Bus and unlock keyring right before test execution to avoid # gnome-keyring auto-locking the collection during the build phase. # Previously this ran before `cargo build`, leaving a 7+ minute idle @@ -313,11 +322,6 @@ runs: ls -la codecov.json shell: bash - - name: Remove Kafka gateway wire fixtures - if: always() && startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway' - run: ./gateways/kafka/scripts/ci-wire-fixtures.sh cleanup - shell: bash - - name: Backwards compatibility check if: inputs.task == 'compat' && (github.event_name != 'pull_request' || !contains(join(github.event.pull_request.labels.*.name, ','), 'breaking:storage')) run: | From eaf9fcbcdf20d61377ad759f7cf7aada2e061afc Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 21 Jun 2026 07:42:39 -0400 Subject: [PATCH 15/15] ci: retrigger checks Co-authored-by: Cursor