diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 43175bbb0c..088748a71d 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -249,6 +249,20 @@ runs: compile_duration=$((compile_end - compile_start)) echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" + # decode_validation_tests need gitignored wire fixtures. Generate when + # iggy-gateway-kafka is in the DAG test scope (rust-gateway job or parent + # rust job — both run gateway tests when gateways/** changes). + NEEDS_KAFKA_FIXTURES=false + if [[ -z "$NEXTEST_FILTER" ]]; then + NEEDS_KAFKA_FIXTURES=true + elif grep -q 'package(iggy-gateway-kafka)' <<< "$NEXTEST_FILTER"; then + NEEDS_KAFKA_FIXTURES=true + fi + if [[ "$NEEDS_KAFKA_FIXTURES" == true ]]; then + ./gateways/kafka/scripts/ci-wire-fixtures.sh generate + trap './gateways/kafka/scripts/ci-wire-fixtures.sh cleanup' EXIT + fi + # Start D-Bus and unlock keyring right before test execution to avoid # gnome-keyring auto-locking the collection during the build phase. # Previously this ran before `cargo build`, leaving a 7+ minute idle diff --git a/.github/config/components.yml b/.github/config/components.yml index 79b11a255c..f1830562cd 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -146,6 +146,7 @@ components: - "rust-connectors" - "rust-mcp" - "rust-integration" + - "rust-gateway" - "ci-infrastructure" paths: - "Dockerfile*" @@ -484,3 +485,20 @@ components: - ".github/actions/**/*.yml" - ".github/ci/**/*.yml" tasks: ["validate"] # Could run workflow validation + + # gateways are not Rust components, but we want to run them in CI + rust-gateway: + depends_on: + - "rust-sdk" + - "rust-workspace" + - "ci-infrastructure" + paths: + - "gateways/**" + tasks: + - "check" + - "fmt" + - "clippy" + - "sort" + - "test-1" + - "test-2" + - "machete" diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 57a6bbc2e4..f485734caa 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -95,3 +95,4 @@ jobs: storage simulator configs + gateways diff --git a/.gitignore b/.gitignore index f0d8b6250b..8c9ec60fc6 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ go.work core/bench/dashboard/frontend/dist LICENSE-binary **/LICENSE-binary +gateways/kafka/tools/kafka-tool/kafka_messages/ diff --git a/Cargo.lock b/Cargo.lock index 12382acfca..3804abe767 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6689,6 +6689,18 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "iggy-gateway-kafka" +version = "0.1.0" +dependencies = [ + "bytes", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", +] + [[package]] name = "iggy-mcp" version = "0.4.1-edge.1" @@ -7587,6 +7599,40 @@ dependencies = [ "rayon", ] +[[package]] +name = "kafka-message-gen" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "clap", + "hex", + "indexmap 2.14.0", + "kafka-protocol", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "kafka-protocol" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66292444a1cd4d430d450d472c30cba839d0724229aba2d79affffcf901516e2" +dependencies = [ + "anyhow", + "bytes", + "crc", + "crc32c", + "flate2", + "indexmap 2.14.0", + "lz4", + "paste", + "snap", + "uuid", + "zstd", +] + [[package]] name = "keccak" version = "0.2.0" @@ -8038,6 +8084,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lz4_flex" version = "0.12.2" diff --git a/Cargo.toml b/Cargo.toml index 4c6777fdea..e00992c6e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ members = [ "core/simulator", "core/tools", "examples/rust", + "gateways/kafka", + "gateways/kafka/tools/kafka-tool", ] exclude = ["foreign/cpp", "foreign/php", "foreign/python"] resolver = "2" diff --git a/gateways/README.md b/gateways/README.md new file mode 100644 index 0000000000..7fd825b4ae --- /dev/null +++ b/gateways/README.md @@ -0,0 +1,9 @@ +# Apache Iggy Gateways + +Protocol gateways that let existing clients talk to Iggy without changing the core server wire surface. + +| Gateway | Issue | Description | +| --------- | ------- | ------------- | +| [kafka](kafka/) | [#3421](https://github.com/apache/iggy/issues/3421) | Kafka wire protocol TCP listener (port 9093) | + +Each gateway is a separate workspace crate under `gateways//`. diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml new file mode 100644 index 0000000000..20618711e2 --- /dev/null +++ b/gateways/kafka/Cargo.toml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "iggy-gateway-kafka" +version = "0.1.0" +description = "Kafka wire protocol gateway foundation for Apache Iggy" +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "kafka", "gateway", "streaming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +readme = "README.md" +publish = false + +[[bin]] +name = "iggy-gateway-kafka" +path = "src/main.rs" + +[dependencies] +bytes = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = [ + "rt-multi-thread", + "macros", + "net", + "io-util", + "time", + "sync", + "signal", +] } +tokio-util = { workspace = true, features = ["rt"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time"] } + +[lints.clippy] +enum_glob_use = "deny" +#Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. +pedantic = "warn" +nursery = "warn" diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md new file mode 100644 index 0000000000..3843706888 --- /dev/null +++ b/gateways/kafka/README.md @@ -0,0 +1,55 @@ +# Kafka gateway (`iggy-gateway-kafka`) + +Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421): a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, and returns stub responses. + +## Run + +```bash +cargo run -p iggy-gateway-kafka +``` + +Default bind: `127.0.0.1:9093`. Environment variables: + +| Variable | Default | Description | +| --- | --- | --- | +| `KAFKA_BIND_ADDR` | `127.0.0.1:9093` | TCP address to listen on | +| `KAFKA_ADVERTISED_HOST` | bind IP | Hostname/IP clients use to reach this broker (required when binding to `0.0.0.0`/`::`) | +| `KAFKA_ADVERTISED_PORT` | bind port | Port advertised in Metadata responses | + +## Test + +```bash +cargo test -p iggy-gateway-kafka +``` + +103 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. + +`decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/` (gitignored locally; CI generates them via `scripts/ci-wire-fixtures.sh`): + +```bash +./gateways/kafka/scripts/ci-wire-fixtures.sh generate +cargo test -p iggy-gateway-kafka +./gateways/kafka/scripts/ci-wire-fixtures.sh cleanup # optional +``` + +Or generate only the keys the tests need: + +```bash +for key in 0 1 2 19; do + cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key "$key" +done +``` + +## Manual testing + +Before check-in, run the procedure in [docs/MANUAL_TESTING.md](docs/MANUAL_TESTING.md) (smoke, version firewall, kcat, adversarial cases). + +## Scoped APIs + +See [docs/SCOPE.md](docs/SCOPE.md) for [#3421](https://github.com/apache/iggy/issues/3421) deliverables, supported API key/version table, and post-foundation TODO backlog. + +## Wire fixture tool + +See [tools/kafka-tool/README.md](tools/kafka-tool/README.md). diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md new file mode 100644 index 0000000000..5d933a231a --- /dev/null +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -0,0 +1,272 @@ +# Kafka gateway — manual testing procedure + +Manual validation for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421) foundation: TCP listener, wire decode, version firewall, stub responses. **No Iggy backend** — success means correct Kafka wire behavior, not message persistence. + +See also: [SCOPE.md](SCOPE.md) (supported API keys), [TEST_SUITE.md](TEST_SUITE.md) (automated coverage). + +--- + +## 1. Environment setup + +### Requirements + +| Tool | Purpose | Install | +| ------ | --------- | --------- | +| Rust toolchain | Build gateway + kafka-tool | [rustup.rs](https://rustup.rs) | +| `kafka-message-gen` | Generate/send wire fixtures | `cargo build -p kafka-message-gen` | +| `kcat` (optional) | Real Kafka client smoke test | `brew install kcat` / `apt install kafkacat` | +| `nc` / `netcat` (optional) | Raw byte injection | Usually preinstalled | +| `xxd` or `hexdump` (optional) | Inspect binary responses | Usually preinstalled | + +### Build and start gateway + +```bash +# From iggy workspace root +cargo build -p iggy-gateway-kafka + +# Terminal 1 — start listener (default 127.0.0.1:9093) +RUST_LOG=info cargo run -p iggy-gateway-kafka +``` + +Expected log: + +```text +kafka listener bound on 127.0.0.1:9093 +``` + +### Generate wire fixtures + +```bash +# Terminal 2 +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 3 --api-key 18 --api-key 19 +``` + +--- + +## 2. Pre-flight automated check + +Run before manual testing to catch regressions: + +```bash +cargo test -p iggy-gateway-kafka +``` + +All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (step above). + +--- + +## 3. Manual test cases + +### Category A — Smoke tests (must pass before check-in) + +| ID | Test | Steps | Expected result | Pass criteria | +| ---- | ------ | ------- | ----------------- | --------------- | +| A1 | Gateway starts | Run `iggy-gateway-kafka` | Binds to `:9093`, no panic | Log shows bind address | +| A2 | ApiVersions v1 | `cargo run -p kafka-message-gen -- send --host 127.0.0.1:9093 --api-key 18 --version 1` | Response received | `ec=0`, non-zero byte count | +| A3 | ApiVersions v3 (flexible) | Same with `--version 3` | Response received | `ec=0` | +| A4 | Metadata v0 | `send --api-key 3 --version 0` | Stub broker in response | `ec=0` or topic error 3 (stub) | +| A5 | Produce v3 | `send --api-key 0 --version 3` | Decode + stub ack | `ec=0` | +| A6 | Fetch v4 | `send --api-key 1 --version 4` | Decode + stub response | `ec=0` | +| A7 | ListOffsets v1 | `send --api-key 2 --version 1` | Decode + stub offsets | `ec=0` | +| A8 | CreateTopics v2 | `send --api-key 19 --version 2` | Decode + stub ack | `ec=0` | +| A9 | Verify all scoped keys | `cargo run -p kafka-message-gen -- verify --host 127.0.0.1:9093 --api-key 0 --api-key 1 --api-key 2 --api-key 3 --api-key 18 --api-key 19` | Exit code 0 | No timeouts or I/O errors | + +### Category B — Version firewall (boundary validation) + +For each API key, test **min−1**, **min**, **max**, **max+1** using `kafka-message-gen send` with `--version N`. + +| API key | Name | Min | Max | Test versions | +| --------- | ------ | ----- | ----- | --------------- | +| 18 | ApiVersions | 0 | 3 | −1, 0, 3, 4 | +| 3 | Metadata | 0 | 9 | −1, 0, 9, 10 | +| 0 | Produce | 3 | 9 | 2, 3, 9, 10 | +| 1 | Fetch | 4 | 12 | 3, 4, 12, 13 | +| 2 | ListOffsets | 1 | 6 | 0, 1, 6, 7 | +| 19 | CreateTopics | 2 | 5 | 1, 2, 5, 6 | + +| ID | Test | Expected for in-range | Expected for out-of-range | +| ---- | ------ | ---------------------- | --------------------------- | +| B1 | ApiVersions negotiation | `error_code=0`; body lists 6 API keys with correct min/max | `error_code=35` (UNSUPPORTED_VERSION) | +| B2 | Metadata out-of-range | N/A | Topic entries show `error_code=35` | +| B3 | Produce/Fetch/ListOffsets/CreateTopics out-of-range | N/A | Version-aware response with `error_code=35` (top-level or per-topic/partition) | +| B4 | ApiVersions lists only scoped keys | Decode response | Contains keys 0,1,2,3,18,19 only — no consumer-group keys | + +**Validation tip:** Use `--hex` when generating to inspect request bytes: + +```bash +cargo run -p kafka-message-gen -- generate --api-key 18 --version 3 --hex +``` + +### Category C — Unsupported API keys + +| ID | API key | Name | Steps | Expected | +| ---- | --------- | ------ | ------- | ---------- | +| C1 | 8 | OffsetCommit | `send --api-key 8 --version 2` | `ec=35`, connection stays open | +| C2 | 10 | FindCoordinator | `send --api-key 10` | `ec=35` | +| C3 | 17 | SaslHandshake | `send --api-key 17` | `ec=35` | +| C4 | 20 | DeleteTopics | `send --api-key 20` | `ec=35` | + +Follow C1 with A2 on the **same** `nc` session to confirm the connection is not dropped. + +### Category D — Flexible vs legacy wire encoding + +| ID | API key | Version | Encoding | Validation | +| ---- | --------- | --------- | ---------- | ------------ | +| D1 | Produce | 8 | Legacy (i32 arrays) | `send` succeeds, `ec=0` | +| D2 | Produce | 9 | Flexible (compact + tagged fields) | `send` succeeds, `ec=0` | +| D3 | Fetch | 11 | Legacy | `send` succeeds | +| D4 | Fetch | 12 | Flexible | `send` succeeds | +| D5 | Metadata | 8 | Legacy | `send` succeeds | +| D6 | Metadata | 9 | Flexible | `send` succeeds | +| D7 | ListOffsets | 5 | Legacy | `send` succeeds | +| D8 | ListOffsets | 6 | Flexible | `send` succeeds | +| D9 | CreateTopics | 4 | Legacy | `send` succeeds | +| D10 | CreateTopics | 5 | Flexible | `send` succeeds | + +### Category E — Metadata stub semantics + +| ID | Test | Steps | Expected | +| ---- | ------ | ------- | ---------- | +| E1 | Broker advertise address | Start gateway on `127.0.0.1:9093`; Metadata v0 | Broker host=`127.0.0.1`, port=`9093` | +| E2 | Wildcard bind + advertised host | `KAFKA_BIND_ADDR=0.0.0.0:19093` + `KAFKA_ADVERTISED_HOST=kafka.internal`, restart | Metadata broker host/port match advertised values | +| E3 | Unknown topic stub | Metadata with topic name `my-topic` | Topic error `3` (UNKNOWN_TOPIC_OR_PARTITION), name `unknown-topic` | +| E4 | Multiple topics | Metadata request listing 3 topics | 3 topic entries, each with error 3 | + +### Category F — TCP / connection behavior + +| ID | Test | Steps | Expected | +| ---- | ------ | ------- | ---------- | +| F1 | Correlation ID echoed | Send ApiVersions with known correlation_id; decode response header | Response correlation_id matches request | +| F2 | Sequential requests | Send ApiVersions then Metadata on same TCP connection | Both get valid responses | +| F3 | Client disconnect | Connect, send partial frame, close | Gateway logs clean disconnect, no panic | +| F4 | Invalid frame length 0 | `printf '\x00\x00\x00\x00' \| nc 127.0.0.1 9093` | Connection closed, gateway continues serving others | +| F5 | Oversized frame | Send 4-byte length > 8 MiB | Connection rejected/closed, no OOM | +| F6 | Graceful shutdown | Ctrl+C on gateway | Log "shutdown requested", in-flight requests drain | + +### Category G — Real Kafka client (kcat) + +Requires `kcat` installed. Gateway does **not** implement SASL or full broker semantics — expect limited success. + +| ID | Test | Command | Expected (foundation) | +| ---- | ------ | --------- | --------------------- | +| G1 | Broker metadata | `kcat -b 127.0.0.1:9093 -L` | ApiVersions + Metadata handshake; broker appears in metadata | +| G2 | Produce (likely fails later) | `echo "hello" \| kcat -b 127.0.0.1:9093 -t test -P` | May fail at coordinator/group stage — document actual error | +| G3 | Consumer (likely fails later) | `kcat -b 127.0.0.1:9093 -t test -C -o beginning` | May fail without consumer groups — document actual error | + +Record kcat version and exact error strings in your test log. G1 passing is the minimum bar for client compatibility smoke. + +### Category H — Adversarial / negative input + +| ID | Test | Steps | Expected | +| ---- | ------ | ------- | ---------- | +| H1 | Truncated Produce body | Send valid header + incomplete body | `error_code=42` (INVALID_REQUEST) or connection error; **no panic** | +| H2 | Random bytes | `dd if=/dev/urandom bs=64 count=1 \| nc 127.0.0.1 9093` | Connection closed or protocol error; gateway stays up | +| H3 | Empty body after header | ApiVersions with valid header, empty body | `ec=0` (ApiVersions accepts empty body) | + +--- + +## 4. Validation reference + +### Kafka error codes used in #3421 + +| Code | Name | When returned | +| ------ | ------ | --------------- | +| 0 | NONE | Successful stub response | +| 42 | INVALID_REQUEST | Produce/Fetch/ListOffsets/CreateTopics decode failure; unsupported request header | +| 3 | UNKNOWN_TOPIC_OR_PARTITION | Metadata stub per-topic error | +| 35 | UNSUPPORTED_VERSION | Out-of-range version or unlisted API key | +| 42 | INVALID_REQUEST | Unsupported request header version | + +### Response header rules + +| API key | Request flexible? | Response header version | +| --------- | -------------------- | ------------------------- | +| 18 ApiVersions | v3+ | Always v0 (correlation_id only) | +| 3 Metadata | v9+ | v1 (correlation_id + tagged fields) | +| 0 Produce | v9+ | v1 | +| 1 Fetch | v12+ | v1 | +| Others | Per SCOPE.md | See `header.rs` lookup table | + +### Frame layout (for manual hex inspection) + +```text +Request frame: + [length: i32 BE] + [api_key: i16][api_version: i16][correlation_id: i32] + [client_id: NULLABLE_STRING or COMPACT_NULLABLE_STRING] + [tagged_fields: 0x00] ← flexible requests only + [request body] + +Response frame: + [length: i32 BE] + [correlation_id: i32] + [tagged_fields: 0x00] ← flexible responses only (not ApiVersions) + [response body] +``` + +### Raw netcat smoke test + +```bash +# ApiVersions v3 — after generating fixtures +cat gateways/kafka/tools/kafka-tool/kafka_messages/018_ApiVersions_v3.bin \ + | nc -w 2 127.0.0.1 9093 | xxd | head -20 +``` + +First bytes after length prefix should include your correlation_id from the fixture. + +--- + +## 5. Manual test execution checklist + +Copy this checklist into your PR or test log: + +```text +Date: ___________ +Tester: ___________ +Gateway commit: ___________ +kcat version (if used): ___________ + +[ ] A1–A9 Smoke tests +[ ] B1–B4 Version firewall (all 6 keys × 4 boundary versions) +[ ] C1–C4 Unsupported API keys +[ ] D1–D10 Flexible vs legacy encoding +[ ] E1–E4 Metadata stub semantics +[ ] F1–F6 TCP / connection behavior +[ ] G1–G3 kcat client (record errors for G2/G3) +[ ] H1–H3 Adversarial input + +Automated regression: +[ ] cargo test -p iggy-gateway-kafka — ___/103 passed +[ ] cargo clippy -p iggy-gateway-kafka — clean / warnings noted + +Notes / failures: +_________________________________ +``` + +--- + +## 6. Troubleshooting + +| Symptom | Likely cause | Fix | +| --------- | -------------- | ----- | +| `Connection refused` on 9093 | Gateway not running | Start `iggy-gateway-kafka` | +| `decode_validation_tests` panic | Missing fixtures | Run `kafka-message-gen generate` | +| `ec=35` for in-range version | Version not in `SUPPORTED_RANGES` | Check `SCOPE.md` and `api.rs` | +| kcat hangs | Timeout waiting for data | Set `-m 1000`; check gateway logs | +| Buffer underflow on Metadata v9+ | Flexible decode mismatch | File issue; check `api.rs` metadata encoder | +| Port already in use | Another process on 9093 | `lsof -i :9093` / change bind port | + +--- + +## 7. What manual testing does NOT cover (deferred) + +These are documented as TODO in [SCOPE.md](SCOPE.md) — do not fail #3421 validation for these: + +- Message persistence to Iggy +- Consumer group join/sync/heartbeat +- SASL authentication +- Accurate partition leadership / ISR +- Transactional produce +- Real offset commit semantics diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md new file mode 100644 index 0000000000..984b812470 --- /dev/null +++ b/gateways/kafka/docs/SCOPE.md @@ -0,0 +1,119 @@ +# Kafka gateway scope — [apache/iggy#3421](https://github.com/apache/iggy/issues/3421) + +## Issue #3421 — in scope (this iteration) + +Foundation layer only: a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, validates request wire formats, and returns stub responses. **No Iggy backend integration.** + +| Deliverable | Status | Location | +| ------------- | -------- | ---------- | +| TCP listener on `127.0.0.1:9093` (configurable) | Done | `src/server.rs`, `src/main.rs` | +| Length-prefixed frame read/write with `max_frame_size` cap | Done | `src/server.rs` | +| Request header v1/v2 auto-detection | Done | `src/protocol/header.rs` | +| Version negotiation firewall (`SUPPORTED_RANGES`) | Done | `src/protocol/api.rs` | +| Request decode + stub encode for 6 API keys | Done | `src/protocol/requests.rs`, `responses.rs`, `api.rs` | +| Produce hot path: RecordBatch as opaque `Bytes` | Done | `src/protocol/requests.rs` | +| Graceful errors (`UNSUPPORTED_VERSION`, corrupt decode, invalid header) | Done | `src/protocol/api.rs`, `src/server.rs` | +| Adversarial decode safety tests | Done | `tests/decode_safety_tests.rs` | +| Regression test suite (103 tests) | Done | `tests/` — catalog in [`TEST_SUITE.md`](TEST_SUITE.md) | +| Manual testing procedure | Done | [`MANUAL_TESTING.md`](MANUAL_TESTING.md) | +| Wire fixture tool for manual/integration testing | Done | `tools/kafka-tool/` | + +Source of truth for supported ranges: `SUPPORTED_RANGES` in [`src/protocol/api.rs`](../src/protocol/api.rs). + +### Governance model + +Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiVersions advertises exactly what the firewall allows; out-of-range requests receive `UNSUPPORTED_VERSION` (35) without dropping the connection. + +--- + +## Supported API keys and versions + +| API key | Name | Min version | Max version | Valid versions | Behavior | +| --------- | ------ | ------------- | ------------- | ---------------- | ---------- | +| 18 | ApiVersions | 0 | 3 | 0, 1, 2, 3 | Advertise supported ranges; flexible encoding at v3+ | +| 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode topic list count; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | +| 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub response | +| 1 | Fetch | 4 | 12 | 4, 5, 6, 7, 8, 9, 10, 11, 12 | Decode request; stub response | +| 2 | ListOffsets | 1 | 6 | 1, 2, 3, 4, 5, 6 | Decode request; stub response | +| 19 | CreateTopics | 2 | 5 | 2, 3, 4, 5 | Decode request; stub response | + +A request is accepted when `min_version ≤ api_version ≤ max_version` for that API key. Any other version for a listed key, or any unlisted API key, receives `UNSUPPORTED_VERSION` (35). + +### Valid versions reference (by API key) + +Use this table when configuring clients or generating wire fixtures with `kafka-message-gen`. + +| API key | Name | Valid versions (inclusive range) | Flexible wire encoding from | +| --------- | ------ | ---------------------------------- | ---------------------------- | +| 0 | Produce | 3–9 | v9 | +| 1 | Fetch | 4–12 | v12 | +| 2 | ListOffsets | 1–6 | v6 | +| 3 | Metadata | 0–9 | v9 | +| 18 | ApiVersions | 0–3 | v3 | +| 19 | CreateTopics | 2–5 | v5 | + +--- + +## Unsupported API keys (foundation) + +All API keys not listed above receive an error-only response with `UNSUPPORTED_VERSION` (35). Examples not in this foundation scope: + +| API key | Name | Notes | +| --------- | ------ | ------- | +| 8 | OffsetCommit | Consumer group — later issue | +| 9 | OffsetFetch | Consumer group — later issue | +| 10 | FindCoordinator | Consumer group — later issue | +| 11–16 | JoinGroup, Heartbeat, LeaveGroup, SyncGroup, DescribeGroups, ListGroups | Consumer group — later issue | +| 17 | SaslHandshake | Auth — later issue | +| 20+ | DeleteTopics, InitProducerId, transactions, ACLs, etc. | Later issues | + +Full reference for future phases: [`kafka_api_keys_reference.md`](kafka_api_keys_reference.md). + +--- + +## Architecture (three layers) + +| Layer | #3421 | Description | +| ------- | ------- | ------------- | +| **1 — Wire framing** | In scope | `server.rs`, `codec.rs`, `header.rs` — keep custom, zero-copy frame I/O | +| **2 — Request/response codecs** | Partial | Custom minimal-parse codecs for 6 hot-path keys; stub responses only | +| **3 — Iggy bridge** | Out of scope | Produce/Fetch → Iggy SDK; deferred to a follow-on issue | + +--- + +## TODO — post-#3421 (architecture review backlog) + +Items from the [hybrid architecture review](https://github.com/apache/iggy/discussions/3252) and maintainer feedback. **Not part of #3421.** + +### Phase 2 — Iggy bridge (new issue) + +- [ ] Add `bridge/` module (`iggy_bridge`): Produce → `send_messages`, Fetch → `poll_messages` +- [ ] Document partition mapping in `docs/BRIDGE_MAPPING.md`: + - Iggy partitions are **0-based** (same as Kafka) — direct `partition_id` mapping, no offset conversion + - Iggy **consumer groups exist** — map Kafka group APIs to Iggy consumer group APIs + - Use `Partitioning::balanced()` only when Kafka sends `partition == -1`; otherwise use request partition ID +- [ ] Idempotent `ensure_stream_and_topic()` (create-if-not-exists) +- [ ] Real Metadata topology (brokers, partitions, leaders) backed by Iggy state + +### Phase 2 — Selective `kafka-protocol` crate (feature-gated) + +- [ ] Add optional `kafka-protocol-cold` feature to `iggy_gateway_kafka` — **not** wholesale replace of `requests.rs`/`responses.rs` +- [ ] Use crate for: RecordBatch decode (compression, CRC), consumer-group API keys (8–14, 10), complex Metadata/FindCoordinator responses +- [ ] **Keep custom codecs** for Produce/Fetch hot paths (opaque RecordBatch bytes) + +### Phase 3 — Consumer groups (~7 API keys) + +- [ ] OffsetCommit (8), OffsetFetch (9), FindCoordinator (10) +- [ ] JoinGroup (11), Heartbeat (12), LeaveGroup (13), SyncGroup (14) +- [ ] DescribeGroups (15), ListGroups (16) as needed by target clients + +### Phase 3+ — Auth, admin, tuning + +- [ ] SASL (17, 36) if required by deployment +- [ ] Tune `max_frame_size` per workload (Kafka defaults: ~1 MiB produce, ~50 MiB fetch; current default 8 MiB) +- [ ] Target **~15–20 API keys** total for a functional bridge — not all 74+ admin keys + +### Open questions (ask maintainers before Phase 2) + +- [ ] Repo placement: `gateways/kafka/` in [apache/iggy](https://github.com/apache/iggy) vs separate proxy repo (affects workspace deps and CI) +- [ ] Confirm bridge dependency strategy with spetz/hubcio ([Discussion #3081](https://github.com/apache/iggy/discussions/3081), [#3252](https://github.com/apache/iggy/discussions/3252)) diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md new file mode 100644 index 0000000000..74dd995268 --- /dev/null +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -0,0 +1,155 @@ +# Kafka gateway — automated regression test suite + +Regression tests live under [`tests/`](../tests/). Run from the workspace root: + +```bash +cargo test -p iggy-gateway-kafka +``` + +**Current count:** 103 tests across 12 suites (as of #3421 foundation). + +## Prerequisites + +### Wire fixtures (required for `decode_validation_tests` and some handler tests) + +```bash +./gateways/kafka/scripts/ci-wire-fixtures.sh generate +``` + +Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the same script before `rust-gateway` test jobs and removes the directory afterward. Tests that need fixtures skip gracefully when a file is missing (`handler_regression_tests`) or panic with a clear path (`decode_validation_tests`). + +--- + +## Test file catalog + +| File | Suite focus | Test count (approx.) | Depends on fixtures | +| ------ | ------------- | ---------------------- | --------------------- | +| [`codec_tests.rs`](../tests/codec_tests.rs) | Primitive encode/decode round-trips, varint, compact strings, tagged fields | 9 | No | +| [`decode_safety_tests.rs`](../tests/decode_safety_tests.rs) | Adversarial wire input — malformed lengths, truncated bodies | 6 | No | +| [`header_tests.rs`](../tests/header_tests.rs) | Request/response header v1/v2, version lookup table | 10 | No | +| [`api_handler_tests.rs`](../tests/api_handler_tests.rs) | ApiVersions, Metadata stub, unsupported key/version | 7 | No | +| [`golden_wire_fixtures_tests.rs`](../tests/golden_wire_fixtures_tests.rs) | Byte-exact golden responses (ApiVersions v1, Metadata v0) | 2 | No | +| [`decode_validation_tests.rs`](../tests/decode_validation_tests.rs) | kafka-tool fixture decode + response structure per version | 14 | **Yes** | +| [`version_firewall_tests.rs`](../tests/version_firewall_tests.rs) | Version boundary matrix, unsupported keys, corrupt bodies | 17 | Partial | +| [`metadata_regression_tests.rs`](../tests/metadata_regression_tests.rs) | Metadata v0–v9, topic counts, broker advertise | 7 | No | +| [`broker_advertise_tests.rs`](../tests/broker_advertise_tests.rs) | `BrokerAdvertise::from_server_config` parsing | 5 | No | +| [`handler_regression_tests.rs`](../tests/handler_regression_tests.rs) | Every scoped key×version via `handle_request`, stub error codes | 5 | Partial | +| [`server_integration_tests.rs`](../tests/server_integration_tests.rs) | `read_frame` / `write_frame` unit-level I/O | 4 | No | +| [`server_e2e_tests.rs`](../tests/server_e2e_tests.rs) | Full `KafkaServer` TCP round-trips | 8 | Partial | +| [`common/mod.rs`](../tests/common/mod.rs) | Shared helpers (not a test binary) | — | — | + +--- + +## Coverage matrix by API key + +### ApiVersions (key 18, v0–v3) + +| Scenario | Test file | Test name | +| ---------- | ----------- | ----------- | +| Non-flexible response (v1) | `api_handler_tests` | `api_versions_v1_response_non_flexible_format` | +| Flexible response (v3) | `api_handler_tests` | `api_versions_v3_response_flexible_format` | +| Golden byte fixture (v1) | `golden_wire_fixtures_tests` | `golden_apiversions_v1_response_fixture` | +| Exact advertised ranges (v1, v3) | `version_firewall_tests` | `apiversions_advertises_exact_supported_ranges_*` | +| All versions return `error_code=0` | `version_firewall_tests` | `apiversions_all_versions_return_success` | +| Out-of-range version | `version_firewall_tests` | `apiversions_out_of_range_returns_unsupported_in_body` | +| E2E correlation ID preserved | `server_e2e_tests` | `e2e_apiversions_v1_*`, `e2e_apiversions_v3_*` | + +### Metadata (key 3, v0–v9) + +| Scenario | Test file | Test name | +| ---------- | ----------- | ----------- | +| Stub broker (default 127.0.0.1:9093) | `api_handler_tests`, `metadata_regression_tests` | `metadata_response_has_broker_*`, `metadata_v0_empty_*` | +| Unsupported version → topic error 35 | `api_handler_tests`, `version_firewall_tests` | `unsupported_version_returns_protocol_error`, `metadata_*_version_returns_topic_error` | +| Golden byte fixture (v0, 1 topic) | `golden_wire_fixtures_tests` | `golden_metadata_v0_single_topic_response_fixture` | +| v1 controller_id, v2 cluster_id | `metadata_regression_tests` | `metadata_v1_*`, `metadata_v2_*` | +| v9 flexible encoding | `metadata_regression_tests` | `metadata_v9_flexible_encoding` | +| Custom broker advertise | `metadata_regression_tests`, `broker_advertise_tests` | `metadata_uses_custom_*`, `metadata_reflects_parsed_*` | +| E2E round-trip | `server_e2e_tests` | `e2e_metadata_v0_returns_stub_broker` | + +### Produce (key 0, v3–v9) + +| Scenario | Test file | Test name | +| ---------- | ----------- | ----------- | +| Decode all versions (fixture) | `decode_validation_tests` | `produce_all_supported_versions_decode` | +| Response encode all versions | `decode_validation_tests` | `produce_response_encodes_for_all_supported_versions` | +| v3 field layout | `decode_validation_tests` | `produce_response_v3_roundtrip` | +| v8 record_errors array | `decode_validation_tests` | `produce_response_v8_includes_record_errors` | +| Unsupported v2 → error 35 | `version_firewall_tests` | `produce_unsupported_version_returns_error_only` | +| Corrupt body → error 42 | `version_firewall_tests` | `corrupt_produce_body_returns_invalid_request_error` | +| Stub partition error 0 | `handler_regression_tests` | `produce_stub_response_has_zero_error_per_partition` | +| E2E round-trip | `server_e2e_tests` | `e2e_produce_v3_round_trip_with_fixture` | + +### Fetch (key 1, v4–v12) + +| Scenario | Test file | Test name | +| ---------- | ----------- | ----------- | +| Decode all versions | `decode_validation_tests` | `fetch_all_supported_versions_decode` | +| Response encode all versions | `decode_validation_tests` | `fetch_response_encodes_for_all_supported_versions` | +| v7 session_id / error_code layout | `decode_validation_tests` | `fetch_response_v7_roundtrip` | +| Unsupported v3 | `version_firewall_tests` | `fetch_unsupported_version_returns_error_only` | +| Corrupt body → error 42 | `version_firewall_tests` | `corrupt_fetch_body_returns_invalid_request_error` | +| Stub partition error 0 | `handler_regression_tests` | `fetch_stub_response_has_zero_partition_error` | + +### ListOffsets (key 2, v1–v6) + +| Scenario | Test file | Test name | +| ---------- | ----------- | ----------- | +| Decode all versions | `decode_validation_tests` | `list_offsets_all_supported_versions_decode` | +| v1 no leader_epoch | `decode_validation_tests` | `list_offsets_response_v1_no_leader_epoch` | +| v4 has leader_epoch | `decode_validation_tests` | `list_offsets_response_v4_has_leader_epoch` | +| Unsupported v0 | `version_firewall_tests` | `list_offsets_unsupported_version_returns_error_only` | +| Stub error 0 | `handler_regression_tests` | `list_offsets_stub_response_has_zero_error` | + +### CreateTopics (key 19, v2–v5) + +| Scenario | Test file | Test name | +| ---------- | ----------- | ----------- | +| Decode all versions | `decode_validation_tests` | `create_topics_all_supported_versions_decode` | +| v2 roundtrip | `decode_validation_tests` | `create_topics_response_v2_roundtrip` | +| v5 flexible roundtrip | `decode_validation_tests` | `create_topics_response_v5_roundtrip` | +| Unsupported v1 | `version_firewall_tests` | `create_topics_unsupported_version_returns_error_only` | +| Stub error 0 | `handler_regression_tests` | `create_topics_stub_response_has_zero_error` | + +--- + +## Cross-cutting scenarios + +| Scenario | Test file | Test name | +| ---------- | ----------- | ----------- | +| Version firewall min/max boundaries | `version_firewall_tests` | `is_supported_version_matches_scope_table` | +| Unknown API keys (8, 9, 10, 17, 20, 999) | `version_firewall_tests`, `api_handler_tests` | `unsupported_api_keys_*`, `unknown_api_key_*` | +| Negative i32 array length | `decode_safety_tests` | `negative_i32_array_length_returns_error_not_panic` | +| Oversized collection count | `decode_safety_tests` | `i32_array_length_above_max_returns_collection_too_large` | +| Compact array varint=0 (null array) | `decode_safety_tests` | `compact_array_varint_zero_decodes_as_empty_without_panic` | +| Malformed varint at shift 63 | `decode_safety_tests` | `varint_terminal_byte_with_extra_bits_at_shift_63_is_rejected` | +| Invalid frame length (0) | `server_integration_tests` | `read_frame_rejects_invalid_lengths` | +| Frame exceeds max_frame_size | `server_integration_tests`, `server_e2e_tests` | `read_frame_rejects_invalid_lengths`, `e2e_oversized_frame_is_rejected` | +| Sequential requests on one TCP connection | `server_e2e_tests` | `e2e_sequential_requests_on_one_connection` | +| Connection survives unsupported API key | `server_e2e_tests` | `e2e_unsupported_api_key_returns_error_without_disconnect` | +| Negative frame length closes connection | `server_e2e_tests` | `e2e_negative_frame_length_closes_connection` | + +--- + +## CI recommendation + +```bash +# 1. Generate fixtures +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 19 + +# 2. Run regression suite +cargo test -p iggy-gateway-kafka + +# 3. Optional lint gate +cargo clippy -p iggy-gateway-kafka -- -D warnings +``` + +--- + +## Adding new tests + +1. **New API key or version range** — update `SUPPORTED_RANGES` in `api.rs`, `SCOPE.md`, and add rows to the coverage matrix above. +2. **New decode path** — add fixture via `kafka-message-gen`, extend `decode_validation_tests.rs`. +3. **New error path** — add to `version_firewall_tests.rs` or `decode_safety_tests.rs`. +4. **New TCP behavior** — add to `server_e2e_tests.rs` using helpers in `tests/common/mod.rs`. diff --git a/gateways/kafka/docs/kafka_api_keys_reference.md b/gateways/kafka/docs/kafka_api_keys_reference.md new file mode 100644 index 0000000000..2555bf96af --- /dev/null +++ b/gateways/kafka/docs/kafka_api_keys_reference.md @@ -0,0 +1,304 @@ +# Kafka Protocol API Key Reference — Kafka 4.0.0 + +> **Source**: [`ApiKeys.java` @ Kafka 4.0.0](https://github.com/apache/kafka/blob/4.0.0/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java) +> and the canonical [protocol message schemas](https://github.com/apache/kafka/tree/4.0.0/clients/src/main/resources/common/message). +> +> Generated for: **Iggy Kafka Bridge Gateway** — `gateways/kafka/` +> Branch: `feat(gateways)/kafka_to_iggy_listener` + +--- + +## Legend + +| Symbol | Meaning | +| -------- | --------- | +| 🔴 Bridge | Core data path — must be fully implemented and forwarded to Iggy | +| 🟠 Required Stub | Client state-machine API — must return a well-formed response or clients will stall/crash | +| 🟡 Optional Stub | Admin/observability — can safely return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` | +| ❌ Reject | Internal broker / KRaft only — return `INVALID_REQUEST` with a well-formed frame; **do not close the connection** | + +> **Header.rs ✓** = The API key is already present in `request_header_version()` / `response_header_version()` with the correct flexible-encoding threshold. + +--- + +## KIP-896 Note — Minimum Version Changes in Kafka 4.0 + +Kafka 4.0 removed all protocol versions older than Kafka 2.1.0 (KIP-896). +Key new minimums: + +| API | Old Min | New Min (4.0) | +| ----- | :-------: | :-------------: | +| Produce | 0 | 3 | +| Fetch | 0 | 4 | +| ListOffsets | 0 | 1 | +| OffsetCommit | 0 | 2 | +| OffsetFetch | 0 | 1 | +| CreateTopics | 0 | 2 | +| OffsetForLeaderEpoch | 0 | 1 | + +> ⚠️ **KAFKA-18659 / librdkafka bug**: Even though Produce's actual minimum in Kafka 4.0 is v3, the +> `ApiVersions` response **must advertise min=0** for Produce to avoid breaking librdkafka clients. +> `ApiKeys.java` has a dedicated constant `PRODUCE_API_VERSIONS_RESPONSE_MIN_VERSION = 0` for this. +> Your gateway's `ApiVersions` response encoder must replicate this special case. + +--- + +## Group 1 — Core Data Path + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 0 | **Produce** | 3 † | 12 | v9 | ✅ | 🔴 Bridge | +| 1 | **Fetch** | 4 | 17 | v12 | ✅ | 🔴 Bridge | +| 2 | **ListOffsets** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | +| 3 | **Metadata** | 1 | 12 | v9 | ✅ | 🔴 Bridge | + +† See KAFKA-18659 librdkafka workaround above — advertise min=0 in ApiVersions response. + +--- + +## Group 2 — API Negotiation & Auth + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 17 | **SaslHandshake** | 0 | 1 | never | ✅ | 🔴 Bridge (auth flow) | +| 18 | **ApiVersions** | 0 | 4 | v3 | ✅ | 🔴 Bridge (advertise Iggy caps) | +| 36 | **SaslAuthenticate** | 0 | 2 | v2 | ✅ | 🔴 Bridge (auth flow) | + +> **ApiVersions special case**: The response header is **always v0** (no tagged fields), regardless of +> the request version. This allows clients that don't yet know the server's encoding to parse the +> discovery response. `header.rs` already handles this correctly via the `api_key == 18` guard. + +--- + +## Group 3 — Classic Consumer Group Protocol + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 8 | **OffsetCommit** | 2 | 9 | v8 | ✅ | 🟠 Required Stub | +| 9 | **OffsetFetch** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | +| 10 | **FindCoordinator** | 1 | 6 | v3 | ✅ | 🟠 Required Stub | +| 11 | **JoinGroup** | 2 | 9 | v6 | ✅ | 🟠 Required Stub | +| 12 | **Heartbeat** | 1 | 4 | v4 | ✅ | 🟠 Required Stub | +| 13 | **LeaveGroup** | 1 | 5 | v4 | ✅ | 🟠 Required Stub | +| 14 | **SyncGroup** | 1 | 5 | v4 | ✅ | 🟠 Required Stub | +| 15 | **DescribeGroups** | 0 | 6 | v5 | ✅ | 🟡 Optional Stub | +| 16 | **ListGroups** | 1 | 5 | v3 | ✅ | 🟡 Optional Stub | +| 42 | **DeleteGroups** | 1 | 2 | v2 | ✅ | 🟡 Optional Stub | + +--- + +## Group 4 — New Consumer Group Protocol (KIP-848, Kafka 3.7+) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 68 | **ConsumerGroupHeartbeat** | 0 | 1 | v0 | ✅ | 🟠 Required Stub | +| 69 | **ConsumerGroupDescribe** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | + +> ⚠️ Kafka 4.0 clients use the **new group protocol by default** and will send key 68. +> A gateway that hard-rejects this breaks all modern Kafka consumers. + +--- + +## Group 5 — Topic Administration + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 19 | **CreateTopics** | 2 | 7 | v5 | ✅ (max v5 ⚠️) | 🟠 Required Stub | +| 20 | **DeleteTopics** | 1 | 6 | v4 | ✅ | 🟡 Optional Stub | +| 21 | **DeleteRecords** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 37 | **CreatePartitions** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | + +> ⚠️ `SUPPORTED_RANGES` in `api.rs` currently advertises CreateTopics max=5; actual max is v7. + +--- + +## Group 6 — Transactions (EOS — Exactly Once Semantics) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 22 | **InitProducerId** | 2 | 5 | v2 | ✅ | 🟡 Optional Stub | +| 23 | **OffsetForLeaderEpoch** | 1 | 5 | v4 | ✅ | 🟡 Optional Stub | +| 24 | **AddPartitionsToTxn** | 1 | 5 | v3 | ✅ | 🟡 Optional Stub | +| 25 | **AddOffsetsToTxn** | 1 | 4 | v3 | ✅ | 🟡 Optional Stub | +| 26 | **EndTxn** | 1 | 4 | v3 | ✅ | 🟡 Optional Stub | +| 27 | **WriteTxnMarkers** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | +| 28 | **TxnOffsetCommit** | 2 | 5 | v3 | ✅ | 🟡 Optional Stub | + +--- + +## Group 7 — Security & ACLs + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 29 | **DescribeAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 30 | **CreateAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 31 | **DeleteAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 38 | **CreateDelegationToken** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 39 | **RenewDelegationToken** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 40 | **ExpireDelegationToken** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 41 | **DescribeDelegationToken** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 50 | **DescribeUserScramCredentials** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 51 | **AlterUserScramCredentials** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | + +--- + +## Group 8 — Configuration & Quotas + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 32 | **DescribeConfigs** | 0 | 4 | v4 | ✅ | 🟡 Optional Stub | +| 33 | **AlterConfigs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 44 | **IncrementalAlterConfigs** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | +| 48 | **DescribeClientQuotas** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | +| 49 | **AlterClientQuotas** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | + +--- + +## Group 9 — Log & Partition Admin + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 34 | **AlterReplicaLogDirs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 35 | **DescribeLogDirs** | 0 | 4 | v2 | ✅ | 🟡 Optional Stub | +| 43 | **ElectLeaders** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 45 | **AlterPartitionReassignments** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 46 | **ListPartitionReassignments** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 47 | **OffsetDelete** | 0 | 0 | never | ✅ | 🟡 Optional Stub | +| 57 | **UpdateFeatures** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | + +--- + +## Group 10 — Cluster Introspection + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 55 | **DescribeQuorum** | 0 | 2 | v0 | ✅ | 🟡 Optional Stub | +| 59 | **FetchSnapshot** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | +| 60 | **DescribeCluster** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | +| 61 | **DescribeProducers** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 64 | **UnregisterBroker** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 65 | **DescribeTransactions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 66 | **ListTransactions** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | +| 75 | **DescribeTopicPartitions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | + +--- + +## Group 11 — Observability / Telemetry (KIP-714, Kafka 3.7+) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 71 | **GetTelemetrySubscriptions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 72 | **PushTelemetry** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 76 | **ListClientMetricsResources** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | + +--- + +## Group 12 — Share Groups (NEW in Kafka 4.0, KIP-932) + +> Keys 77–80 use flexible header framing from v0 (added to `header.rs`). Keys 84–88 are internal +> coordinator APIs and can be rejected. + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 77 | **ShareGroupHeartbeat** | 0 | 0 | v0 | ✅ | 🟠 Required Stub | +| 78 | **ShareGroupDescribe** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 79 | **ShareFetch** | 0 | 0 | v0 | ✅ | 🔴 Bridge (share consume) | +| 80 | **ShareAcknowledge** | 0 | 0 | v0 | ✅ | 🟠 Required Stub | + +--- + +## Group 13 — KRaft Raft Voter Management (NEW in Kafka 4.0) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 81 | **AddRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 82 | **RemoveRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 83 | **UpdateRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | + +--- + +## Group 14 — KRaft Internal / Broker-Only (Always Reject at Gateway) + +> These APIs must **never** be handled by a client-facing gateway. +> Return `INVALID_REQUEST` (error code 42) with a properly framed response — **do not drop the connection**. + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | +| 4 | **LeaderAndIsr** | 0 | 7 | v4 | ✅ | ❌ Reject (broker-only) | +| 5 | **StopReplica** | 0 | 4 | v2 | ✅ | ❌ Reject (broker-only) | +| 6 | **UpdateMetadata** | 0 | 8 | v6 | ✅ | ❌ Reject (broker-only) | +| 7 | **ControlledShutdown** | 0 | 3 | v3 | ✅ | ❌ Reject (broker-only) | +| 52 | **Vote** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 53 | **BeginQuorumEpoch** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 54 | **EndQuorumEpoch** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 56 | **AlterPartition** | 0 | 3 | v0 | ✅ | ❌ Reject (KRaft) | +| 58 | **Envelope** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 62 | **BrokerRegistration** | 0 | 4 | v0 | ✅ | ❌ Reject (KRaft) | +| 63 | **BrokerHeartbeat** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 67 | **AllocateProducerIds** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 70 | **ControllerRegistration** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 74 | **AssignReplicasToDirs** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 84 | **InitializeShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 85 | **ReadShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 86 | **WriteShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 87 | **DeleteShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 88 | **ReadShareGroupStateSummary** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | + +--- + +## Summary Counts + +| Category | Count | Notes | +| ---------- | :-----: | ------- | +| 🔴 Bridge (data path) | 6 | Produce, Fetch, Metadata, ApiVersions, SaslHandshake, SaslAuthenticate, ShareFetch | +| 🟠 Required Stub (client state machine) | 14 | Consumer group, CreateTopics, ConsumerGroupHeartbeat (68), ShareGroupHeartbeat (77), ShareAcknowledge (80) | +| 🟡 Optional Stub (admin/observability) | 44 | Can return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` safely | +| ❌ Reject (broker/KRaft internal) | 19 | Return `INVALID_REQUEST` with valid frame — never close the TCP connection | +| **Total API Keys in Kafka 4.0** | **83** | Key IDs 0–88 with gap at 73 | + +--- + +## Current Implementation Gaps in `api.rs` + +### `SUPPORTED_RANGES` is behind the latest Kafka 4.0 max versions + +| API | Declared range | Kafka 4.0 max | Gap | +| ----- | :---: | :---: | :---: | +| Produce | v3-v9 | v12 | 3 versions behind | +| Fetch | v4-v12 | v17 | 5 versions behind | +| ListOffsets | v1-v6 | v9 | 3 versions behind | +| Metadata | v0-v9 | v12 | 3 versions behind | +| ApiVersions | v0-v3 | v4 | 1 version behind | +| CreateTopics | v2-v5 | v7 | 2 versions behind | + +### Missing from `SUPPORTED_RANGES` (77 API keys) + +Every key not in the table above falls through to `encode_error_only_response` +(2-byte error frame), including: + +- **Client bootstrap blockers**: OffsetCommit (8), OffsetFetch (9), FindCoordinator (10) +- **Classic consumer group protocol**: JoinGroup (11), Heartbeat (12), LeaveGroup (13), SyncGroup (14) +- **New consumer group protocol**: ConsumerGroupHeartbeat (68) — default in Kafka 4.0 +- **Share groups (KIP-932)**: ShareFetch (79), ShareGroupHeartbeat (77), ShareAcknowledge (80) +- **Auth flow**: SaslHandshake (17), SaslAuthenticate (36) +- **All 19 broker/KRaft-internal keys** (Group 14) — should return `INVALID_REQUEST` with a + valid frame instead of the bare 2-byte fallback, so the connection is never dropped. + +### `header.rs` `request_header_version()` — remaining gaps + +Keys 77-80 (ShareGroupHeartbeat, ShareGroupDescribe, ShareFetch, ShareAcknowledge) are +covered (`flexible_from = 0`). Keys 81-88 — AddRaftVoter, RemoveRaftVoter, UpdateRaftVoter, +and the five ShareGroupState keys (84-88) — are all always-flexible per KIP-932/KRaft but +still fall through to the `_ => i16::MAX` (non-flexible) arm, which would misframe them if +ever dispatched. + +## References + +- [ApiKeys.java @ Kafka 4.0.0](https://github.com/apache/kafka/blob/4.0.0/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java) +- [Kafka Protocol Message Schemas @ 4.0.0](https://github.com/apache/kafka/tree/4.0.0/clients/src/main/resources/common/message) +- [KIP-896: Remove old client protocol API versions in Kafka 4.0](https://cwiki.apache.org/confluence/display/KAFKA/KIP-896%3A+Remove+old+client+protocol+API+versions+in+Kafka+4.0) +- [KIP-848: Apache Kafka Consumer Rebalance Protocol](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol) +- [KIP-932: Queues for Kafka (Share Groups)](https://cwiki.apache.org/confluence/display/KAFKA/KIP-932%3A+Queues+for+Kafka) +- [KIP-714: Client Metrics and Observability](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) +- [Kafka Wire Protocol Documentation](https://kafka.apache.org/protocol.html) +- [kafka-protocol Rust crate](https://crates.io/crates/kafka-protocol) diff --git a/gateways/kafka/scripts/ci-wire-fixtures.sh b/gateways/kafka/scripts/ci-wire-fixtures.sh new file mode 100755 index 0000000000..0cdb87aefd --- /dev/null +++ b/gateways/kafka/scripts/ci-wire-fixtures.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generate or remove gitignored kafka-tool wire fixtures for CI. +# Run from the iggy workspace root. + +set -euo pipefail + +FIXTURES_DIR="gateways/kafka/tools/kafka-tool/kafka_messages" + +# API keys exercised by decode_validation_tests and handler_regression_tests. +FIXTURE_API_KEYS=(0 1 2 19) + +usage() { + echo "Usage: $0 {generate|cleanup}" >&2 + exit 2 +} + +generate() { + mkdir -p "$FIXTURES_DIR" + for key in "${FIXTURE_API_KEYS[@]}"; do + cargo run --locked -p kafka-message-gen -- generate \ + --output "$FIXTURES_DIR" \ + --api-key "$key" + done + echo "Generated wire fixtures under ${FIXTURES_DIR}/" +} + +cleanup() { + rm -rf "$FIXTURES_DIR" + echo "Removed ${FIXTURES_DIR}/" +} + +case "${1:-}" in + generate) generate ;; + cleanup) cleanup ;; + *) usage ;; +esac diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs new file mode 100644 index 0000000000..16b9de936c --- /dev/null +++ b/gateways/kafka/src/error.rs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum KafkaProtocolError { + #[error("invalid server configuration: {0}")] + InvalidConfig(String), + #[error("buffer underflow: needed {needed} bytes, remaining {remaining}")] + BufferUnderflow { needed: usize, remaining: usize }, + #[error("invalid frame length: {0}")] + InvalidFrameLength(i32), + #[error("request exceeds max frame size ({max_bytes} bytes): {actual_bytes} bytes")] + FrameTooLarge { + max_bytes: usize, + actual_bytes: usize, + }, + #[error("invalid utf8 string")] + InvalidUtf8, + #[error("varint overflows 64 bits")] + InvalidVarint, + #[error("unsupported request header version: {0}")] + UnsupportedHeaderVersion(i16), + #[error("invalid array length: {0}")] + InvalidArrayLength(i32), + #[error("collection length {count} exceeds maximum {max}")] + CollectionTooLarge { count: usize, max: usize }, + #[error("string length {length} exceeds i16::MAX")] + StringTooLong { length: usize }, + #[error("null topic name in request")] + NullTopicName, + #[error("io error: {0}")] + Io(#[from] std::io::Error), +} + +pub type Result = std::result::Result; diff --git a/gateways/kafka/src/lib.rs b/gateways/kafka/src/lib.rs new file mode 100644 index 0000000000..a75b625786 --- /dev/null +++ b/gateways/kafka/src/lib.rs @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka wire protocol gateway foundation for Apache Iggy. + +pub mod error; +pub mod protocol; +pub mod server; + +pub use server::{KafkaServer, ServerConfig}; diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs new file mode 100644 index 0000000000..0f48ea943e --- /dev/null +++ b/gateways/kafka/src/main.rs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use tokio::net::TcpListener; +use tokio::signal; +use tokio::sync::broadcast; + +use iggy_gateway_kafka::server::init_tracing; +use iggy_gateway_kafka::{KafkaServer, ServerConfig}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + init_tracing(); + + let mut config = ServerConfig::default(); + if let Ok(bind_addr) = std::env::var("KAFKA_BIND_ADDR") { + config.bind_addr = bind_addr; + } + if let Ok(advertised_host) = std::env::var("KAFKA_ADVERTISED_HOST") { + config.advertised_host = Some(advertised_host); + } + if let Ok(advertised_port) = std::env::var("KAFKA_ADVERTISED_PORT") { + config.advertised_port = Some( + advertised_port + .parse() + .map_err(|e| format!("invalid KAFKA_ADVERTISED_PORT `{advertised_port}`: {e}"))?, + ); + } + let listener = TcpListener::bind(&config.bind_addr) + .await + .map_err(|e| format!("failed to bind {}: {e}", config.bind_addr))?; + let server = KafkaServer::new(config); + + let (tx, rx) = broadcast::channel(1); + let mut server_task = tokio::spawn(async move { server.run(listener, rx).await }); + + tokio::select! { + result = &mut server_task => { + return Ok(result??); + } + _ = signal::ctrl_c() => { + let _ = tx.send(()); + } + } + + server_task.await??; + Ok(()) +} diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs new file mode 100644 index 0000000000..2c1d3b39b2 --- /dev/null +++ b/gateways/kafka/src/protocol/api.rs @@ -0,0 +1,357 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use crate::error::Result; +use crate::protocol::codec::{Decoder, Encoder}; +use crate::protocol::requests::{ + decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, + decode_produce_request, +}; +use crate::protocol::responses::{ + encode_create_topics_error_response, encode_create_topics_response, + encode_fetch_error_response, encode_fetch_response, encode_list_offsets_error_response, + encode_list_offsets_response, encode_produce_error_response, encode_produce_response, +}; + +pub const API_KEY_PRODUCE: i16 = 0; +pub const API_KEY_FETCH: i16 = 1; +pub const API_KEY_LIST_OFFSETS: i16 = 2; +pub const API_KEY_METADATA: i16 = 3; +pub const API_KEY_API_VERSIONS: i16 = 18; +pub const API_KEY_CREATE_TOPICS: i16 = 19; + +pub const DEFAULT_KAFKA_PORT: u16 = 9093; + +pub const ERROR_NONE: i16 = 0; +pub const ERROR_UNKNOWN_TOPIC_OR_PARTITION: i16 = 3; +pub const ERROR_UNSUPPORTED_VERSION: i16 = 35; +pub const ERROR_INVALID_PARTITIONS: i16 = 37; +pub const ERROR_INVALID_REQUEST: i16 = 42; + +const MAX_SUPPORTED_METADATA_VERSION: i16 = 9; + +/// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. +const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; + +#[derive(Debug, Clone)] +pub struct BrokerAdvertise { + pub host: String, + pub port: i32, +} + +impl Default for BrokerAdvertise { + fn default() -> Self { + Self { + host: "127.0.0.1".to_string(), + port: i32::from(DEFAULT_KAFKA_PORT), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ApiVersionRange { + pub api_key: i16, + pub min_version: i16, + pub max_version: i16, +} + +static SUPPORTED_RANGES: &[ApiVersionRange] = &[ + ApiVersionRange { + api_key: API_KEY_PRODUCE, + min_version: 3, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_FETCH, + min_version: 4, + max_version: 12, + }, + ApiVersionRange { + api_key: API_KEY_LIST_OFFSETS, + min_version: 1, + max_version: 6, + }, + ApiVersionRange { + api_key: API_KEY_METADATA, + min_version: 0, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_API_VERSIONS, + min_version: 0, + max_version: 3, + }, + ApiVersionRange { + api_key: API_KEY_CREATE_TOPICS, + min_version: 2, + max_version: 5, + }, +]; + +#[must_use] +pub fn supported_api_ranges() -> &'static [ApiVersionRange] { + SUPPORTED_RANGES +} + +pub fn handle_request( + api_key: i16, + api_version: i16, + body: Bytes, + broker: &BrokerAdvertise, +) -> Bytes { + match api_key { + API_KEY_API_VERSIONS => { + if is_supported_version(api_key, api_version) { + encode_api_versions_response(api_version, ERROR_NONE) + } else { + // KIP-511: reply with v0 when the requested version is not understood. + encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_METADATA => { + if is_supported_version(api_key, api_version) { + encode_metadata_response(api_version, body, broker, ERROR_NONE) + } else { + // Encode at the highest version we implement, not the client's unknown version. + encode_metadata_response( + api_version.clamp(0, MAX_SUPPORTED_METADATA_VERSION), + body, + broker, + ERROR_UNSUPPORTED_VERSION, + ) + } + } + API_KEY_PRODUCE => { + if is_supported_version(api_key, api_version) { + match decode_produce_request(api_version, body) { + Ok(req) => encode_produce_response(api_version, &req), + Err(e) => { + tracing::warn!("Failed to decode Produce request: {:?}", e); + encode_produce_error_response(api_version, ERROR_INVALID_REQUEST) + } + } + } else { + encode_produce_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_FETCH => { + if is_supported_version(api_key, api_version) { + match decode_fetch_request(api_version, body) { + Ok(req) => encode_fetch_response(api_version, &req), + Err(e) => { + tracing::warn!("Failed to decode Fetch request: {:?}", e); + encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST) + } + } + } else { + encode_fetch_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_LIST_OFFSETS => { + if is_supported_version(api_key, api_version) { + match decode_list_offsets_request(api_version, body) { + Ok(req) => encode_list_offsets_response(api_version, &req), + Err(e) => { + tracing::warn!("Failed to decode ListOffsets request: {:?}", e); + encode_list_offsets_error_response(api_version, ERROR_INVALID_REQUEST) + } + } + } else { + encode_list_offsets_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_CREATE_TOPICS => { + if is_supported_version(api_key, api_version) { + match decode_create_topics_request(api_version, body) { + Ok(req) => encode_create_topics_response(api_version, &req), + Err(e) => { + tracing::warn!("Failed to decode CreateTopics request: {:?}", e); + encode_create_topics_error_response(api_version, ERROR_INVALID_REQUEST) + } + } + } else { + encode_create_topics_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + } + } + _ => encode_error_only_response(ERROR_UNSUPPORTED_VERSION), + } +} + +#[must_use] +pub fn is_supported_version(api_key: i16, api_version: i16) -> bool { + SUPPORTED_RANGES + .iter() + .find(|r| r.api_key == api_key) + .is_some_and(|r| api_version >= r.min_version && api_version <= r.max_version) +} + +/// Min version advertised in `ApiVersions` (may differ from the firewall min). +/// +/// Produce must advertise min=0 per KAFKA-18659 / `PRODUCE_API_VERSIONS_RESPONSE_MIN_VERSION` +/// even though this gateway only accepts Produce v3+. +#[must_use] +pub const fn advertised_min_version(api_key: i16, firewall_min: i16) -> i16 { + if api_key == API_KEY_PRODUCE { + 0 + } else { + firewall_min + } +} + +fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { + let flexible = api_version >= 3; + let ranges = SUPPORTED_RANGES; + let mut e = Encoder::with_capacity(128); + + e.write_i16(error_code); + + if flexible { + e.write_varint((ranges.len() + 1) as u64); + for r in ranges { + e.write_i16(r.api_key); + e.write_i16(advertised_min_version(r.api_key, r.min_version)); + e.write_i16(r.max_version); + e.write_empty_tagged_fields(); + } + } else { + e.write_i32(i32::try_from(ranges.len()).expect("supported range table is small")); + for r in ranges { + e.write_i16(r.api_key); + e.write_i16(advertised_min_version(r.api_key, r.min_version)); + e.write_i16(r.max_version); + } + } + + if api_version >= 1 { + e.write_i32(0); + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +fn encode_metadata_response( + api_version: i16, + body: Bytes, + broker: &BrokerAdvertise, + top_level_error_code: i16, +) -> Bytes { + let flexible = api_version >= 9; + // Empty body = all-topics request; 0 topics is correct for this stub. + // Non-empty body that fails to decode = malformed request; return 0 topics. + // Kafka Metadata response has no top-level error code field: errors are per-topic only. + // 0 topics is spec-correct and unambiguous for a decode failure. + let (topics_count, effective_error) = if body.is_empty() { + (0usize, top_level_error_code) + } else { + split_metadata_request_topics(body, api_version) + .map_or((0, ERROR_INVALID_REQUEST), |n| (n, top_level_error_code)) + }; + let topic_error = if effective_error == ERROR_NONE { + ERROR_UNKNOWN_TOPIC_OR_PARTITION + } else { + effective_error + }; + + let mut e = Encoder::with_capacity(256); + + if api_version >= 3 { + e.write_i32(0); // throttle_time_ms (Metadata v3+) + } + + if flexible { + e.write_varint(2); // one broker (N+1) + e.write_i32(1); + e.write_compact_nullable_string(Some(&broker.host)); + e.write_i32(broker.port); + e.write_compact_nullable_string(None); // rack + e.write_empty_tagged_fields(); + + e.write_compact_nullable_string(None); // cluster_id (v2+) + e.write_i32(1); // controller_id (v1+) + + e.write_varint((topics_count + 1) as u64); + for _ in 0..topics_count { + e.write_i16(topic_error); + e.write_compact_nullable_string(Some("unknown-topic")); + e.write_bool(false); // is_internal (v1+) + e.write_varint(1); // empty partitions array + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations (v8+) + e.write_empty_tagged_fields(); + } + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // cluster_authorized_operations (v8+) + e.write_empty_tagged_fields(); + } else { + e.write_i32(1); // brokers array length + e.write_i32(1); // node_id + // broker.host is config-derived (KAFKA_ADVERTISED_HOST), not request-decoded — use + // the checked variant so an overly long hostname returns an error instead of panicking. + if e.write_nullable_string(Some(&broker.host)).is_err() { + return encode_error_only_response(ERROR_INVALID_REQUEST); + } + e.write_i32(broker.port); + if api_version >= 1 { + e.write_nullable_string_unchecked(None); // rack + } + + if api_version >= 2 { + e.write_nullable_string_unchecked(None); // cluster_id + } + if api_version >= 1 { + e.write_i32(1); // controller_id — must come before topics array + } + + e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); + for _ in 0..topics_count { + e.write_i16(topic_error); + e.write_nullable_string_unchecked(Some("unknown-topic")); + if api_version >= 1 { + e.write_bool(false); // is_internal + } + e.write_i32(0); // partitions array (empty) + if api_version >= 8 { + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations + } + } + if api_version >= 8 { + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // cluster_authorized_operations + } + } + + e.freeze() +} + +#[must_use] +pub fn encode_error_only_response(error_code: i16) -> Bytes { + let mut e = Encoder::with_capacity(2); + e.write_i16(error_code); + e.freeze() +} + +pub(crate) fn split_metadata_request_topics(body: Bytes, api_version: i16) -> Result { + let mut d = Decoder::new(body); + if api_version >= 9 { + d.read_compact_array_count() + } else { + d.read_i32_array_count() + } +} diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs new file mode 100644 index 0000000000..214c79af9e --- /dev/null +++ b/gateways/kafka/src/protocol/codec.rs @@ -0,0 +1,376 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Low-level Kafka primitive encoders/decoders (ported wire codec). + +#![allow(clippy::missing_const_for_fn, clippy::bool_to_int_with_if)] +#![allow( + clippy::missing_errors_doc, + clippy::cast_sign_loss, + clippy::must_use_candidate, + clippy::missing_panics_doc, + clippy::cast_possible_truncation, + clippy::cast_lossless +)] + +use bytes::{Buf, BufMut, Bytes, BytesMut}; + +use crate::error::{KafkaProtocolError, Result}; + +/// Upper bound for Kafka array/collection element counts decoded from the wire. +/// Matches typical broker limits and prevents OOM from adversarial length prefixes. +pub const MAX_COLLECTION_LEN: usize = 65_536; + +pub struct Decoder { + bytes: Bytes, +} + +impl Decoder { + pub fn new(bytes: Bytes) -> Self { + Self { bytes } + } + + pub fn remaining(&self) -> usize { + self.bytes.remaining() + } + + pub fn read_u8(&mut self) -> Result { + self.ensure(1)?; + Ok(self.bytes.get_u8()) + } + + pub fn read_i8(&mut self) -> Result { + self.ensure(1)?; + Ok(self.bytes.get_i8()) + } + + pub fn read_i16(&mut self) -> Result { + self.ensure(2)?; + Ok(self.bytes.get_i16()) + } + + pub fn read_i32(&mut self) -> Result { + self.ensure(4)?; + Ok(self.bytes.get_i32()) + } + + pub fn read_i64(&mut self) -> Result { + self.ensure(8)?; + Ok(self.bytes.get_i64()) + } + + pub fn read_bool(&mut self) -> Result { + Ok(self.read_i8()? != 0) + } + + /// Unsigned varint (Kafka uses this for compact array lengths and tagged-field counts). + /// Value is encoded with 7 bits per byte, LSB first; the high bit of each byte signals + /// that more bytes follow. + pub fn read_varint(&mut self) -> Result { + let mut result: u64 = 0; + let mut shift = 0u32; + loop { + let byte = self.read_u8()?; + if shift == 63 && byte & 0x7E != 0 { + return Err(KafkaProtocolError::InvalidVarint); + } + result |= ((byte & 0x7F) as u64) << shift; + if byte & 0x80 == 0 { + return Ok(result); + } + shift += 7; + if shift >= 64 { + return Err(KafkaProtocolError::InvalidVarint); + } + } + } + + /// Legacy array length: signed i32 count (must be non-negative). + pub fn read_i32_array_count(&mut self) -> Result { + let n = self.read_i32()?; + if n < 0 { + return Err(KafkaProtocolError::InvalidArrayLength(n)); + } + // Safe: n is in [0, i32::MAX]; i32::MAX (2_147_483_647) fits in usize + // on all 32-bit and 64-bit platforms this crate targets. + let count = n as usize; + if count > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count, + max: MAX_COLLECTION_LEN, + }); + } + Ok(count) + } + + /// Compact array length: unsigned varint holding `element_count + 1`. + /// Per the Kafka spec, varint=0 encodes a null (absent) array; treat as empty (0 elements) + /// so optional fields like `forgotten_topics` are skipped rather than rejected. + pub fn read_compact_array_count(&mut self) -> Result { + let n = self.read_varint()?; + if n == 0 { + return Ok(0); + } + let count = usize::try_from(n - 1).map_err(|_| KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + })?; + if count > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count, + max: MAX_COLLECTION_LEN, + }); + } + Ok(count) + } + + /// Legacy nullable string: i16 length prefix (-1 = null). + pub fn read_nullable_string(&mut self) -> Result> { + let len = self.read_i16()?; + if len < 0 { + return Ok(None); + } + let len = len as usize; + self.ensure(len)?; + let s = std::str::from_utf8(&self.bytes.chunk()[..len]) + .map_err(|_| KafkaProtocolError::InvalidUtf8)? + .to_owned(); + self.bytes.advance(len); + Ok(Some(s)) + } + + /// Compact nullable string (flexible versions): varint(len+1) prefix, 0 = null. + pub fn read_compact_nullable_string(&mut self) -> Result> { + let len_plus_one = self.read_varint()?; + if len_plus_one == 0 { + return Ok(None); + } + let len = usize::try_from(len_plus_one - 1).map_err(|_| { + KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + } + })?; + self.ensure(len)?; + let s = std::str::from_utf8(&self.bytes.chunk()[..len]) + .map_err(|_| KafkaProtocolError::InvalidUtf8)? + .to_owned(); + self.bytes.advance(len); + Ok(Some(s)) + } + + /// Legacy nullable bytes: i32 length prefix (-1 = null). + pub fn read_nullable_bytes(&mut self) -> Result> { + let len = self.read_i32()?; + if len < 0 { + return Ok(None); + } + let len = len as usize; + self.ensure(len)?; + Ok(Some(self.bytes.copy_to_bytes(len))) + } + + /// Compact nullable bytes (flexible versions): varint(len+1) prefix, 0 = null. + pub fn read_compact_nullable_bytes(&mut self) -> Result> { + let len_plus_one = self.read_varint()?; + if len_plus_one == 0 { + return Ok(None); + } + let len = usize::try_from(len_plus_one - 1).map_err(|_| { + KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + } + })?; + self.ensure(len)?; + Ok(Some(self.bytes.copy_to_bytes(len))) + } + + pub fn read_bytes(&mut self, len: usize) -> Result { + self.ensure(len)?; + Ok(self.bytes.copy_to_bytes(len)) + } + + /// Skip over a tagged-fields section. Each field is: tag (varint) + size (varint) + bytes. + /// A count of 0 is the common case (single byte 0x00). + pub fn read_tagged_fields(&mut self) -> Result<()> { + let count = self.read_varint()?; + if count > MAX_COLLECTION_LEN as u64 { + return Err(KafkaProtocolError::CollectionTooLarge { + count: count as usize, + max: MAX_COLLECTION_LEN, + }); + } + let count = count as usize; + + for _ in 0..count { + self.read_varint()?; // tag number + let size = usize::try_from(self.read_varint()?).map_err(|_| { + KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + } + })?; + self.ensure(size)?; + self.bytes.advance(size); + } + Ok(()) + } + + fn ensure(&self, needed: usize) -> Result<()> { + let remaining = self.bytes.remaining(); + if remaining < needed { + return Err(KafkaProtocolError::BufferUnderflow { needed, remaining }); + } + Ok(()) + } +} + +pub struct Encoder { + bytes: BytesMut, +} + +impl Encoder { + pub fn with_capacity(capacity: usize) -> Self { + Self { + bytes: BytesMut::with_capacity(capacity), + } + } + + pub fn write_u8(&mut self, v: u8) { + self.bytes.put_u8(v); + } + + pub fn write_i8(&mut self, v: i8) { + self.bytes.put_i8(v); + } + + pub fn write_i16(&mut self, v: i16) { + self.bytes.put_i16(v); + } + + pub fn write_i32(&mut self, v: i32) { + self.bytes.put_i32(v); + } + + pub fn write_i64(&mut self, v: i64) { + self.bytes.put_i64(v); + } + + pub fn write_bool(&mut self, v: bool) { + self.write_i8(if v { 1 } else { 0 }); + } + + /// Unsigned varint, 7 bits per byte, LSB first. + pub fn write_varint(&mut self, mut v: u64) { + loop { + let byte = (v & 0x7F) as u8; + v >>= 7; + if v == 0 { + self.bytes.put_u8(byte); + return; + } + self.bytes.put_u8(byte | 0x80); + } + } + + /// Legacy nullable string: i16 length prefix, -1 for null. + pub fn write_nullable_string(&mut self, v: Option<&str>) -> Result<()> { + match v { + None => self.write_i16(-1), + Some(s) => { + if s.len() > i16::MAX as usize { + return Err(KafkaProtocolError::StringTooLong { length: s.len() }); + } + self.write_i16(i16::try_from(s.len()).expect("checked above")); + self.bytes.put_slice(s.as_bytes()); + } + } + Ok(()) + } + + /// Infallible variant for response-encoding paths where the string originated from a decoded + /// Kafka request and is therefore already bounded to `i16::MAX` bytes. + pub fn write_nullable_string_unchecked(&mut self, v: Option<&str>) { + match v { + None => self.write_i16(-1), + Some(s) => { + debug_assert!(i16::try_from(s.len()).is_ok()); + self.write_i16(i16::try_from(s.len()).expect("caller guarantees len <= i16::MAX")); + self.bytes.put_slice(s.as_bytes()); + } + } + } + + /// Compact nullable string (flexible versions): varint(len+1), 0 for null. + pub fn write_compact_nullable_string(&mut self, v: Option<&str>) { + match v { + None => self.write_varint(0), + Some(s) => { + self.write_varint((s.len() + 1) as u64); + self.bytes.put_slice(s.as_bytes()); + } + } + } + + /// Write a null bytes field (i32 -1). Infallible; use instead of `write_nullable_bytes(None)`. + pub fn write_null_bytes(&mut self) { + self.write_i32(-1); + } + + /// Legacy nullable bytes: i32 length prefix, -1 for null. + pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) -> Result<()> { + match v { + None => self.write_i32(-1), + Some(b) => { + if b.len() > i32::MAX as usize { + return Err(KafkaProtocolError::CollectionTooLarge { + count: b.len(), + max: i32::MAX as usize, + }); + } + self.write_i32(i32::try_from(b.len()).expect("checked above")); + self.bytes.put_slice(b); + } + } + Ok(()) + } + + /// Compact nullable bytes (flexible versions): varint(len+1), 0 for null. + pub fn write_compact_nullable_bytes(&mut self, v: Option<&[u8]>) { + match v { + None => self.write_varint(0), + Some(b) => { + self.write_varint((b.len() + 1) as u64); + self.bytes.put_slice(b); + } + } + } + + pub fn write_bytes(&mut self, b: &[u8]) { + self.bytes.put_slice(b); + } + + /// Write an empty tagged-fields section (single 0x00 byte). + pub fn write_empty_tagged_fields(&mut self) { + self.write_varint(0); + } + #[must_use] + pub fn freeze(self) -> Bytes { + self.bytes.freeze() + } +} diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs new file mode 100644 index 0000000000..7977f4e5c9 --- /dev/null +++ b/gateways/kafka/src/protocol/header.rs @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![allow( + clippy::pedantic, + clippy::missing_const_for_fn, + clippy::match_same_arms +)] + +use bytes::{BufMut, Bytes}; + +use crate::error::{KafkaProtocolError, Result}; +use crate::protocol::codec::{Decoder, Encoder}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RequestHeader { + pub api_key: i16, + pub api_version: i16, + pub correlation_id: i32, + pub client_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResponseHeader { + pub correlation_id: i32, +} + +/// Returns the request header version to use for a given (api_key, api_version) pair. +/// +/// Header v1 is the standard non-flexible format (nullable string client_id). +/// Header v2 is the flexible format (compact nullable string client_id + empty tagged fields). +/// The threshold at which each API key switches from v1 to v2 is defined by the Kafka protocol. +pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { + let flexible_from: i16 = match api_key { + 0 => 9, // Produce + 1 => 12, // Fetch + 2 => 6, // ListOffsets + 3 => 9, // Metadata + 4 => 4, // LeaderAndIsr + 5 => 2, // StopReplica + 6 => 6, // UpdateMetadata + 7 => 3, // ControlledShutdown + 8 => 8, // OffsetCommit + 9 => 6, // OffsetFetch + 10 => 3, // FindCoordinator + 11 => 6, // JoinGroup + 12 => 4, // Heartbeat + 13 => 4, // LeaveGroup + 14 => 4, // SyncGroup + 15 => 5, // DescribeGroups + 16 => 3, // ListGroups + 17 => i16::MAX, // SaslHandshake — never flexible + 18 => 3, // ApiVersions + 19 => 5, // CreateTopics + 20 => 4, // DeleteTopics + 21 => 2, // DeleteRecords + 22 => 2, // InitProducerId + 23 => 4, // OffsetForLeaderEpoch + 24 => 3, // AddPartitionsToTxn + 25 => 3, // AddOffsetsToTxn + 26 => 3, // EndTxn + 27 => 1, // WriteTxnMarkers + 28 => 3, // TxnOffsetCommit + 29 => 2, // DescribeAcls + 30 => 2, // CreateAcls + 31 => 2, // DeleteAcls + 32 => 4, // DescribeConfigs + 33 => 2, // AlterConfigs + 34 => 2, // AlterReplicaLogDirs + 35 => 2, // DescribeLogDirs + 36 => 2, // SaslAuthenticate + 37 => 2, // CreatePartitions + 38 => 2, // CreateDelegationToken + 39 => 2, // RenewDelegationToken + 40 => 2, // ExpireDelegationToken + 41 => 2, // DescribeDelegationToken + 42 => 2, // DeleteGroups + 43 => 2, // ElectLeaders + 44 => 1, // IncrementalAlterConfigs + 45 => 0, // AlterPartitionReassignments — always flexible + 46 => 0, // ListPartitionReassignments — always flexible + 47 => i16::MAX, // OffsetDelete — never flexible + 48 => 1, // DescribeClientQuotas + 49 => 1, // AlterClientQuotas + 50 => 0, // DescribeUserScramCredentials — always flexible + 51 => 0, // AlterUserScramCredentials — always flexible + 55 => 0, // DescribeQuorum — always flexible + 56 => 0, // AlterPartition — always flexible + 57 => 1, // UpdateFeatures + 60 => 0, // DescribeCluster — always flexible + 61 => 0, // DescribeProducers — always flexible + 64 => 0, // UnregisterBroker — always flexible + 65 => 0, // DescribeTransactions — always flexible + 66 => 0, // ListTransactions — always flexible + 67 => 0, // AllocateProducerIds — always flexible + 68 => 0, // ConsumerGroupHeartbeat — always flexible + 69 => 0, // ConsumerGroupDescribe — always flexible + 71 => 0, // GetTelemetrySubscriptions — always flexible + 72 => 0, // PushTelemetry — always flexible + 74 => 0, // AssignReplicasToDirs — always flexible + 75 => 0, // DescribeTopicPartitions — always flexible + 76 => 0, // ListClientMetricsResources — always flexible + 77 => 0, // ShareGroupHeartbeat — always flexible (Kafka 4.0) + 78 => 0, // ShareGroupDescribe — always flexible + 79 => 0, // ShareFetch — always flexible + 80 => 0, // ShareAcknowledge — always flexible + _ => i16::MAX, // Unknown API — assume non-flexible + }; + if api_version >= flexible_from { 2 } else { 1 } +} + +/// Returns the response header version to use when replying to a given (api_key, api_version). +/// +/// ApiVersions (18) is a special case: the server ALWAYS returns response header v0 (no tagged +/// fields) so that clients that don't yet know the server supports flexible encoding can still +/// parse the discovery response. All other flexible-version APIs use response header v1. +pub fn response_header_version(api_key: i16, api_version: i16) -> i16 { + if api_key == 18 { + return 0; + } + if request_header_version(api_key, api_version) >= 2 { + 1 + } else { + 0 + } +} + +impl RequestHeader { + pub fn decode(bytes: Bytes, header_version: i16) -> Result { + let mut d = Decoder::new(bytes); + Self::decode_from(&mut d, header_version) + } + + /// Decode from a shared `Decoder`. + /// + /// Header v1 (non-flexible): + /// api_key i16 | api_version i16 | correlation_id i32 | client_id NULLABLE_STRING + /// + /// Header v2 (flexible): + /// api_key i16 | api_version i16 | correlation_id i32 + /// | client_id COMPACT_NULLABLE_STRING | _tagged_fields UNSIGNED_VARINT + pub fn decode_from(d: &mut Decoder, header_version: i16) -> Result { + match header_version { + 1 => { + let api_key = d.read_i16()?; + let api_version = d.read_i16()?; + let correlation_id = d.read_i32()?; + let client_id = d.read_nullable_string()?; + Ok(Self { + api_key, + api_version, + correlation_id, + client_id, + }) + } + 2 => { + let api_key = d.read_i16()?; + let api_version = d.read_i16()?; + let correlation_id = d.read_i32()?; + let client_id = d.read_compact_nullable_string()?; + d.read_tagged_fields()?; + Ok(Self { + api_key, + api_version, + correlation_id, + client_id, + }) + } + v => Err(KafkaProtocolError::UnsupportedHeaderVersion(v)), + } + } +} + +impl ResponseHeader { + /// Encode the response header. + /// + /// v0: correlation_id i32 (non-flexible APIs and ApiVersions) + /// v1: correlation_id i32 + empty tagged fields (flexible APIs) + pub fn encode(&self, header_version: i16) -> Bytes { + let mut e = Encoder::with_capacity(5); + e.write_i32(self.correlation_id); + if header_version >= 1 { + e.write_empty_tagged_fields(); + } + e.freeze() + } + + /// Write this header directly into an existing buffer (avoids a separate heap alloc). + pub fn encode_into(&self, buf: &mut bytes::BytesMut, header_version: i16) { + buf.put_i32(self.correlation_id); + if header_version >= 1 { + buf.put_u8(0); // empty tagged fields + } + } + + /// Byte size of the encoded header for a given version. + #[must_use] + pub fn encoded_size(header_version: i16) -> usize { + if header_version >= 1 { 5 } else { 4 } + } +} diff --git a/gateways/kafka/src/protocol/mod.rs b/gateways/kafka/src/protocol/mod.rs new file mode 100644 index 0000000000..3fe1fb544f --- /dev/null +++ b/gateways/kafka/src/protocol/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod api; +pub mod codec; +pub mod header; +pub mod requests; +pub mod responses; diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs new file mode 100644 index 0000000000..91e2a40b66 --- /dev/null +++ b/gateways/kafka/src/protocol/requests.rs @@ -0,0 +1,460 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka request decoders for critical API keys + +#![allow(clippy::pedantic)] + +use crate::error::{KafkaProtocolError, Result}; +use crate::protocol::codec::Decoder; +use bytes::Bytes; + +/// Produce Request (API Key 0) +#[derive(Debug, Clone)] +pub struct ProduceRequest { + pub transactional_id: Option, + pub acks: i16, + pub timeout_ms: i32, + pub topics: Vec, +} + +#[derive(Debug, Clone)] +pub struct ProduceTopicData { + pub topic: String, + pub partitions: Vec, +} + +#[derive(Debug, Clone)] +pub struct ProducePartitionData { + pub partition: i32, + pub records: Option, // Raw RecordBatch bytes +} + +pub fn decode_produce_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 9; + + // transactional_id (v3+) + let transactional_id = if version >= 3 { + if flexible { + d.read_compact_nullable_string()? + } else { + d.read_nullable_string()? + } + } else { + None + }; + + let acks = d.read_i16()?; + let timeout_ms = d.read_i32()?; + + // topics array + let topics_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let topic = if flexible { + d.read_compact_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + } else { + d.read_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + }; + + let partitions_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + + let mut partitions = Vec::with_capacity(partitions_count); + for _ in 0..partitions_count { + let partition = d.read_i32()?; + let records = if flexible { + d.read_compact_nullable_bytes()? + } else { + d.read_nullable_bytes()? + }; + partitions.push(ProducePartitionData { partition, records }); + if flexible { + d.read_tagged_fields()?; + } + } + + topics.push(ProduceTopicData { topic, partitions }); + if flexible { + d.read_tagged_fields()?; + } + } + + if flexible { + d.read_tagged_fields()?; + } + + Ok(ProduceRequest { + transactional_id, + acks, + timeout_ms, + topics, + }) +} + +/// Fetch Request (API Key 1) +#[derive(Debug, Clone)] +pub struct FetchRequest { + pub max_wait_ms: i32, + pub min_bytes: i32, + pub max_bytes: i32, + pub isolation_level: i8, + pub topics: Vec, +} + +#[derive(Debug, Clone)] +pub struct FetchTopic { + pub topic: String, + pub partitions: Vec, +} + +#[derive(Debug, Clone)] +pub struct FetchPartition { + pub partition: i32, + pub fetch_offset: i64, + pub partition_max_bytes: i32, +} + +pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 12; + + let _replica_id = d.read_i32()?; + let max_wait_ms = d.read_i32()?; + let min_bytes = d.read_i32()?; + + let max_bytes = if version >= 3 { + d.read_i32()? + } else { + 52_428_800 // default 50MB + }; + + let isolation_level = if version >= 4 { d.read_i8()? } else { 0 }; + + // session_id and session_epoch (v7+) — read and discard (stub path) + if version >= 7 { + d.read_i32()?; // session_id + d.read_i32()?; // session_epoch + } + + // topics array + let topics_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let topic = if flexible { + d.read_compact_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + } else { + d.read_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + }; + + let partitions_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + + let mut partitions = Vec::with_capacity(partitions_count); + for _ in 0..partitions_count { + let partition = d.read_i32()?; + + if version >= 9 { + d.read_i32()?; // current_leader_epoch + } + + let fetch_offset = d.read_i64()?; + + if version >= 12 { + d.read_i32()?; // last_fetched_epoch + } + + if version >= 5 { + d.read_i64()?; // log_start_offset + } + + let partition_max_bytes = d.read_i32()?; + + partitions.push(FetchPartition { + partition, + fetch_offset, + partition_max_bytes, + }); + + if flexible { + d.read_tagged_fields()?; + } + } + + topics.push(FetchTopic { topic, partitions }); + if flexible { + d.read_tagged_fields()?; + } + } + + // forgotten_topics_data (v7+) — skip + if version >= 7 { + let forgotten_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + for _ in 0..forgotten_count { + if flexible { + d.read_compact_nullable_string()?; + let partitions_count = d.read_compact_array_count()?; + for _ in 0..partitions_count { + d.read_i32()?; + } + d.read_tagged_fields()?; + } else { + d.read_nullable_string()?; + let partitions_count = d.read_i32_array_count()?; + for _ in 0..partitions_count { + d.read_i32()?; + } + } + } + } + + // rack_id (v11+) + if version >= 11 { + if flexible { + d.read_compact_nullable_string()?; + } else { + d.read_nullable_string()?; + } + } + + if flexible { + d.read_tagged_fields()?; + } + + Ok(FetchRequest { + max_wait_ms, + min_bytes, + max_bytes, + isolation_level, + topics, + }) +} + +/// ListOffsets Request (API Key 2) +#[derive(Debug, Clone)] +pub struct ListOffsetsRequest { + pub isolation_level: i8, + pub topics: Vec, +} + +#[derive(Debug, Clone)] +pub struct ListOffsetsTopic { + pub topic: String, + pub partitions: Vec, +} + +#[derive(Debug, Clone)] +pub struct ListOffsetsPartition { + pub partition: i32, + pub timestamp: i64, // -2 = earliest, -1 = latest +} + +pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 6; + + let _replica_id = d.read_i32()?; + + let isolation_level = if version >= 2 { d.read_i8()? } else { 0 }; + + let topics_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let topic = if flexible { + d.read_compact_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + } else { + d.read_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + }; + + let partitions_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + + let mut partitions = Vec::with_capacity(partitions_count); + for _ in 0..partitions_count { + let partition = d.read_i32()?; + + if version >= 4 { + d.read_i32()?; // current_leader_epoch + } + + let timestamp = d.read_i64()?; + + if version == 0 { + d.read_i32()?; // max_num_offsets (deprecated) + } + + partitions.push(ListOffsetsPartition { + partition, + timestamp, + }); + + if flexible { + d.read_tagged_fields()?; + } + } + + topics.push(ListOffsetsTopic { topic, partitions }); + if flexible { + d.read_tagged_fields()?; + } + } + + if flexible { + d.read_tagged_fields()?; + } + + Ok(ListOffsetsRequest { + isolation_level, + topics, + }) +} + +/// CreateTopics Request (API Key 19) +#[derive(Debug, Clone)] +pub struct CreateTopicsRequest { + pub topics: Vec, + pub timeout_ms: i32, + pub validate_only: bool, +} + +#[derive(Debug, Clone)] +pub struct CreatableTopic { + pub name: String, + pub num_partitions: i32, + pub replication_factor: i16, +} + +pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 5; + + let topics_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let name = if flexible { + d.read_compact_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + } else { + d.read_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + }; + + let num_partitions = d.read_i32()?; + let replication_factor = d.read_i16()?; + + // assignments (COMPACT_ARRAY or ARRAY) — skip + let assignments_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + for _ in 0..assignments_count { + d.read_i32()?; // partition_index + let replicas_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + for _ in 0..replicas_count { + d.read_i32()?; // broker_id + } + if flexible { + d.read_tagged_fields()?; + } + } + + // configs (COMPACT_ARRAY or ARRAY) — skip + let configs_count = if flexible { + d.read_compact_array_count()? + } else { + d.read_i32_array_count()? + }; + for _ in 0..configs_count { + if flexible { + d.read_compact_nullable_string()?; // name + d.read_compact_nullable_string()?; // value + d.read_tagged_fields()?; + } else { + d.read_nullable_string()?; + d.read_nullable_string()?; + } + } + + topics.push(CreatableTopic { + name, + num_partitions, + replication_factor, + }); + + if flexible { + d.read_tagged_fields()?; + } + } + + let timeout_ms = d.read_i32()?; + let validate_only = if version >= 1 { d.read_bool()? } else { false }; + + if flexible { + d.read_tagged_fields()?; + } + + Ok(CreateTopicsRequest { + topics, + timeout_ms, + validate_only, + }) +} diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs new file mode 100644 index 0000000000..8c44692393 --- /dev/null +++ b/gateways/kafka/src/protocol/responses.rs @@ -0,0 +1,366 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka response encoders (stub implementations). + +#![allow(clippy::pedantic)] + +use crate::protocol::api::{ERROR_INVALID_PARTITIONS, ERROR_NONE}; +use crate::protocol::codec::Encoder; +use crate::protocol::requests::{ + CreateTopicsRequest, FetchRequest, ListOffsetsRequest, ProducePartitionData, ProduceRequest, + ProduceTopicData, +}; +use bytes::Bytes; + +/// Well-formed Produce response with a single placeholder topic/partition. +pub fn encode_produce_error_response(version: i16, error_code: i16) -> Bytes { + let topics = vec![ProduceTopicData { + topic: String::new(), // TODO topic name will be populated in the end to end functional completion + partitions: vec![ProducePartitionData { + partition: 0, + records: None, + }], + }]; + encode_produce_response_inner(version, &topics, error_code) +} + +pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { + encode_produce_response_inner(version, &req.topics, ERROR_NONE) +} + +fn encode_produce_response_inner( + version: i16, + topics: &[ProduceTopicData], + partition_error: i16, +) -> Bytes { + let flexible = version >= 9; + let mut e = Encoder::with_capacity(512); + + if flexible { + e.write_varint((topics.len() + 1) as u64); + } else { + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); + } + + for topic in topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.topic)); + } else { + e.write_nullable_string_unchecked(Some(&topic.topic)); + } + + if flexible { + e.write_varint((topic.partitions.len() + 1) as u64); + } else { + e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); + } + + for p in &topic.partitions { + e.write_i32(p.partition); + e.write_i16(partition_error); + e.write_i64(0); + if version >= 2 { + e.write_i64(-1); + } + if version >= 5 { + e.write_i64(0); + } + if version >= 8 { + if flexible { + e.write_varint(1); + e.write_compact_nullable_string(None); + } else { + e.write_i32(0); + e.write_nullable_string_unchecked(None); + } + } + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if version >= 1 { + e.write_i32(0); + } + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +/// Well-formed Fetch response. Uses top-level `error_code` at v7+, or a single +/// placeholder topic/partition with per-partition `error_code` below v7. +pub fn encode_fetch_error_response(version: i16, error_code: i16) -> Bytes { + use crate::protocol::requests::{FetchPartition, FetchTopic}; + + if version >= 7 { + return encode_fetch_response_inner(version, &[], Some(error_code), error_code); + } + + let topics = vec![FetchTopic { + topic: String::new(), + partitions: vec![FetchPartition { + partition: 0, + fetch_offset: 0, + partition_max_bytes: 1, + }], + }]; + encode_fetch_response_inner(version, &topics, Some(ERROR_NONE), error_code) +} + +pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { + encode_fetch_response_inner(version, &req.topics, Some(ERROR_NONE), ERROR_NONE) +} + +fn encode_fetch_response_inner( + version: i16, + topics: &[crate::protocol::requests::FetchTopic], + top_level_error: Option, + partition_error: i16, +) -> Bytes { + let flexible = version >= 12; + let mut e = Encoder::with_capacity(512); + + if version >= 1 { + e.write_i32(0); + } + if version >= 7 { + e.write_i16(top_level_error.unwrap_or(ERROR_NONE)); + e.write_i32(0); + } + + if flexible { + e.write_varint((topics.len() + 1) as u64); + } else { + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); + } + + for topic in topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.topic)); + } else { + e.write_nullable_string_unchecked(Some(&topic.topic)); + } + + if flexible { + e.write_varint((topic.partitions.len() + 1) as u64); + } else { + e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); + } + + for partition in &topic.partitions { + e.write_i32(partition.partition); + e.write_i16(partition_error); + e.write_i64(0); // high_watermark + if version >= 4 { + e.write_i64(0); // last_stable_offset + } + if version >= 5 { + e.write_i64(0); // log_start_offset + } + if version >= 4 { + if flexible { + e.write_varint(1); // empty aborted_transactions + } else { + e.write_i32(0); // empty aborted_transactions + } + } + if version >= 11 { + e.write_i32(-1); // preferred_read_replica + } + if flexible { + e.write_compact_nullable_bytes(None); + } else { + e.write_null_bytes(); + } + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +/// Well-formed ListOffsets response with a single placeholder topic/partition. +pub fn encode_list_offsets_error_response(version: i16, error_code: i16) -> Bytes { + use crate::protocol::requests::{ListOffsetsPartition, ListOffsetsTopic}; + + let topics = vec![ListOffsetsTopic { + topic: String::new(), + partitions: vec![ListOffsetsPartition { + partition: 0, + timestamp: -1, + }], + }]; + encode_list_offsets_response_inner(version, &topics, error_code) +} + +pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> Bytes { + encode_list_offsets_response_inner(version, &req.topics, ERROR_NONE) +} + +fn encode_list_offsets_response_inner( + version: i16, + topics: &[crate::protocol::requests::ListOffsetsTopic], + partition_error: i16, +) -> Bytes { + let flexible = version >= 6; + let mut e = Encoder::with_capacity(256); + + if version >= 2 { + e.write_i32(0); + } + + if flexible { + e.write_varint((topics.len() + 1) as u64); + } else { + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); + } + + for topic in topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.topic)); + } else { + e.write_nullable_string_unchecked(Some(&topic.topic)); + } + + if flexible { + e.write_varint((topic.partitions.len() + 1) as u64); + } else { + e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); + } + + for partition in &topic.partitions { + e.write_i32(partition.partition); + e.write_i16(partition_error); + + let offset = 0i64; + if version >= 1 { + e.write_i64(-1); // -1 = timestamp not available (Kafka sentinel) + } + e.write_i64(offset); + if version >= 4 { + e.write_i32(-1); + } + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +/// Well-formed CreateTopics response with a single placeholder topic. +pub fn encode_create_topics_error_response(version: i16, error_code: i16) -> Bytes { + use crate::protocol::requests::CreatableTopic; + + let topics = vec![CreatableTopic { + name: String::new(), + num_partitions: 1, + replication_factor: 1, + }]; + encode_create_topics_response_inner(version, &topics, error_code) +} + +pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> Bytes { + encode_create_topics_response_inner(version, &req.topics, ERROR_NONE) +} + +fn encode_create_topics_response_inner( + version: i16, + topics: &[crate::protocol::requests::CreatableTopic], + topic_error: i16, +) -> Bytes { + let flexible = version >= 5; + let mut e = Encoder::with_capacity(256); + + if version >= 2 { + e.write_i32(0); + } + + if flexible { + e.write_varint((topics.len() + 1) as u64); + } else { + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); + } + + for topic in topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.name)); + } else { + e.write_nullable_string_unchecked(Some(&topic.name)); + } + + let error_code = if topic_error != ERROR_NONE { + topic_error + } else if topic.num_partitions <= 0 { + ERROR_INVALID_PARTITIONS + } else { + ERROR_NONE + }; + e.write_i16(error_code); + + if version >= 1 { + if flexible { + e.write_compact_nullable_string(None); + } else { + e.write_nullable_string_unchecked(None); + } + } + + if version >= 5 { + e.write_i32(topic.num_partitions); + e.write_i16(topic.replication_factor); + e.write_varint(1); + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs new file mode 100644 index 0000000000..a41d5fa9bd --- /dev/null +++ b/gateways/kafka/src/server.rs @@ -0,0 +1,394 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use bytes::{BufMut, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::broadcast; +use tokio::time::{timeout, timeout_at}; +use tokio_util::task::TaskTracker; +use tracing::{debug, error, info, warn}; + +use crate::error::{KafkaProtocolError, Result}; +use crate::protocol::api::{ + BrokerAdvertise, DEFAULT_KAFKA_PORT, ERROR_INVALID_REQUEST, encode_error_only_response, + handle_request, +}; +use crate::protocol::codec::Decoder; +use crate::protocol::header::{ + RequestHeader, ResponseHeader, request_header_version, response_header_version, +}; +use std::io; + +const READ_CHUNK: usize = 65536; + +#[derive(Debug, Clone)] +pub struct ServerConfig { + pub bind_addr: String, + /// Hostname or IP advertised in Metadata (`KAFKA_ADVERTISED_HOST`). Required when `bind_addr` + /// uses a wildcard address (`0.0.0.0` / `::`). + pub advertised_host: Option, + /// Port advertised in Metadata (`KAFKA_ADVERTISED_PORT`). Defaults to the bind port. + pub advertised_port: Option, + pub max_frame_size: usize, + pub read_timeout: Duration, + pub write_timeout: Duration, +} + +impl Default for ServerConfig { + fn default() -> Self { + Self { + bind_addr: format!("127.0.0.1:{DEFAULT_KAFKA_PORT}"), + advertised_host: None, + advertised_port: None, + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(15), + write_timeout: Duration::from_secs(10), + } + } +} + +impl BrokerAdvertise { + /// Resolve the broker endpoint advertised in Metadata. + /// + /// `local_addr` is the address the listener is actually bound to (from `listener.local_addr()`). + /// + /// # Errors + /// + /// Returns `InvalidConfig` when `advertised_host` is empty or the listener binds to a wildcard + /// without an explicit advertised host. + pub fn from_server_config(config: &ServerConfig, local_addr: SocketAddr) -> Result { + let port = config + .advertised_port + .map_or_else(|| i32::from(local_addr.port()), i32::from); + + let host = if let Some(ref advertised) = config.advertised_host { + let trimmed = advertised.trim(); + if trimmed.is_empty() { + return Err(KafkaProtocolError::InvalidConfig( + "KAFKA_ADVERTISED_HOST must not be empty".into(), + )); + } + if trimmed.len() > i16::MAX as usize { + return Err(KafkaProtocolError::InvalidConfig( + "KAFKA_ADVERTISED_HOST exceeds Kafka nullable string limit (32767 bytes)" + .into(), + )); + } + trimmed.to_string() + } else if local_addr.ip().is_unspecified() { + return Err(KafkaProtocolError::InvalidConfig( + "binding to a wildcard address (0.0.0.0 or ::) requires KAFKA_ADVERTISED_HOST \ + to be set to a reachable hostname or IP for Metadata broker advertisement" + .into(), + )); + } else { + local_addr.ip().to_string() + }; + + Ok(Self { host, port }) + } +} + +pub struct KafkaServer { + config: Arc, +} + +impl KafkaServer { + #[must_use] + pub fn new(config: ServerConfig) -> Self { + Self { + config: Arc::new(config), + } + } + + /// Accept Kafka wire connections until `shutdown` fires, then drain in-flight tasks. + /// + /// `listener` must already be bound by the caller. This lets tests and `main` bind + /// the port before spawning the task, eliminating the TOCTOU race of bind-drop-rebind. + /// + /// # Errors + /// + /// Returns an error on invalid config or a non-transient `accept()` error. + pub async fn run( + self, + listener: TcpListener, + mut shutdown: broadcast::Receiver<()>, + ) -> Result<()> { + let local_addr = listener.local_addr()?; + let broker = Arc::new(BrokerAdvertise::from_server_config( + &self.config, + local_addr, + )?); + info!( + "kafka listener bound on {} (advertised as {}:{})", + local_addr, broker.host, broker.port + ); + + let tracker = TaskTracker::new(); + let broker = Arc::clone(&broker); + + loop { + tokio::select! { + result = shutdown.recv() => { + match result { + Ok(()) => { + info!("kafka listener shutdown requested"); + tracker.close(); + tracker.wait().await; + break; + } + // Capacity-1 channel: lagged means a signal was sent before we polled — treat as shutdown. + Err(broadcast::error::RecvError::Lagged(_)) => { + info!("kafka listener shutdown requested (lagged)"); + tracker.close(); + tracker.wait().await; + break; + } + Err(broadcast::error::RecvError::Closed) => { + tracker.close(); + tracker.wait().await; + break; + } + } + } + accept_result = listener.accept() => { + match accept_result { + Ok((stream, peer)) => { + if let Err(e) = stream.set_nodelay(true) { + warn!(%peer, "TCP_NODELAY failed: {e}"); + } + let cfg = Arc::clone(&self.config); + let broker = Arc::clone(&broker); + tracker.spawn(async move { + if let Err(err) = handle_connection(stream, cfg, peer, broker).await { + warn!(%peer, "connection closed with error: {err}"); + } + }); + } + Err(e) if is_transient_accept_error(&e) => { + // Brief backoff on fd exhaustion to avoid busy-spinning. + if matches!(e.raw_os_error(), Some(23 | 24)) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + warn!(%e, "transient accept error, continuing"); + } + Err(e) => return Err(e.into()), + } + } + } + } + Ok(()) + } +} + +fn is_transient_accept_error(err: &std::io::Error) -> bool { + use std::io::ErrorKind; + + matches!( + err.kind(), + ErrorKind::Interrupted | ErrorKind::ConnectionAborted | ErrorKind::WouldBlock + ) || matches!( + err.raw_os_error(), + // EMFILE / ENFILE are common across Unix platforms when fd limits are hit. + Some(23 | 24) + ) +} + +async fn handle_connection( + mut stream: TcpStream, + config: Arc, + peer: SocketAddr, + broker: Arc, +) -> Result<()> { + debug!(%peer, "connection accepted"); + + loop { + let frame = match read_frame(&mut stream, config.max_frame_size, config.read_timeout).await + { + Ok(f) => f, + Err(KafkaProtocolError::Io(ref e)) + if e.kind() == std::io::ErrorKind::UnexpectedEof + || e.kind() == std::io::ErrorKind::ConnectionReset => + { + info!(%peer, "connection closed by client"); + return Ok(()); + } + Err(e) => return Err(e), + }; + + if frame.len() < 8 { + return Err(KafkaProtocolError::BufferUnderflow { + needed: 8, + remaining: frame.len(), + }); + } + let api_key = i16::from_be_bytes([frame[0], frame[1]]); + let api_version = i16::from_be_bytes([frame[2], frame[3]]); + let req_hdr_ver = request_header_version(api_key, api_version); + let resp_hdr_ver = response_header_version(api_key, api_version); + let correlation_id = correlation_id_from_frame(&frame); + + let mut decoder = Decoder::new(frame); + let req = match RequestHeader::decode_from(&mut decoder, req_hdr_ver) { + Ok(req) => req, + Err(KafkaProtocolError::UnsupportedHeaderVersion(_)) => { + warn!(%peer, api_key, api_version, "unsupported request header version"); + let body_response = encode_error_only_response(ERROR_INVALID_REQUEST); + let resp_header = ResponseHeader { correlation_id }; + send_response( + &mut stream, + &resp_header, + 0, + &body_response, + config.write_timeout, + ) + .await?; + return Ok(()); + } + Err(e) => return Err(e), + }; + + debug!( + %peer, + api_key = req.api_key, + api_version = req.api_version, + correlation_id = req.correlation_id, + client_id = req.client_id.as_deref().unwrap_or(""), + "received request" + ); + + let body = decoder.read_bytes(decoder.remaining())?; + let body_response = handle_request(req.api_key, req.api_version, body, &broker); + + let resp_header = ResponseHeader { + correlation_id: req.correlation_id, + }; + send_response( + &mut stream, + &resp_header, + resp_hdr_ver, + &body_response, + config.write_timeout, + ) + .await?; + } +} + +/// Write a single length-prefixed Kafka frame using one allocation. +/// Avoids the separate header-encode + payload-concat + length-prefix allocations. +async fn send_response( + stream: &mut TcpStream, + header: &ResponseHeader, + header_version: i16, + body: &[u8], + write_timeout: Duration, +) -> Result<()> { + let header_size = ResponseHeader::encoded_size(header_version); + let payload_size = header_size + body.len(); + let payload_len_i32 = + i32::try_from(payload_size).map_err(|_| KafkaProtocolError::FrameTooLarge { + max_bytes: i32::MAX as usize, + actual_bytes: payload_size, + })?; + let mut frame = BytesMut::with_capacity(4 + payload_size); + frame.put_i32(payload_len_i32); + header.encode_into(&mut frame, header_version); + frame.put_slice(body); + timeout(write_timeout, stream.write_all(&frame)) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "write timeout"))??; + Ok(()) +} + +fn correlation_id_from_frame(frame: &bytes::Bytes) -> i32 { + i32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]) +} + +/// Read one length-prefixed Kafka frame from `stream`. +/// +/// # Errors +/// +/// Returns an error on timeout, invalid length, or I/O failure. +pub async fn read_frame( + stream: &mut TcpStream, + max_frame_size: usize, + read_timeout: Duration, +) -> Result { + // Single deadline for both the length-prefix read and the body read. Without this, a + // slow-drip sender could hold a connection open for 2x read_timeout by sending one byte + // per timeout window. + let deadline = tokio::time::Instant::now() + read_timeout; + let mut len_buf = [0u8; 4]; + timeout_at(deadline, stream.read_exact(&mut len_buf)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; + + let frame_len_i32 = i32::from_be_bytes(len_buf); + if frame_len_i32 <= 0 { + return Err(KafkaProtocolError::InvalidFrameLength(frame_len_i32)); + } + + let frame_len = + usize::try_from(frame_len_i32).map_err(|_| KafkaProtocolError::FrameTooLarge { + max_bytes: max_frame_size, + actual_bytes: usize::MAX, + })?; + if frame_len > max_frame_size { + return Err(KafkaProtocolError::FrameTooLarge { + max_bytes: max_frame_size, + actual_bytes: frame_len, + }); + } + + // read_buf() exposes all BytesMut spare capacity to the OS; after reserve(n) the + // allocator may give more than n bytes, so the OS can fill past frame_len and silently + // consume bytes belonging to the next pipelined frame. Use read() with a bounded slice + // so each OS call is limited to exactly the remaining bytes needed. + let mut data = BytesMut::with_capacity(frame_len); + while data.len() < frame_len { + let remaining = frame_len - data.len(); + let chunk = remaining.min(READ_CHUNK); + let prev = data.len(); + data.resize(prev + chunk, 0); + let n = match timeout_at(deadline, stream.read(&mut data[prev..prev + chunk])).await { + Err(_) => return Err(io::Error::new(io::ErrorKind::TimedOut, "read timeout").into()), + Ok(Ok(0)) => { + return Err( + io::Error::new(io::ErrorKind::UnexpectedEof, "connection closed").into(), + ); + } + Ok(Err(e)) => return Err(e.into()), + Ok(Ok(n)) => n, + }; + data.truncate(prev + n); + } + Ok(data.freeze()) +} + +pub fn init_tracing() { + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + let _ = tracing_subscriber::fmt() + .with_env_filter(filter) + .try_init() + .map_err(|e| error!("failed to initialize tracing: {e}")); +} diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs new file mode 100644 index 0000000000..ab3e762332 --- /dev/null +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, ERROR_UNSUPPORTED_VERSION, + handle_request, is_supported_version, supported_api_ranges, +}; + +fn test_broker() -> BrokerAdvertise { + BrokerAdvertise::default() +} +use iggy_gateway_kafka::protocol::codec::Decoder; + +// ── ApiVersions ───────────────────────────────────────────────────────────── + +#[test] +fn api_versions_v1_response_non_flexible_format() { + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &test_broker()); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i16().unwrap(), 0); // error_code + + // Non-flexible: i32 array count + let count = d.read_i32().unwrap(); + assert!(count >= 2); + let mut keys = Vec::new(); + for _ in 0..count { + keys.push(d.read_i16().unwrap()); + d.read_i16().unwrap(); // min + d.read_i16().unwrap(); // max + } + assert_eq!(d.read_i32().unwrap(), 0); // throttle_time_ms + + let expected_keys: Vec = supported_api_ranges().iter().map(|r| r.api_key).collect(); + for k in expected_keys { + assert!(keys.contains(&k)); + } +} + +#[test] +fn api_versions_v3_response_flexible_format() { + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &test_broker()); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i16().unwrap(), 0); // error_code + + // Flexible: varint(len+1) compact array + let count_plus_one = d.read_varint().unwrap(); + assert!(count_plus_one >= 3); // at least 2 entries → varint = 3+ + let count = i32::try_from(count_plus_one - 1).expect("api count fits i32"); + + let mut keys = Vec::new(); + for _ in 0..count { + keys.push(d.read_i16().unwrap()); + d.read_i16().unwrap(); // min + d.read_i16().unwrap(); // max + d.read_tagged_fields().unwrap(); // per-entry tagged fields + } + assert_eq!(d.read_i32().unwrap(), 0); // throttle_time_ms + d.read_tagged_fields().unwrap(); // top-level tagged fields + + let expected_keys: Vec = supported_api_ranges().iter().map(|r| r.api_key).collect(); + for k in expected_keys { + assert!(keys.contains(&k)); + } +} + +// ── Metadata ───────────────────────────────────────────────────────────────── + +#[test] +fn metadata_response_has_broker_array_and_topic_array() { + let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &test_broker()); + let mut d = Decoder::new(body); + + let broker_count = d.read_i32().unwrap(); + assert_eq!(broker_count, 1); + let node_id = d.read_i32().unwrap(); + assert_eq!(node_id, 1); + let host = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(host, "127.0.0.1"); + let port = d.read_i32().unwrap(); + assert_eq!(port, 9093); + + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 0); +} + +#[test] +fn unsupported_version_returns_protocol_error() { + // v99 client sends a compact-array body (count+1 = 2 = 0x02 for 1 topic). + // The gateway caps at v9 (highest supported Metadata version) for both parsing + // and encoding, so the response uses the flexible (v9) wire format. + let body = handle_request( + API_KEY_METADATA, + 99, + Bytes::from_static(&[0x02]), + &test_broker(), + ); + let mut d = Decoder::new(body); + // v9 flexible response layout: + d.read_i32().unwrap(); // throttle_time_ms (v3+) + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); // node_id + d.read_compact_nullable_string().unwrap(); // host + d.read_i32().unwrap(); // port + d.read_compact_nullable_string().unwrap(); // rack + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); // cluster_id (v2+) + d.read_i32().unwrap(); // controller_id (v1+) + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!(topic_count, 1); + let topic_error = d.read_i16().unwrap(); + assert_eq!(topic_error, ERROR_UNSUPPORTED_VERSION); + let topic_name = d.read_compact_nullable_string().unwrap(); + assert_eq!(topic_name, Some("unknown-topic".to_string())); +} + +// ── Misc ──────────────────────────────────────────────────────────────────── + +#[test] +fn unknown_api_key_returns_error_only_payload() { + let body = handle_request(999, 0, Bytes::new(), &test_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn version_support_table_is_applied() { + assert!(is_supported_version(API_KEY_API_VERSIONS, 3)); + assert!(!is_supported_version(API_KEY_API_VERSIONS, 10)); + assert!(is_supported_version(API_KEY_METADATA, 1)); + assert!(!is_supported_version(API_KEY_METADATA, -1)); +} + +#[test] +fn apiversions_unsupported_version_uses_v0_encoding_without_throttle() { + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &test_broker()); + // v0: error_code(2) + api_keys i32 count(4) + 6 entries × 6 bytes = 42 — no throttle_time_ms. + assert_eq!(body.len(), 42); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i32().unwrap(), 6); + assert_eq!(d.remaining(), 36); +} diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs new file mode 100644 index 0000000000..a445540dd7 --- /dev/null +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `BrokerAdvertise` parsing and metadata reflection. + +use std::net::SocketAddr; + +use iggy_gateway_kafka::ServerConfig; +use iggy_gateway_kafka::protocol::api::{API_KEY_METADATA, BrokerAdvertise, handle_request}; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +#[test] +fn default_matches_standard_gateway_port() { + let b = BrokerAdvertise::default(); + assert_eq!(b.host, "127.0.0.1"); + assert_eq!(b.port, 9093); +} + +#[test] +fn metadata_reflects_broker_addr() { + let broker = BrokerAdvertise { + host: "203.0.113.7".to_string(), + port: 9093, + }; + let mut req = Encoder::with_capacity(4); + req.write_i32(0); + let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker); + + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + assert_eq!(host, "203.0.113.7"); + assert_eq!(port, 9093); +} + +#[test] +fn from_server_config_uses_explicit_advertised_host_on_wildcard_bind() { + let config = ServerConfig { + bind_addr: "0.0.0.0:9093".to_string(), + advertised_host: Some("kafka.internal".to_string()), + ..ServerConfig::default() + }; + let local_addr: SocketAddr = "0.0.0.0:9093".parse().unwrap(); + let broker = BrokerAdvertise::from_server_config(&config, local_addr).expect("valid config"); + assert_eq!(broker.host, "kafka.internal"); + assert_eq!(broker.port, 9093); +} + +#[test] +fn from_server_config_rejects_wildcard_bind_without_advertised_host() { + let config = ServerConfig { + bind_addr: "0.0.0.0:9093".to_string(), + ..ServerConfig::default() + }; + let local_addr: SocketAddr = "0.0.0.0:9093".parse().unwrap(); + let err = BrokerAdvertise::from_server_config(&config, local_addr).unwrap_err(); + assert!(err.to_string().contains("KAFKA_ADVERTISED_HOST")); +} + +#[test] +fn from_server_config_uses_bind_ip_for_non_wildcard_listener() { + let config = ServerConfig { + bind_addr: "192.168.1.10:19092".to_string(), + ..ServerConfig::default() + }; + let local_addr: SocketAddr = "192.168.1.10:19092".parse().unwrap(); + let broker = BrokerAdvertise::from_server_config(&config, local_addr).expect("valid config"); + assert_eq!(broker.host, "192.168.1.10"); + assert_eq!(broker.port, 19092); +} + +#[test] +fn from_server_config_rejects_advertised_host_exceeding_kafka_string_limit() { + let config = ServerConfig { + bind_addr: "127.0.0.1:9093".to_string(), + advertised_host: Some("x".repeat(i16::MAX as usize + 1)), + ..ServerConfig::default() + }; + let local_addr: SocketAddr = "127.0.0.1:9093".parse().unwrap(); + let err = BrokerAdvertise::from_server_config(&config, local_addr).unwrap_err(); + assert!(err.to_string().contains("KAFKA_ADVERTISED_HOST")); +} + +#[test] +fn from_server_config_honors_advertised_port_override() { + let config = ServerConfig { + bind_addr: "127.0.0.1:9093".to_string(), + advertised_host: Some("broker.example.com".to_string()), + advertised_port: Some(19093), + ..ServerConfig::default() + }; + let local_addr: SocketAddr = "127.0.0.1:9093".parse().unwrap(); + let broker = BrokerAdvertise::from_server_config(&config, local_addr).expect("valid config"); + assert_eq!(broker.host, "broker.example.com"); + assert_eq!(broker.port, 19093); +} diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs new file mode 100644 index 0000000000..fcaa966477 --- /dev/null +++ b/gateways/kafka/tests/codec_tests.rs @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +#[test] +fn codec_round_trip_primitives_and_nullable_fields() { + let mut enc = Encoder::with_capacity(128); + enc.write_i8(-3); + enc.write_i16(42); + enc.write_i32(123_456); + enc.write_i64(9_999_999); + enc.write_nullable_string(Some("client-a")).unwrap(); + enc.write_nullable_string(None).unwrap(); + enc.write_nullable_bytes(Some(&[1, 2, 3])).unwrap(); + enc.write_nullable_bytes(None).unwrap(); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!(dec.read_i8().unwrap(), -3); + assert_eq!(dec.read_i16().unwrap(), 42); + assert_eq!(dec.read_i32().unwrap(), 123_456); + assert_eq!(dec.read_i64().unwrap(), 9_999_999); + assert_eq!( + dec.read_nullable_string().unwrap().as_deref(), + Some("client-a") + ); + assert_eq!(dec.read_nullable_string().unwrap(), None); + assert_eq!( + dec.read_nullable_bytes().unwrap().unwrap(), + Bytes::from_static(&[1, 2, 3]) + ); + assert_eq!(dec.read_nullable_bytes().unwrap(), None); +} + +#[test] +fn decoder_returns_underflow_error() { + let mut dec = Decoder::new(Bytes::from_static(&[0x00])); + let err = dec.read_i32().expect_err("must fail"); + assert!(err.to_string().contains("buffer underflow")); +} + +#[test] +fn codec_u8_and_bool() { + let mut enc = Encoder::with_capacity(8); + enc.write_u8(0xFF); + enc.write_bool(true); + enc.write_bool(false); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!(dec.read_u8().unwrap(), 0xFF); + assert!(dec.read_bool().unwrap()); + assert!(!dec.read_bool().unwrap()); +} + +#[test] +fn varint_round_trip_small_values() { + for v in [ + 0u64, + 1, + 127, + 128, + 255, + 300, + 16383, + 16384, + u64::from(u32::MAX), + ] { + let mut enc = Encoder::with_capacity(16); + enc.write_varint(v); + let mut dec = Decoder::new(enc.freeze()); + assert_eq!(dec.read_varint().unwrap(), v, "failed for v={v}"); + } +} + +#[test] +fn varint_single_byte_for_values_below_128() { + let mut enc = Encoder::with_capacity(1); + enc.write_varint(42); + let bytes = enc.freeze(); + assert_eq!(bytes.len(), 1); + assert_eq!(bytes[0], 42); +} + +#[test] +fn varint_two_bytes_for_128() { + let mut enc = Encoder::with_capacity(2); + enc.write_varint(128); + let bytes = enc.freeze(); + // 128 = 0x80 → first byte 0x80 | 0x80 = 0x80 (continue), second byte 0x01 + assert_eq!(bytes.as_ref(), &[0x80, 0x01]); +} + +#[test] +fn compact_nullable_string_round_trip() { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_string(Some("hello")); + enc.write_compact_nullable_string(None); + enc.write_compact_nullable_string(Some("")); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!( + dec.read_compact_nullable_string().unwrap().as_deref(), + Some("hello") + ); + assert_eq!(dec.read_compact_nullable_string().unwrap(), None); + assert_eq!( + dec.read_compact_nullable_string().unwrap().as_deref(), + Some("") + ); +} + +#[test] +fn compact_nullable_bytes_round_trip() { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_bytes(Some(&[10, 20, 30])); + enc.write_compact_nullable_bytes(None); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!( + dec.read_compact_nullable_bytes().unwrap().unwrap(), + Bytes::from_static(&[10, 20, 30]) + ); + assert_eq!(dec.read_compact_nullable_bytes().unwrap(), None); +} + +#[test] +fn tagged_fields_empty_section_round_trip() { + let mut enc = Encoder::with_capacity(8); + enc.write_i32(42); + enc.write_empty_tagged_fields(); + enc.write_i16(7); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!(dec.read_i32().unwrap(), 42); + dec.read_tagged_fields().unwrap(); // should consume the single 0x00 byte + assert_eq!(dec.read_i16().unwrap(), 7); + assert_eq!(dec.remaining(), 0); +} diff --git a/gateways/kafka/tests/common/fixtures.rs b/gateways/kafka/tests/common/fixtures.rs new file mode 100644 index 0000000000..02eb43d369 --- /dev/null +++ b/gateways/kafka/tests/common/fixtures.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Fixture loaders — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use std::path::PathBuf; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::header::{RequestHeader, request_header_version}; + +pub fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") +} + +pub fn fixture_exists(api_key: i16, api_name: &str, version: i16) -> bool { + let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); + fixtures_dir().join(filename).is_file() +} + +/// Load request body bytes from a kafka-tool `.bin` fixture (skips frame header). +pub fn load_fixture_body(api_key: i16, api_name: &str, version: i16) -> Bytes { + let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); + let path = fixtures_dir().join(&filename); + let data = std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {filename}: {e}")); + extract_body_from_framed_message(api_key, version, &data) +} + +/// Strip the 4-byte length prefix and Kafka request header from a framed message. +pub fn extract_body_from_framed_message(api_key: i16, api_version: i16, data: &[u8]) -> Bytes { + let frame = Bytes::copy_from_slice(&data[4..]); + let hdr_ver = request_header_version(api_key, api_version); + let mut decoder = Decoder::new(frame); + RequestHeader::decode_from(&mut decoder, hdr_ver).expect("fixture request header must decode"); + decoder + .read_bytes(decoder.remaining()) + .expect("fixture request body must decode") +} diff --git a/gateways/kafka/tests/common/scope.rs b/gateways/kafka/tests/common/scope.rs new file mode 100644 index 0000000000..61a3d37f51 --- /dev/null +++ b/gateways/kafka/tests/common/scope.rs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared scope constants — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use iggy_gateway_kafka::protocol::api::BrokerAdvertise; + +/// Scoped API keys exercised by the #3421 regression suite. +pub const SCOPED_API_KEYS: &[(i16, &str, i16, i16)] = &[ + (0, "Produce", 3, 9), + (1, "Fetch", 4, 12), + (2, "ListOffsets", 1, 6), + (3, "Metadata", 0, 9), + (18, "ApiVersions", 0, 3), + (19, "CreateTopics", 2, 5), +]; + +pub fn default_broker() -> BrokerAdvertise { + BrokerAdvertise::default() +} diff --git a/gateways/kafka/tests/common/server.rs b/gateways/kafka/tests/common/server.rs new file mode 100644 index 0000000000..a009b46423 --- /dev/null +++ b/gateways/kafka/tests/common/server.rs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Test server spawn helper — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use std::net::SocketAddr; +use std::time::Duration; + +use tokio::sync::broadcast; + +use iggy_gateway_kafka::{KafkaServer, ServerConfig}; + +/// Bind an ephemeral port, start `KafkaServer`, return address + shutdown sender. +pub async fn spawn_test_server() -> (SocketAddr, broadcast::Sender<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("local addr"); + + let config = ServerConfig { + bind_addr: addr.to_string(), + advertised_host: None, + advertised_port: None, + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(5), + write_timeout: Duration::from_secs(5), + }; + let (shutdown_tx, shutdown_rx) = broadcast::channel(1); + let server = KafkaServer::new(config); + tokio::spawn(async move { + let _ = server.run(listener, shutdown_rx).await; + }); + + (addr, shutdown_tx) +} diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs new file mode 100644 index 0000000000..6eea90f237 --- /dev/null +++ b/gateways/kafka/tests/common/tcp.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! TCP round-trip helpers — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use std::net::SocketAddr; + +use bytes::{BufMut, Bytes, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::header::{request_header_version, response_header_version}; + +/// Build a complete length-prefixed Kafka request frame (header + body). +pub fn build_request_frame( + api_key: i16, + api_version: i16, + correlation_id: i32, + client_id: Option<&str>, + body: &[u8], +) -> Bytes { + let hdr_ver = request_header_version(api_key, api_version); + let mut enc = iggy_gateway_kafka::protocol::codec::Encoder::with_capacity(64 + body.len()); + enc.write_i16(api_key); + enc.write_i16(api_version); + enc.write_i32(correlation_id); + if hdr_ver >= 2 { + enc.write_compact_nullable_string(client_id); + enc.write_empty_tagged_fields(); + } else { + enc.write_nullable_string(client_id) + .expect("test client_id fits i16"); + } + enc.write_bytes(body); + + let payload = enc.freeze(); + let payload_len = i32::try_from(payload.len()).expect("test payload fits i32"); + let mut frame = BytesMut::with_capacity(4 + payload.len()); + frame.put_i32(payload_len); + frame.extend_from_slice(&payload); + frame.freeze() +} + +/// Parse correlation id and response body from a raw response payload (no length prefix). +pub fn parse_response_payload(api_key: i16, api_version: i16, payload: Bytes) -> (i32, Bytes) { + let resp_hdr_ver = response_header_version(api_key, api_version); + let mut d = Decoder::new(payload); + let correlation_id = d.read_i32().expect("correlation_id"); + if resp_hdr_ver >= 1 { + d.read_tagged_fields().expect("response tagged fields"); + } + let body = d.read_bytes(d.remaining()).expect("response body"); + (correlation_id, body) +} + +/// Read one length-prefixed response frame from the stream. +pub async fn read_response_frame(stream: &mut TcpStream, max_size: usize) -> Bytes { + let mut len_buf = [0u8; 4]; + stream + .read_exact(&mut len_buf) + .await + .expect("response length prefix"); + let frame_len_i32 = i32::from_be_bytes(len_buf); + assert!(frame_len_i32 > 0, "response frame length must be positive"); + let frame_len = usize::try_from(frame_len_i32).expect("positive i32 frame length fits usize"); + assert!( + frame_len <= max_size, + "response frame too large: {frame_len}" + ); + let mut buf = vec![0u8; frame_len]; + stream.read_exact(&mut buf).await.expect("response body"); + Bytes::from(buf) +} + +/// Send one request frame and return parsed `(correlation_id, response_body)`. +pub async fn round_trip( + addr: SocketAddr, + api_key: i16, + api_version: i16, + correlation_id: i32, + body: &[u8], +) -> (i32, Bytes) { + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let frame = build_request_frame( + api_key, + api_version, + correlation_id, + Some("regression-test"), + body, + ); + stream.write_all(&frame).await.expect("write request"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + parse_response_payload(api_key, api_version, payload) +} diff --git a/gateways/kafka/tests/decode_safety_tests.rs b/gateways/kafka/tests/decode_safety_tests.rs new file mode 100644 index 0000000000..f5d8d84a9a --- /dev/null +++ b/gateways/kafka/tests/decode_safety_tests.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Adversarial wire-input tests for #3421 — malformed lengths must return errors, never panic. + +use bytes::Bytes; + +use iggy_gateway_kafka::error::KafkaProtocolError; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder, MAX_COLLECTION_LEN}; +use iggy_gateway_kafka::protocol::requests::decode_produce_request; + +#[test] +fn compact_array_varint_zero_decodes_as_empty_without_panic() { + // Per Kafka spec, compact-array varint=0 means null/absent → 0 elements (not an error). + let mut d = Decoder::new(Bytes::from_static(&[0x00])); + assert_eq!(d.read_compact_array_count().unwrap(), 0); +} + +#[test] +fn negative_i32_array_length_returns_error_not_panic() { + let mut raw = Vec::new(); + raw.extend_from_slice(&(-1_i32).to_be_bytes()); + let mut d = Decoder::new(Bytes::from(raw)); + let err = d.read_i32_array_count().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidArrayLength(-1))); +} + +#[test] +fn i32_array_length_above_max_returns_collection_too_large() { + let mut raw = Vec::new(); + let oversized = i32::try_from(MAX_COLLECTION_LEN + 1).expect("test value fits i32"); + raw.extend_from_slice(&oversized.to_be_bytes()); + let mut d = Decoder::new(Bytes::from(raw)); + let err = d.read_i32_array_count().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::CollectionTooLarge { .. })); +} + +#[test] +fn produce_decoder_rejects_truncated_flexible_body() { + let mut body = Vec::new(); + body.push(0x00); // transactional_id null (compact) + body.extend_from_slice(&1_i16.to_be_bytes()); // acks + body.extend_from_slice(&1000_i32.to_be_bytes()); // timeout + body.push(0x02); // topics compact array: 1 element (varint = count+1) + // truncated before topic name + + let err = decode_produce_request(9, Bytes::from(body)).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn write_nullable_string_rejects_oversized_length() { + let mut enc = Encoder::with_capacity(8); + let long = "x".repeat(i16::MAX as usize + 1); + let err = enc.write_nullable_string(Some(&long)).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::StringTooLong { .. })); +} + +#[test] +fn varint_terminal_byte_with_extra_bits_at_shift_63_is_rejected() { + // Nine continuation bytes then terminal 0x7E at shift 63 (bits 1-6 set, bit 7 clear). + let mut d = Decoder::new(Bytes::from_static(&[ + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x7E, + ])); + let err = d.read_varint().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidVarint)); +} diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs new file mode 100644 index 0000000000..8a51e45b06 --- /dev/null +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -0,0 +1,433 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Validates request decoders and response encoders against the binary fixtures +//! produced by tools/kafka-tool. +//! +//! Frame layout written by kafka-tool (all versions): +//! [4-byte length prefix] +//! [`api_key` i16][`api_version` i16][`correlation_id` i32] +//! header v1: [`client_id`] `NULLABLE_STRING` +//! header v2: [`client_id`] `COMPACT_NULLABLE_STRING` + request-header tagged fields +//! [request body] ← properly encoded per spec (flexible or not) + +use std::path::PathBuf; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::header::{RequestHeader, request_header_version}; +use iggy_gateway_kafka::protocol::requests::{ + decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, + decode_produce_request, +}; +use iggy_gateway_kafka::protocol::responses::{ + encode_create_topics_response, encode_fetch_response, encode_list_offsets_response, + encode_produce_response, +}; + +// ── helpers ─────────────────────────────────────────────────────────────────── + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") +} + +/// Load a kafka-tool `.bin` file and return just the request body bytes. +fn load_body(api_key: i16, api_name: &str, version: i16) -> Bytes { + let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); + let path = fixtures_dir().join(&filename); + let data = std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {filename}: {e}")); + + let frame = Bytes::copy_from_slice(&data[4..]); + let hdr_ver = request_header_version(api_key, version); + let mut decoder = Decoder::new(frame); + RequestHeader::decode_from(&mut decoder, hdr_ver).expect("fixture request header must decode"); + decoder + .read_bytes(decoder.remaining()) + .expect("fixture request body must decode") +} + +// ── Produce (API key 0) ─────────────────────────────────────────────────────── + +#[test] +fn produce_all_supported_versions_decode() { + for version in 3i16..=9 { + let body = load_body(0, "Produce", version); + let req = decode_produce_request(version, body) + .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); + + assert_eq!(req.acks, -1, "Produce v{version}: unexpected acks"); + assert_eq!( + req.timeout_ms, 5000, + "Produce v{version}: unexpected timeout_ms" + ); + assert_eq!(req.topics.len(), 1, "Produce v{version}: expected 1 topic"); + assert_eq!( + req.topics[0].topic, "test-topic", + "Produce v{version}: wrong topic name" + ); + assert_eq!( + req.topics[0].partitions.len(), + 1, + "Produce v{version}: expected 1 partition" + ); + assert_eq!( + req.topics[0].partitions[0].partition, 0, + "Produce v{version}: wrong partition index" + ); + assert!( + req.topics[0].partitions[0].records.is_some(), + "Produce v{version}: records should be present" + ); + } +} + +#[test] +fn produce_response_encodes_for_all_supported_versions() { + for version in 3i16..=9 { + let body = load_body(0, "Produce", version); + let req = decode_produce_request(version, body) + .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); + let resp = encode_produce_response(version, &req); + assert!( + !resp.is_empty(), + "Produce v{version}: response must not be empty" + ); + } +} + +#[test] +fn produce_response_v3_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(0, "Produce", 3); + let req = decode_produce_request(3, body).unwrap(); + let resp = encode_produce_response(3, &req); + + let mut d = Decoder::new(resp); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let topic_name = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(topic_name, "test-topic"); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let partition = d.read_i32().unwrap(); + assert_eq!(partition, 0); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let base_offset = d.read_i64().unwrap(); + assert_eq!(base_offset, 0); + // log_append_time_ms (v2+) + let _log_append = d.read_i64().unwrap(); + // log_start_offset (v5+) — not present for v3 + let throttle = d.read_i32().unwrap(); + assert_eq!(throttle, 0); +} + +#[test] +fn produce_response_v8_includes_record_errors() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(0, "Produce", 8); + let req = decode_produce_request(8, body).unwrap(); + let resp = encode_produce_response(8, &req); + + let mut d = Decoder::new(resp); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let _topic_name = d.read_nullable_string().unwrap(); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let _partition = d.read_i32().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _base_offset = d.read_i64().unwrap(); + let _log_append_time = d.read_i64().unwrap(); // v2+ + let _log_start_offset = d.read_i64().unwrap(); // v5+ + let record_errors_count = d.read_i32().unwrap(); // v8+: should be 0 + assert_eq!( + record_errors_count, 0, + "v8 must emit empty record_errors array" + ); + let error_message = d.read_nullable_string().unwrap(); // v8+: should be null + assert!(error_message.is_none(), "v8 error_message must be null"); +} + +// ── Fetch (API key 1) ───────────────────────────────────────────────────────── + +#[test] +fn fetch_all_supported_versions_decode() { + for version in 4i16..=12 { + let body = load_body(1, "Fetch", version); + let req = decode_fetch_request(version, body) + .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); + + assert_eq!( + req.max_wait_ms, 500, + "Fetch v{version}: unexpected max_wait_ms" + ); + assert_eq!(req.min_bytes, 1, "Fetch v{version}: unexpected min_bytes"); + assert_eq!(req.topics.len(), 1, "Fetch v{version}: expected 1 topic"); + assert_eq!( + req.topics[0].topic, "test-topic", + "Fetch v{version}: wrong topic name" + ); + assert_eq!( + req.topics[0].partitions.len(), + 1, + "Fetch v{version}: expected 1 partition" + ); + assert_eq!( + req.topics[0].partitions[0].partition, 0, + "Fetch v{version}: wrong partition index" + ); + assert_eq!( + req.topics[0].partitions[0].fetch_offset, 0, + "Fetch v{version}: wrong fetch_offset" + ); + } +} + +#[test] +fn fetch_response_encodes_for_all_supported_versions() { + for version in 4i16..=12 { + let body = load_body(1, "Fetch", version); + let req = decode_fetch_request(version, body) + .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); + let resp = encode_fetch_response(version, &req); + assert!( + !resp.is_empty(), + "Fetch v{version}: response must not be empty" + ); + } +} + +#[test] +fn fetch_response_v7_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(1, "Fetch", 7); + let req = decode_fetch_request(7, body).unwrap(); + let resp = encode_fetch_response(7, &req); + + let mut d = Decoder::new(resp); + let throttle_ms = d.read_i32().unwrap(); // v1+ + assert_eq!(throttle_ms, 0); + let error_code = d.read_i16().unwrap(); // v7+ + assert_eq!(error_code, 0); + let session_id = d.read_i32().unwrap(); // v7+ + assert_eq!(session_id, 0); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let topic_name = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(topic_name, "test-topic"); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let partition = d.read_i32().unwrap(); + assert_eq!(partition, 0); + let partition_error = d.read_i16().unwrap(); + assert_eq!(partition_error, 0); + let high_watermark = d.read_i64().unwrap(); + assert_eq!(high_watermark, 0); +} + +// ── ListOffsets (API key 2) ─────────────────────────────────────────────────── + +#[test] +fn list_offsets_all_supported_versions_decode() { + for version in 1i16..=6 { + let body = load_body(2, "ListOffsets", version); + let req = decode_list_offsets_request(version, body) + .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); + + assert_eq!( + req.topics.len(), + 1, + "ListOffsets v{version}: expected 1 topic" + ); + assert_eq!( + req.topics[0].topic, "test-topic", + "ListOffsets v{version}: wrong topic name" + ); + assert_eq!( + req.topics[0].partitions.len(), + 1, + "ListOffsets v{version}: expected 1 partition" + ); + assert_eq!( + req.topics[0].partitions[0].partition, 0, + "ListOffsets v{version}: wrong partition index" + ); + } +} + +#[test] +fn list_offsets_response_encodes_for_all_supported_versions() { + for version in 1i16..=6 { + let body = load_body(2, "ListOffsets", version); + let req = decode_list_offsets_request(version, body) + .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); + let resp = encode_list_offsets_response(version, &req); + assert!( + !resp.is_empty(), + "ListOffsets v{version}: response must not be empty" + ); + } +} + +#[test] +fn list_offsets_response_v1_no_leader_epoch() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(2, "ListOffsets", 1); + let req = decode_list_offsets_request(1, body).unwrap(); + let resp = encode_list_offsets_response(1, &req); + + let mut d = Decoder::new(resp); + // v1: no throttle_time_ms + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let _topic_name = d.read_nullable_string().unwrap(); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let _partition = d.read_i32().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _timestamp = d.read_i64().unwrap(); // v1+ + let _offset = d.read_i64().unwrap(); + // v1 must NOT have a leader_epoch field — assert all bytes consumed + assert_eq!( + d.remaining(), + 0, + "v1 response must have no trailing bytes (leader_epoch must NOT be written)" + ); +} + +#[test] +fn list_offsets_response_v4_has_leader_epoch() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(2, "ListOffsets", 4); + let req = decode_list_offsets_request(4, body).unwrap(); + let resp = encode_list_offsets_response(4, &req); + + let mut d = Decoder::new(resp); + let _throttle = d.read_i32().unwrap(); // v2+ + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let _topic_name = d.read_nullable_string().unwrap(); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let _partition = d.read_i32().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _timestamp = d.read_i64().unwrap(); + let _offset = d.read_i64().unwrap(); + let leader_epoch = d.read_i32().unwrap(); // v4+ + assert_eq!(leader_epoch, -1, "v4 must have leader_epoch = -1"); + assert_eq!(d.remaining(), 0); +} + +// ── CreateTopics (API key 19) ───────────────────────────────────────────────── + +#[test] +fn create_topics_all_supported_versions_decode() { + for version in 2i16..=5 { + let body = load_body(19, "CreateTopics", version); + let req = decode_create_topics_request(version, body) + .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); + + assert_eq!( + req.topics.len(), + 1, + "CreateTopics v{version}: expected 1 topic" + ); + assert_eq!( + req.topics[0].num_partitions, 1, + "CreateTopics v{version}: wrong num_partitions" + ); + assert_eq!( + req.topics[0].replication_factor, 1, + "CreateTopics v{version}: wrong replication_factor" + ); + assert!( + !req.topics[0].name.is_empty(), + "CreateTopics v{version}: topic name must not be empty" + ); + assert_eq!( + req.timeout_ms, 30000, + "CreateTopics v{version}: unexpected timeout_ms" + ); + } +} + +#[test] +fn create_topics_response_encodes_for_all_supported_versions() { + for version in 2i16..=5 { + let body = load_body(19, "CreateTopics", version); + let req = decode_create_topics_request(version, body) + .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); + let resp = encode_create_topics_response(version, &req); + assert!( + !resp.is_empty(), + "CreateTopics v{version}: response must not be empty" + ); + } +} + +#[test] +fn create_topics_response_v2_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(19, "CreateTopics", 2); + let req = decode_create_topics_request(2, body).unwrap(); + let topic_name = req.topics[0].name.clone(); + let resp = encode_create_topics_response(2, &req); + + let mut d = Decoder::new(resp); + let _throttle = d.read_i32().unwrap(); // v2+ + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let resp_topic = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(resp_topic, topic_name); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let error_msg = d.read_nullable_string().unwrap(); // v1+ + assert!(error_msg.is_none()); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn create_topics_response_v5_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(19, "CreateTopics", 5); + let req = decode_create_topics_request(5, body).unwrap(); + let resp = encode_create_topics_response(5, &req); + + let mut d = Decoder::new(resp); + let _throttle = d.read_i32().unwrap(); // v2+ + let topic_count_plus_one = d.read_varint().unwrap(); // flexible compact array + assert_eq!(topic_count_plus_one, 2); // 1 topic → varint = 2 + + let _topic_name = d.read_compact_nullable_string().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _error_msg = d.read_compact_nullable_string().unwrap(); // v1+ + let num_partitions = d.read_i32().unwrap(); + assert_eq!(num_partitions, 1); + let replication_factor = d.read_i16().unwrap(); + assert_eq!(replication_factor, 1); + let configs_count_plus_one = d.read_varint().unwrap(); // empty compact array + assert_eq!(configs_count_plus_one, 1); // empty = varint(1) + d.read_tagged_fields().unwrap(); // per-entry tagged_fields + d.read_tagged_fields().unwrap(); // top-level tagged_fields + assert_eq!(d.remaining(), 0); +} diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs new file mode 100644 index 0000000000..f146215d34 --- /dev/null +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, handle_request, +}; +use iggy_gateway_kafka::protocol::codec::Encoder; + +#[test] +fn golden_apiversions_v1_response_fixture() { + let broker = BrokerAdvertise::default(); + let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker); + + // error_code=0, api_count=6 + // key 0 (Produce) min=0 max=9 (KAFKA-18659 advertise min=0) + // key 1 (Fetch) min=4 max=12 + // key 2 (ListOffsets) min=1 max=6 + // key 3 (Metadata) min=0 max=9 + // key 18 (ApiVersions) min=0 max=3 + // key 19 (CreateTopics) min=2 max=5 + // throttle_ms=0 + let expected: [u8; 46] = [ + 0x00, 0x00, // error_code + 0x00, 0x00, 0x00, 0x06, // api count = 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, // key 0: Produce 0–9 (advertised) + 0x00, 0x01, 0x00, 0x04, 0x00, 0x0C, // key 1: Fetch 4–12 + 0x00, 0x02, 0x00, 0x01, 0x00, 0x06, // key 2: ListOffsets 1–6 + 0x00, 0x03, 0x00, 0x00, 0x00, 0x09, // key 3: Metadata 0–9 + 0x00, 0x12, 0x00, 0x00, 0x00, 0x03, // key 18: ApiVersions 0–3 + 0x00, 0x13, 0x00, 0x02, 0x00, 0x05, // key 19: CreateTopics 2–5 + 0x00, 0x00, 0x00, 0x00, // throttle_ms + ]; + assert_eq!(actual.as_ref(), &expected); +} + +#[test] +fn golden_metadata_v0_single_topic_response_fixture() { + let mut request = Encoder::with_capacity(32); + request.write_i32(1); // one topic + let req_bytes = request.freeze(); + + let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()); + + // Metadata v0 layout: brokers[], topics[] (no controller_id — added in v1) + // brokers[1]: node_id=1, host=127.0.0.1, port=9093 + // topics[1]: topic_error=3, topic_name=unknown-topic, partitions[0] + let expected: [u8; 48] = [ + 0x00, 0x00, 0x00, 0x01, // broker count + 0x00, 0x00, 0x00, 0x01, // node id + 0x00, 0x09, // host len + 0x31, 0x32, 0x37, 0x2e, 0x30, 0x2e, 0x30, 0x2e, 0x31, // "127.0.0.1" + 0x00, 0x00, 0x23, 0x85, // port 9093 + 0x00, 0x00, 0x00, 0x01, // topic count + 0x00, 0x03, // topic error code + 0x00, 0x0d, // topic name len + 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x2d, 0x74, 0x6f, 0x70, 0x69, + 0x63, // unknown-topic + 0x00, 0x00, 0x00, 0x00, // partition count + ]; + assert_eq!(actual.as_ref(), &expected); +} diff --git a/gateways/kafka/tests/handler_regression_tests.rs b/gateways/kafka/tests/handler_regression_tests.rs new file mode 100644 index 0000000000..df569f4c12 --- /dev/null +++ b/gateways/kafka/tests/handler_regression_tests.rs @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Full handler regression — every scoped API key × version through `handle_request`. + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/scope.rs"] +mod scope; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, API_KEY_PRODUCE, ERROR_NONE, + handle_request, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::{fixture_exists, load_fixture_body}; +use scope::{SCOPED_API_KEYS, default_broker}; + +#[test] +fn handle_request_succeeds_for_every_supported_version_with_fixture() { + for &(api_key, name, min_ver, max_ver) in SCOPED_API_KEYS { + if api_key == 3 || api_key == 18 { + // Metadata / ApiVersions: empty body is valid + for version in min_ver..=max_ver { + let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()); + assert!( + !resp.is_empty(), + "{name} v{version} returned empty response" + ); + } + continue; + } + + for version in min_ver..=max_ver { + if !fixture_exists(api_key, name, version) { + continue; + } + let body = load_fixture_body(api_key, name, version); + let resp = handle_request(api_key, version, body, &default_broker()); + assert!( + !resp.is_empty(), + "{name} v{version} returned empty response" + ); + } + } +} + +#[test] +fn produce_stub_response_has_zero_error_per_partition() { + for version in 3i16..=9 { + if !fixture_exists(0, "Produce", version) { + continue; + } + let body = load_fixture_body(0, "Produce", version); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()); + let flexible = version >= 9; + let mut d = Decoder::new(resp); + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "Produce v{version}"); + } +} + +#[test] +fn fetch_stub_response_has_zero_partition_error() { + for version in 4i16..=12 { + if !fixture_exists(1, "Fetch", version) { + continue; + } + let body = load_fixture_body(1, "Fetch", version); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()); + let flexible = version >= 12; + let mut d = Decoder::new(resp); + if version >= 1 { + let _throttle = d.read_i32().unwrap(); + } + if version >= 7 { + assert_eq!(d.read_i16().unwrap(), ERROR_NONE); + let _session = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NONE, + "Fetch v{version} partition error" + ); + } +} + +#[test] +fn list_offsets_stub_response_has_zero_error() { + for version in 1i16..=6 { + if !fixture_exists(2, "ListOffsets", version) { + continue; + } + let body = load_fixture_body(2, "ListOffsets", version); + let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()); + let flexible = version >= 6; + let mut d = Decoder::new(resp); + if version >= 2 { + let _throttle = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "ListOffsets v{version}"); + } +} + +#[test] +fn create_topics_stub_response_has_zero_error() { + for version in 2i16..=5 { + if !fixture_exists(19, "CreateTopics", version) { + continue; + } + let body = load_fixture_body(19, "CreateTopics", version); + let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()); + let flexible = version >= 5; + let mut d = Decoder::new(resp); + if version >= 2 { + let _throttle = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + } + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "CreateTopics v{version}"); + } +} diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs new file mode 100644 index 0000000000..6ea323fdfe --- /dev/null +++ b/gateways/kafka/tests/header_tests.rs @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_gateway_kafka::protocol::codec::Encoder; +use iggy_gateway_kafka::protocol::header::{ + RequestHeader, ResponseHeader, request_header_version, response_header_version, +}; + +// ── Request header v1 (non-flexible) ─────────────────────────────────────── + +#[test] +fn request_header_v1_decodes() { + let mut enc = Encoder::with_capacity(64); + enc.write_i16(18); // api_key: ApiVersions + enc.write_i16(2); // api_version + enc.write_i32(101); + enc.write_nullable_string(Some("kafka-cli")).unwrap(); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 1).expect("decode should succeed"); + assert_eq!(header.api_key, 18); + assert_eq!(header.api_version, 2); + assert_eq!(header.correlation_id, 101); + assert_eq!(header.client_id.as_deref(), Some("kafka-cli")); +} + +#[test] +fn request_header_v1_null_client_id() { + let mut enc = Encoder::with_capacity(32); + enc.write_i16(18); + enc.write_i16(1); + enc.write_i32(5); + enc.write_nullable_string(None).unwrap(); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 1).unwrap(); + assert_eq!(header.client_id, None); +} + +// ── Request header v2 (flexible — compact client_id + tagged fields) ─────── + +#[test] +fn request_header_v2_decodes() { + let mut enc = Encoder::with_capacity(64); + enc.write_i16(18); // api_key: ApiVersions + enc.write_i16(3); // api_version (flexible threshold for ApiVersions is 3) + enc.write_i32(202); + enc.write_compact_nullable_string(Some("my-client")); + enc.write_empty_tagged_fields(); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 2).expect("flexible decode should succeed"); + assert_eq!(header.api_key, 18); + assert_eq!(header.api_version, 3); + assert_eq!(header.correlation_id, 202); + assert_eq!(header.client_id.as_deref(), Some("my-client")); +} + +#[test] +fn request_header_v2_null_client_id() { + let mut enc = Encoder::with_capacity(32); + enc.write_i16(18); + enc.write_i16(3); + enc.write_i32(303); + enc.write_compact_nullable_string(None); + enc.write_empty_tagged_fields(); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 2).unwrap(); + assert_eq!(header.client_id, None); +} + +// ── Response header encode ────────────────────────────────────────────────── + +#[test] +fn response_header_v0_encodes_correlation_id_only() { + let header = ResponseHeader { correlation_id: 77 }; + let bytes = header.encode(0); + assert_eq!(bytes.as_ref(), &[0, 0, 0, 77]); +} + +#[test] +fn response_header_v1_encodes_correlation_id_plus_tagged_fields() { + let header = ResponseHeader { correlation_id: 1 }; + let bytes = header.encode(1); + // [0,0,0,1] correlation_id + [0x00] empty tagged fields + assert_eq!(bytes.as_ref(), &[0, 0, 0, 1, 0x00]); +} + +// ── Header version lookup ─────────────────────────────────────────────────── + +#[test] +fn request_header_version_non_flexible_below_threshold() { + // ApiVersions v0-2 → header v1 + assert_eq!(request_header_version(18, 0), 1); + assert_eq!(request_header_version(18, 2), 1); + // Metadata v0-8 → header v1 + assert_eq!(request_header_version(3, 0), 1); + assert_eq!(request_header_version(3, 8), 1); +} + +#[test] +fn request_header_version_flexible_at_threshold() { + // ApiVersions v3 → header v2 + assert_eq!(request_header_version(18, 3), 2); + // Metadata v9 → header v2 + assert_eq!(request_header_version(3, 9), 2); + // ConsumerGroupHeartbeat (68) always flexible + assert_eq!(request_header_version(68, 0), 2); +} + +#[test] +fn response_header_version_apiversions_always_zero() { + // ApiVersions is a special case: response header is always v0 + assert_eq!(response_header_version(18, 0), 0); + assert_eq!(response_header_version(18, 3), 0); // even flexible request → v0 response +} + +#[test] +fn share_group_api_keys_use_flexible_header_from_v0() { + for key in [77, 78, 79, 80] { + assert_eq!( + request_header_version(key, 0), + 2, + "api_key {key} must use flexible header v2" + ); + } +} + +#[test] +fn response_header_version_flexible_non_apiversions() { + // Metadata v9+ is flexible → response header v1 + assert_eq!(response_header_version(3, 9), 1); + // Metadata v0 is non-flexible → response header v0 + assert_eq!(response_header_version(3, 0), 0); +} diff --git a/gateways/kafka/tests/metadata_regression_tests.rs b/gateways/kafka/tests/metadata_regression_tests.rs new file mode 100644 index 0000000000..1c33e38b7a --- /dev/null +++ b/gateways/kafka/tests/metadata_regression_tests.rs @@ -0,0 +1,242 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Metadata API regression — all supported versions, broker advertise, topic counts. + +#[path = "common/scope.rs"] +mod scope; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_METADATA, BrokerAdvertise, ERROR_UNKNOWN_TOPIC_OR_PARTITION, handle_request, +}; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +use scope::default_broker; + +fn metadata_request_legacy(topic_count: i32) -> Bytes { + let mut enc = Encoder::with_capacity(8); + enc.write_i32(topic_count); + enc.freeze() +} + +fn metadata_request_flexible(topic_count: usize) -> Bytes { + let mut enc = Encoder::with_capacity(8); + enc.write_varint((topic_count + 1) as u64); + enc.freeze() +} + +fn read_broker_legacy(d: &mut Decoder) -> (String, i32) { + let count = d.read_i32().unwrap(); + assert_eq!(count, 1); + let _node = d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + (host, port) +} + +fn read_broker_flexible(d: &mut Decoder) -> (String, i32) { + let count_plus_one = d.read_varint().unwrap(); + assert_eq!(count_plus_one, 2); // one broker + let _node = d.read_i32().unwrap(); + let host = d.read_compact_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + let _rack = d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + (host, port) +} + +#[test] +fn metadata_corrupt_partial_body_returns_zero_topics() { + let body = handle_request( + API_KEY_METADATA, + 0, + Bytes::from_static(&[0x00, 0x00]), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v0_empty_topics_stub_broker() { + let body = handle_request( + API_KEY_METADATA, + 0, + metadata_request_legacy(0), + &default_broker(), + ); + let mut d = Decoder::new(body); + let (host, port) = read_broker_legacy(&mut d); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 9093); + assert_eq!(d.read_i32().unwrap(), 0); +} + +#[test] +fn metadata_v0_three_topics_each_unknown() { + let body = handle_request( + API_KEY_METADATA, + 0, + metadata_request_legacy(3), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + assert_eq!(d.read_i32().unwrap(), 3); + for _ in 0..3 { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!(d.read_nullable_string().unwrap().unwrap(), "unknown-topic"); + assert_eq!(d.read_i32().unwrap(), 0); + } +} + +#[test] +fn metadata_v1_includes_controller_id() { + let body = handle_request( + API_KEY_METADATA, + 1, + metadata_request_legacy(0), + &default_broker(), + ); + let mut d = Decoder::new(body); + // Metadata v1 has no throttle_time_ms (added in v3). + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let controller = d.read_i32().unwrap(); + assert_eq!(controller, 1); +} + +#[test] +fn metadata_v2_includes_cluster_id_field() { + let body = handle_request( + API_KEY_METADATA, + 2, + metadata_request_legacy(0), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let _cluster_id = d.read_nullable_string().unwrap(); + let _controller = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); +} + +#[test] +fn metadata_all_legacy_versions_produce_valid_response() { + for version in 0i16..=8 { + let body = handle_request( + API_KEY_METADATA, + version, + metadata_request_legacy(1), + &default_broker(), + ); + let mut d = Decoder::new(body); + if version >= 3 { + let _throttle = d.read_i32().unwrap(); + } + let _ = read_broker_legacy(&mut d); + if version >= 1 { + let _rack = d.read_nullable_string().unwrap(); + } + if version >= 2 { + let _cluster = d.read_nullable_string().unwrap(); + } + if version >= 1 { + let _controller = d.read_i32().unwrap(); + } + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + } +} + +#[test] +fn metadata_v9_flexible_encoding() { + let body = handle_request( + API_KEY_METADATA, + 9, + metadata_request_flexible(2), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _throttle = d.read_i32().unwrap(); + let (host, port) = read_broker_flexible(&mut d); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 9093); + let _cluster = d.read_compact_nullable_string().unwrap(); + let controller = d.read_i32().unwrap(); + assert_eq!(controller, 1); + + let topics_plus_one = d.read_varint().unwrap(); + assert_eq!(topics_plus_one, 3); // 2 topics + for _ in 0..2 { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!( + d.read_compact_nullable_string().unwrap().unwrap(), + "unknown-topic" + ); + let _internal = d.read_bool().unwrap(); + let parts_plus_one = d.read_varint().unwrap(); + assert_eq!(parts_plus_one, 1); // empty partitions + assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations (v8+) + d.read_tagged_fields().unwrap(); + } + assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations (v8+) + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v8_includes_authorized_operations_legacy() { + let body = handle_request( + API_KEY_METADATA, + 8, + metadata_request_legacy(1), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _throttle = d.read_i32().unwrap(); + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let _cluster = d.read_nullable_string().unwrap(); + let _controller = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + let _topic_error = d.read_i16().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _internal = d.read_bool().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); // empty partitions + assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations + assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_uses_custom_broker_advertise() { + let broker = BrokerAdvertise { + host: "10.0.0.42".to_string(), + port: 29093, + }; + let body = handle_request(API_KEY_METADATA, 0, metadata_request_legacy(0), &broker); + let mut d = Decoder::new(body); + let (host, port) = read_broker_legacy(&mut d); + assert_eq!(host, "10.0.0.42"); + assert_eq!(port, 29093); +} diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs new file mode 100644 index 0000000000..d0846ba49f --- /dev/null +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end TCP tests through `KafkaServer` (full request/response cycle). + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/server.rs"] +mod server; +#[path = "common/tcp.rs"] +mod tcp; + +use bytes::{BufMut, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_METADATA, API_KEY_PRODUCE, ERROR_UNSUPPORTED_VERSION, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::load_fixture_body; +use server::spawn_test_server; +use tcp::{build_request_frame, parse_response_payload, read_response_frame, round_trip}; + +#[tokio::test] +async fn e2e_apiversions_v1_preserves_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 1, 42_001, &[]).await; + assert_eq!(corr, 42_001); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); +} + +#[tokio::test] +async fn e2e_apiversions_v3_flexible_preserves_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 3, 42_002, &[]).await; + assert_eq!(corr, 42_002); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); + assert_eq!(count, 6); +} + +#[tokio::test] +async fn e2e_metadata_v0_returns_stub_broker() { + let (addr, _shutdown) = spawn_test_server().await; + let mut req = BytesMut::new(); + req.put_i32(0); // empty topics + let (corr, body) = round_trip(addr, API_KEY_METADATA, 0, 77, &req).await; + assert_eq!(corr, 77); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(host, "127.0.0.1"); +} + +#[tokio::test] +async fn e2e_produce_v3_round_trip_with_fixture() { + let (addr, _shutdown) = spawn_test_server().await; + let body = load_fixture_body(0, "Produce", 3); + let (corr, resp_body) = round_trip(addr, API_KEY_PRODUCE, 3, 88, &body).await; + assert_eq!(corr, 88); + assert!(!resp_body.is_empty()); +} + +#[tokio::test] +async fn e2e_unsupported_api_key_returns_error_without_disconnect() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + + let frame1 = build_request_frame(8, 2, 99, Some("e2e-test"), &[]); + stream.write_all(&frame1).await.unwrap(); + let payload1 = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr, body) = parse_response_payload(8, 2, payload1); + assert_eq!(corr, 99); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + + // Second request on same connection must still work. + let frame2 = build_request_frame(API_KEY_API_VERSIONS, 1, 100, Some("e2e-test"), &[]); + stream.write_all(&frame2).await.unwrap(); + let payload2 = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr2, body2) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload2); + assert_eq!(corr2, 100); + let mut d2 = Decoder::new(body2); + assert_eq!(d2.read_i16().unwrap(), 0); +} + +#[tokio::test] +async fn e2e_sequential_requests_on_one_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + + let requests = [(API_KEY_API_VERSIONS, 1i16), (API_KEY_METADATA, 0i16)]; + for (i, (key, ver)) in requests.iter().enumerate() { + let meta_body = { + let mut b = BytesMut::new(); + b.put_i32(0); + b + }; + let body: &[u8] = if *key == API_KEY_METADATA { + &meta_body + } else { + &[] + }; + let correlation_id = 1000 + i32::try_from(i).expect("test index fits i32"); + let frame = build_request_frame(*key, *ver, correlation_id, Some("seq-test"), body); + stream.write_all(&frame).await.unwrap(); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr, _) = parse_response_payload(*key, *ver, payload); + assert_eq!(corr, correlation_id); + } +} + +#[tokio::test] +async fn e2e_negative_frame_length_closes_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream.write_all(&(-1i32).to_be_bytes()).await.unwrap(); + + let mut buf = [0u8; 1]; + let n = stream.read(&mut buf).await.unwrap_or(0); + assert_eq!(n, 0, "server should close after invalid frame length"); +} + +#[tokio::test] +async fn e2e_oversized_frame_is_rejected() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + + let mut frame = BytesMut::new(); + frame.put_i32(10_000_000); // exceeds default 8 MiB cap + frame.resize(4 + 100, 0); + stream.write_all(&frame).await.unwrap(); + + let mut buf = [0u8; 1]; + let n = stream.read(&mut buf).await.unwrap_or(0); + assert_eq!(n, 0, "server should close after oversized frame"); +} diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs new file mode 100644 index 0000000000..a8d6d921bc --- /dev/null +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::time::Duration; + +use bytes::{Buf, BufMut, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +use iggy_gateway_kafka::protocol::codec::Encoder; +use iggy_gateway_kafka::server::read_frame; + +async fn tcp_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let client = tokio::spawn(async move { TcpStream::connect(addr).await.unwrap() }); + let (server, _) = listener.accept().await.unwrap(); + let client = client.await.unwrap(); + (client, server) +} + +/// Raw length-prefixed write (no Kafka response header) — mirrors `server::write_frame`. +async fn write_length_prefixed( + stream: &mut TcpStream, + payload: &[u8], + write_timeout: Duration, +) -> Result<(), Box> { + let len = payload.len(); + assert!(i32::try_from(len).is_ok()); + let mut frame = BytesMut::with_capacity(4 + len); + frame.put_i32(i32::try_from(len).expect("len fits i32")); + frame.extend_from_slice(payload); + tokio::time::timeout(write_timeout, stream.write_all(&frame)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timeout"))??; + Ok(()) +} + +#[tokio::test] +async fn read_frame_reads_valid_payload() { + let (mut client, mut server) = tcp_pair().await; + + let mut enc = Encoder::with_capacity(64); + enc.write_i16(18); + enc.write_i16(3); + enc.write_i32(123); + enc.write_nullable_string(Some("test-client")).unwrap(); + let payload = enc.freeze(); + + let mut frame = BytesMut::with_capacity(4 + payload.len()); + frame.extend_from_slice( + &i32::try_from(payload.len()) + .expect("test payload fits i32") + .to_be_bytes(), + ); + frame.extend_from_slice(&payload); + client.write_all(&frame).await.unwrap(); + + let parsed = read_frame(&mut server, 4096, Duration::from_secs(1)) + .await + .unwrap(); + assert_eq!(parsed, payload); +} + +#[tokio::test] +async fn write_frame_writes_length_prefixed_payload() { + let (mut client, mut server) = tcp_pair().await; + let payload = b"abc123"; + write_length_prefixed(&mut server, payload, Duration::from_secs(1)) + .await + .unwrap(); + + let mut len = [0u8; 4]; + client.read_exact(&mut len).await.unwrap(); + let len = usize::try_from(i32::from_be_bytes(len)).expect("positive frame length"); + assert_eq!(len, payload.len()); + + let mut body = vec![0u8; len]; + client.read_exact(&mut body).await.unwrap(); + assert_eq!(body, payload); +} + +#[tokio::test] +async fn read_frame_rejects_invalid_lengths() { + let (mut client, mut server) = tcp_pair().await; + + client.write_all(&0i32.to_be_bytes()).await.unwrap(); + let err = read_frame(&mut server, 128, Duration::from_secs(1)) + .await + .expect_err("zero frame must fail"); + assert!(err.to_string().contains("invalid frame length")); + + // Ensure connection can still be reused for a second scenario by writing a valid new prefix+payload. + let mut frame = BytesMut::new(); + frame.extend_from_slice(&(200i32).to_be_bytes()); + frame.resize(4 + 200, 0); + client.write_all(&frame).await.unwrap(); + let err = read_frame(&mut server, 64, Duration::from_secs(1)) + .await + .expect_err("large frame must fail"); + assert!(err.to_string().contains("exceeds max frame size")); +} + +#[tokio::test] +async fn write_frame_length_prefix_is_big_endian() { + let (mut client, mut server) = tcp_pair().await; + write_length_prefixed(&mut server, &[1, 2, 3, 4], Duration::from_secs(1)) + .await + .unwrap(); + + let mut len_and_data = [0u8; 8]; + client.read_exact(&mut len_and_data).await.unwrap(); + let mut buf = &len_and_data[..]; + let len = buf.get_i32(); + assert_eq!(len, 4); + assert_eq!(&len_and_data[4..], &[1, 2, 3, 4]); +} + +#[tokio::test] +async fn read_frame_does_not_consume_pipelined_frame_bytes() { + let (mut client, mut server) = tcp_pair().await; + + let payload1 = b"first-request-body-data"; + let payload2 = b"second-pipelined-request"; + + // Write both frames in a single syscall so the OS delivers them together. + // With the old read_buf approach, allocator rounding causes the first read_frame + // call to consume bytes from payload2, which truncate() then silently discards. + let mut both = BytesMut::with_capacity(4 + payload1.len() + 4 + payload2.len()); + both.put_i32(i32::try_from(payload1.len()).unwrap()); + both.extend_from_slice(payload1); + both.put_i32(i32::try_from(payload2.len()).unwrap()); + both.extend_from_slice(payload2); + client.write_all(&both).await.unwrap(); + + let timeout = Duration::from_secs(1); + let frame1 = read_frame(&mut server, 4096, timeout).await.unwrap(); + let frame2 = read_frame(&mut server, 4096, timeout).await.unwrap(); + + assert_eq!(&frame1[..], payload1); + assert_eq!(&frame2[..], payload2); +} diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs new file mode 100644 index 0000000000..77c4e768f8 --- /dev/null +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -0,0 +1,323 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Version negotiation firewall — boundary tests for every scoped API key. + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/scope.rs"] +mod scope; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, + API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, ERROR_UNSUPPORTED_VERSION, + advertised_min_version, handle_request, is_supported_version, supported_api_ranges, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::load_fixture_body; +use scope::{SCOPED_API_KEYS, default_broker}; + +#[test] +fn supported_ranges_table_has_six_entries() { + assert_eq!(supported_api_ranges().len(), 6); +} + +#[test] +fn is_supported_version_matches_scope_table() { + for &(api_key, _, min_ver, max_ver) in SCOPED_API_KEYS { + assert!( + !is_supported_version(api_key, min_ver - 1), + "key {api_key} must reject v{}", + min_ver - 1 + ); + assert!( + is_supported_version(api_key, min_ver), + "key {api_key} must accept min v{min_ver}" + ); + assert!( + is_supported_version(api_key, max_ver), + "key {api_key} must accept max v{max_ver}" + ); + assert!( + !is_supported_version(api_key, max_ver + 1), + "key {api_key} must reject v{}", + max_ver + 1 + ); + } +} + +#[test] +fn apiversions_advertises_exact_supported_ranges_v1() { + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_i32().unwrap()).expect("api count fits usize"); + assert_eq!(count, supported_api_ranges().len()); + + for expected in supported_api_ranges() { + let key = d.read_i16().unwrap(); + let min = d.read_i16().unwrap(); + let max = d.read_i16().unwrap(); + assert_eq!(key, expected.api_key); + assert_eq!( + min, + advertised_min_version(expected.api_key, expected.min_version) + ); + assert_eq!(max, expected.max_version); + } + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.remaining(), 0); +} + +#[test] +fn apiversions_advertises_exact_supported_ranges_v3_flexible() { + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); + assert_eq!(count, supported_api_ranges().len()); + + for expected in supported_api_ranges() { + let key = d.read_i16().unwrap(); + let min = d.read_i16().unwrap(); + let max = d.read_i16().unwrap(); + d.read_tagged_fields().unwrap(); + assert_eq!(key, expected.api_key); + assert_eq!( + min, + advertised_min_version(expected.api_key, expected.min_version) + ); + assert_eq!(max, expected.max_version); + } + assert_eq!(d.read_i32().unwrap(), 0); + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn apiversions_advertises_produce_min_zero_while_firewall_stays_three() { + let range = supported_api_ranges() + .iter() + .find(|r| r.api_key == API_KEY_PRODUCE) + .expect("produce range"); + assert_eq!(range.min_version, 3); + assert_eq!( + advertised_min_version(API_KEY_PRODUCE, range.min_version), + 0 + ); + assert!(!is_supported_version(API_KEY_PRODUCE, 0)); +} + +#[test] +fn apiversions_all_versions_return_success() { + for version in 0i16..=3 { + let body = handle_request( + API_KEY_API_VERSIONS, + version, + Bytes::new(), + &default_broker(), + ); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0, "ApiVersions v{version}"); + } +} + +#[test] +fn apiversions_out_of_range_returns_unsupported_in_body() { + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +fn metadata_request_one_topic() -> Bytes { + let mut raw = Vec::new(); + raw.extend_from_slice(&1_i32.to_be_bytes()); + Bytes::from(raw) +} + +#[test] +fn metadata_below_min_version_returns_topic_error() { + let body = handle_request( + API_KEY_METADATA, + -1, + metadata_request_one_topic(), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _brokers = d.read_i32().unwrap(); + let _ = d.read_i32().unwrap(); + let _ = d.read_nullable_string().unwrap(); + let _ = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); // mirrors request topic count + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn metadata_above_max_version_returns_topic_error() { + // v10 uses flexible encoding; compact array varint(2) = 1 topic. + let body = handle_request( + API_KEY_METADATA, + 10, + Bytes::from_static(&[0x02]), + &default_broker(), + ); + // Response is in v9 flexible format (highest supported). + let mut d = Decoder::new(body); + d.read_i32().unwrap(); // throttle_time_ms (v3+) + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); // cluster_id + d.read_i32().unwrap(); // controller_id + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!(topic_count, 1); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn produce_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + let _ = d.read_i64().unwrap(); + let _ = d.read_i64().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn fetch_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i64().unwrap(), 0); + assert_eq!(d.read_nullable_bytes().unwrap(), None); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn fetch_unsupported_version_above_max_uses_top_level_error() { + let body = handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_varint().unwrap(), 1); + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn list_offsets_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i64().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn create_topics_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_nullable_string().unwrap(), None); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn unsupported_api_keys_return_error_only() { + for key in [8, 9, 10, 11, 17, 20, 42, 999] { + let body = handle_request(key, 0, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "api_key {key}" + ); + } +} + +#[test] +fn supported_produce_versions_accept_valid_fixture() { + for version in 3i16..=9 { + let body = load_fixture_body(0, "Produce", version); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()); + assert!(!resp.is_empty(), "Produce v{version} response empty"); + } +} + +#[test] +fn supported_fetch_versions_accept_valid_fixture() { + for version in 4i16..=12 { + let body = load_fixture_body(1, "Fetch", version); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()); + assert!(!resp.is_empty(), "Fetch v{version} response empty"); + } +} + +#[test] +fn corrupt_produce_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +#[test] +fn corrupt_fetch_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_FETCH, 4, body, &default_broker()); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} diff --git a/gateways/kafka/tools/kafka-tool/Cargo.toml b/gateways/kafka/tools/kafka-tool/Cargo.toml new file mode 100644 index 0000000000..41473e434a --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/Cargo.toml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "kafka-message-gen" +version = "0.1.0" +edition = "2024" +description = "Generates binary Kafka protocol messages for testing the Iggy Kafka gateway" +license = "Apache-2.0" +repository = "https://github.com/apache/iggy" +keywords = ["kafka", "protocol", "testing", "iggy", "wire-format"] +publish = false + +[[bin]] +name = "kafka-message-gen" +path = "src/main.rs" + +[dependencies] +anyhow = { workspace = true } +bytes = { workspace = true } +clap = { workspace = true } +hex = "0.4" +indexmap = "2" +kafka-protocol = "0.17" +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/gateways/kafka/tools/kafka-tool/README.md b/gateways/kafka/tools/kafka-tool/README.md new file mode 100644 index 0000000000..eb44ee3759 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/README.md @@ -0,0 +1,251 @@ +# kafka-message-gen + +A Rust CLI tool that generates correct, fully-framed Kafka binary wire protocol messages for **every API key** and **every supported version** — built for testing Kafka-compatible server implementations such as [Apache Iggy](https://iggy.apache.org)'s Kafka compatibility listener. + +## What It Does + +Each output `.bin` file is a complete, TCP-ready Kafka request: + +```text +[total_length: i32][api_key: i16][api_version: i16] +[correlation_id: i32][client_id: NULLABLE_STRING] +[tagged_fields: 0x00] ← only for flexible versions +[payload: bytes] ← API-specific encoded body +``` + +The tool covers **65 API keys** and **~280 versioned messages**, sourced directly from the official [Apache Kafka JSON schema files](https://github.com/apache/kafka/tree/trunk/clients/src/main/resources/common/message) (Kafka 4.1.0). + +--- + +## Why This Tool Exists + +When implementing a Kafka protocol compatibility layer (e.g. inside Apache Iggy), you need to: + +1. Verify your server correctly **parses** every API key at every version +2. Verify your server sends **valid responses** back (correct correlation ID, error codes) +3. Do this without spinning up a full Kafka client or running JVM-based tests + +This tool solves all three: generate the binary messages once, then `cat` them directly to your server's port 9092 and inspect the response. + +--- + +## Dependency on `kafka-protocol` + +This tool is built on the [`kafka-protocol`](https://crates.io/crates/kafka-protocol) Rust crate, which is itself **code-generated from Kafka's official JSON schema files**. This ensures byte-perfect correctness — the same schemas that generate Kafka's own Java serialization code generate the Rust structs used here. + +--- + +## Installation + +```bash +cd gateways/kafka/tools/kafka-tool +cargo build --release +# Binary is at: ./target/release/kafka-message-gen +``` + +**Requirements:** Rust 1.75+ (MSRV follows `kafka-protocol` crate) + +--- + +## Usage + +### List all API keys and version ranges + +```bash +cargo run -- list +``` + +Output: + +```text +Key Name MinVer MaxVer Count +────────────────────────────────────────────────────────────────────────────── +0 Produce 3 13 11 +1 Fetch 4 18 15 +2 ListOffsets 1 11 11 +3 Metadata 0 13 14 +8 OffsetCommit 2 10 9 +... +────────────────────────────────────────────────────────────────────────────── +Total: 65 API keys | ~280 versioned messages +``` + +--- + +### Generate all binary messages + +```bash +cargo run -- generate --output ./kafka_messages/ +``` + +Creates one `.bin` file per API key × version: + +```text +kafka_messages/ + 000_Produce_v3.bin + 000_Produce_v4.bin + ... + 000_Produce_v13.bin + 001_Fetch_v4.bin + ... + 003_Metadata_v0.bin + 003_Metadata_v13.bin + 018_ApiVersions_v0.bin + 018_ApiVersions_v3.bin + ... +``` + +#### Options + +| Flag | Description | Default | +| ------ | ------------- | --------- | +| `--output` | Output directory | `kafka_messages/` | +| `--api-key N` | Generate only for API key N | all | +| `--version N` | Generate only for version N | all | +| `--hex` | Print hex dump to stdout | off | + +```bash +# Generate only Metadata messages +cargo run -- generate --api-key 3 + +# Generate only ApiVersions v3 with hex dump +cargo run -- generate --api-key 18 --version 3 --hex +``` + +--- + +### Send messages to a live server + +```bash +# Start Iggy with Kafka compat listener on port 9092, then: +cargo run -- send --host 127.0.0.1:9092 +``` + +Output (one line per API key × version): + +```text +✓ ApiVersions v3 → 32 bytes ec=0 +✓ Metadata v12 → 148 bytes ec=0 +⚠ Produce v9 → 24 bytes ec=3 ← ec=3 = UnknownTopicOrPartition (expected) +✓ Fetch v12 → 36 bytes ec=0 +... +Result: 243 OK 37 failed +``` + +#### Options + +| Flag | Description | Default | +| ------ | ------------- | --------- | +| `--host` | Server address | `127.0.0.1:9092` | +| `--api-key N` | Test only API key N | all | +| `--version N` | Test only version N | all | +| `--timeout-ms N` | Per-request timeout | `5000` | + +--- + +### Verify compatibility (CI-friendly) + +```bash +cargo run -- verify --host 127.0.0.1:9092 +``` + +Exits with code **0** if all messages get a response, **1** if any fail (timeout or IO error). Useful in CI pipelines testing a Kafka-compatible server implementation. + +--- + +### Quick raw test with netcat + +No Rust needed for a quick smoke test: + +```bash +# Generate first +cargo run -- generate + +# Send ApiVersions v3 directly via netcat and inspect response +cat kafka_messages/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd | head + +# Send and decode with Wireshark (capture on loopback, filter: kafka) +``` + +--- + +## Supported API Keys (Kafka 4.1.0) + +| Key | Name | Versions | Phase 1 Priority | +| ----- | ------ | ---------- | ----------------- | +| 0 | Produce | v3–v13 | ✅ Critical | +| 1 | Fetch | v4–v18 | ✅ Critical | +| 2 | ListOffsets | v1–v11 | ✅ Critical | +| 3 | Metadata | v0–v13 | ✅ Critical | +| 8 | OffsetCommit | v2–v10 | ✅ Critical | +| 9 | OffsetFetch | v1–v10 | ✅ Critical | +| 10 | FindCoordinator | v0–v6 | ✅ Critical | +| 11 | JoinGroup | v0–v9 | ✅ Critical | +| 12 | Heartbeat | v0–v4 | ✅ Critical | +| 13 | LeaveGroup | v0–v5 | ✅ Critical | +| 14 | SyncGroup | v0–v5 | ✅ Critical | +| 15 | DescribeGroups | v0–v6 | 🟡 Important | +| 16 | ListGroups | v0–v5 | 🟡 Important | +| 17 | SaslHandshake | v0–v1 | 🟡 Important | +| 18 | ApiVersions | v0–v5 | ✅ Critical | +| 19 | CreateTopics | v2–v7 | ✅ Critical | +| 20 | DeleteTopics | v1–v6 | 🟡 Important | +| 21 | DeleteRecords | v0–v2 | 🔵 Phase 2 | +| 22 | InitProducerId | v0–v6 | 🔵 Phase 2 | +| 24 | AddPartitionsToTxn | v0–v5 | 🔵 Phase 2 | +| 25 | AddOffsetsToTxn | v0–v4 | 🔵 Phase 2 | +| 26 | EndTxn | v0–v5 | 🔵 Phase 2 | +| 28 | TxnOffsetCommit | v0–v5 | 🔵 Phase 2 | +| 29–31 | ACL APIs | v1–v3 | 🔵 Phase 2 | +| 32 | DescribeConfigs | v1–v4 | 🟡 Important | +| 36 | SaslAuthenticate | v0–v2 | 🟡 Important | +| ... | 40+ more | various | 🔵 Phase 3 | + +--- + +## Project Structure + +```text +tools/kafka-tool/ +├── Cargo.toml ← package manifest and dependencies +├── src/ +│ └── main.rs ← complete CLI implementation +└── README.md ← this file +``` + +--- + +## How It Works + +### Protocol Source + +All API schemas come from the official Kafka repository: +`apache/kafka/trunk/clients/src/main/resources/common/message/*.json` + +The `kafka-protocol` crate processes these JSON files and generates Rust structs with `encode()` and `decode()` methods. This guarantees byte-level compatibility with what official Kafka clients send. + +### Flexible vs Legacy Encoding + +Kafka introduced "flexible" encoding (compact ULEB128 strings/arrays) starting at different versions per API. The tool automatically detects whether a version uses flexible or legacy encoding and sets the request header format accordingly (header v1 for legacy, header v2 for flexible with tagged fields section). + +### API Key Coverage + +- **Explicit builders (23 API keys):** Produce, Fetch, ListOffsets, Metadata, OffsetCommit, OffsetFetch, FindCoordinator, JoinGroup, Heartbeat, LeaveGroup, SyncGroup, DescribeGroups, ListGroups, SaslHandshake, ApiVersions, CreateTopics, DeleteTopics, DeleteRecords, InitProducerId, AddPartitionsToTxn, AddOffsetsToTxn, EndTxn, TxnOffsetCommit, DescribeConfigs, SaslAuthenticate +- **Header-framing test (42 API keys):** All remaining API keys are framed correctly with an empty payload — useful for testing that your server returns a proper error response rather than crashing + +--- + +## Contributing + +The tool is intentionally simple. To add an explicit builder for a new API key: + +1. Find the JSON schema in `apache/kafka/.../message/YourRequest.json` +2. Add a new match arm in `build_payload()` in `src/main.rs` +3. Use the kafka-protocol crate's generated struct (e.g. `YourRequest::default().with_field(value)`) +4. Open a PR + +--- + +## License + +Apache License 2.0 — same as Apache Kafka and Apache Iggy. diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs new file mode 100644 index 0000000000..b92bf729c1 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -0,0 +1,894 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::{Context, Result}; +use bytes::{BufMut, Bytes, BytesMut}; +use clap::{Parser, Subcommand}; +use kafka_protocol::messages::*; +use kafka_protocol::protocol::{Encodable, StrBytes}; +use std::path::PathBuf; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; +use tracing::{info, warn}; + +mod response; + +#[derive(Parser)] +#[command( + name = "kafka-message-gen", + about = "Generate Kafka wire protocol binary messages for all API keys and versions", + long_about = "Generates correctly-framed Kafka protocol requests from Kafka 4.1.0 schemas.\n\ +Each output .bin file is TCP-ready: [len:i32][api_key:i16][api_version:i16][correlation_id:i32][client_id][payload]" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// List all supported API keys with name and version range + List, + /// Generate binary .bin files for all API keys and versions + Generate { + #[arg(short, long, default_value = "kafka_messages")] + output: PathBuf, + /// Filter to a single API key integer + #[arg(long)] + api_key: Option, + /// Filter to a single version + #[arg(long)] + version: Option, + /// Print hex dump to stdout + #[arg(long)] + hex: bool, + }, + /// Send messages to a live Kafka-compatible server and show responses + Send { + #[arg(long, default_value = "127.0.0.1:9092")] + host: String, + #[arg(long)] + api_key: Option, + #[arg(long)] + version: Option, + #[arg(long, default_value = "5000")] + timeout_ms: u64, + /// Compact one-line output (default is verbose decoded response) + #[arg(long)] + quiet: bool, + }, + Verify { + #[arg(long, default_value = "127.0.0.1:9092")] + host: String, + /// Limit to these API keys (repeatable). Defaults to all gateway-scoped keys. + #[arg(long, action = clap::ArgAction::Append)] + api_key: Vec, + /// Limit to a single protocol version + #[arg(long)] + version: Option, + #[arg(long, default_value = "5000")] + timeout_ms: u64, + /// Stop on the first failure + #[arg(long)] + fail_fast: bool, + /// Use the full Kafka 4.1 registry (for real brokers), not the Iggy gateway scope + #[arg(long)] + all_apis: bool, + /// Compact one-line output (default is verbose decoded response) + #[arg(long)] + quiet: bool, + }, +} + +// ── API Registry ───────────────────────────────────────────────────────────── +// Source: validVersions in apache/kafka trunk JSON schema files, Kafka 4.1.0 +// Format: (api_key, name, min_version, max_version) +const API_REGISTRY: &[(i16, &str, i16, i16)] = &[ + (0, "Produce", 3, 13), + (1, "Fetch", 4, 18), + (2, "ListOffsets", 1, 11), + (3, "Metadata", 0, 13), + (8, "OffsetCommit", 2, 10), + (9, "OffsetFetch", 1, 10), + (10, "FindCoordinator", 0, 6), + (11, "JoinGroup", 0, 9), + (12, "Heartbeat", 0, 4), + (13, "LeaveGroup", 0, 5), + (14, "SyncGroup", 0, 5), + (15, "DescribeGroups", 0, 6), + (16, "ListGroups", 0, 5), + (17, "SaslHandshake", 0, 1), + (18, "ApiVersions", 0, 5), + (19, "CreateTopics", 2, 7), + (20, "DeleteTopics", 1, 6), + (21, "DeleteRecords", 0, 2), + (22, "InitProducerId", 0, 6), + (23, "OffsetForLeaderEpoch", 2, 4), + (24, "AddPartitionsToTxn", 0, 5), + (25, "AddOffsetsToTxn", 0, 4), + (26, "EndTxn", 0, 5), + (27, "WriteTxnMarkers", 1, 2), + (28, "TxnOffsetCommit", 0, 5), + (29, "DescribeAcls", 1, 3), + (30, "CreateAcls", 1, 3), + (31, "DeleteAcls", 1, 3), + (32, "DescribeConfigs", 1, 4), + (33, "AlterConfigs", 0, 2), + (34, "AlterReplicaLogDirs", 1, 2), + (35, "DescribeLogDirs", 1, 5), + (36, "SaslAuthenticate", 0, 2), + (37, "CreatePartitions", 0, 3), + (38, "CreateDelegationToken", 1, 3), + (39, "RenewDelegationToken", 1, 2), + (40, "ExpireDelegationToken", 1, 2), + (41, "DescribeDelegationToken", 1, 3), + (42, "DeleteGroups", 0, 2), + (43, "ElectLeaders", 0, 2), + (44, "IncrementalAlterConfigs", 0, 1), + (45, "AlterPartitionReassignments", 0, 1), + (46, "ListPartitionReassignments", 0, 1), + (47, "OffsetDelete", 0, 0), + (48, "DescribeClientQuotas", 0, 1), + (49, "AlterClientQuotas", 0, 1), + (50, "DescribeUserScramCredentials", 0, 0), + (51, "AlterUserScramCredentials", 0, 0), + (55, "DescribeQuorum", 2, 3), + (56, "AlterPartition", 2, 3), + (57, "UpdateFeatures", 0, 2), + (60, "DescribeCluster", 0, 2), + (61, "DescribeProducers", 0, 0), + (64, "UnregisterBroker", 0, 0), + (65, "DescribeTransactions", 0, 0), + (66, "ListTransactions", 0, 1), + (67, "AllocateProducerIds", 0, 0), + (68, "ConsumerGroupHeartbeat", 0, 1), + (69, "ConsumerGroupDescribe", 0, 1), + (71, "GetTelemetrySubscriptions", 0, 0), + (72, "PushTelemetry", 0, 0), + (74, "AssignReplicasToDirs", 0, 0), + (75, "DescribeTopicPartitions", 0, 0), + (76, "ListClientMetricsResources", 0, 0), +]; + +/// Iggy Kafka gateway #3421 scope — mirrors `SUPPORTED_RANGES` in `iggy_gateway_kafka`. +const GATEWAY_REGISTRY: &[(i16, &str, i16, i16)] = &[ + (0, "Produce", 3, 9), + (1, "Fetch", 4, 12), + (2, "ListOffsets", 1, 6), + (3, "Metadata", 0, 9), + (18, "ApiVersions", 0, 3), + (19, "CreateTopics", 2, 5), +]; + +// ── Flexible version table ──────────────────────────────────────────────────── +// Source: flexibleVersions field in each Kafka JSON schema. +// Returns the first version using compact encoding, or None if never flexible. +fn first_flexible_version(api_key: i16) -> Option { + match api_key { + 0 => Some(9), + 1 => Some(12), + 2 => Some(6), + 3 => Some(9), + 8 => Some(8), + 9 => Some(6), + 10 => Some(3), + 11 => Some(6), + 12 => Some(4), + 13 => Some(4), + 14 => Some(4), + 15 => Some(5), + 16 => Some(3), + 17 => None, + 18 => Some(3), + 19 => Some(5), + 20 => Some(4), + 21 => Some(2), + 22 => Some(2), + 23 => Some(4), + 24 => Some(3), + 25 => Some(3), + 26 => Some(3), + 27 => Some(1), + 28 => Some(3), + 29 => Some(2), + 30 => Some(2), + 31 => Some(2), + 32 => Some(4), + 33 => Some(2), + 34 => Some(2), + 35 => Some(2), + 36 => Some(2), + 37 => Some(2), + 38 => Some(2), + 39 => Some(2), + 40 => Some(2), + 41 => Some(2), + 42 => Some(2), + 43 => Some(2), + 44 => Some(1), + 45 => Some(1), + 46 => Some(1), + 47 => Some(0), + 48 => Some(1), + 49 => Some(1), + 50 => Some(0), + 51 => Some(0), + 55 => Some(2), + 56 => Some(2), + 57 => Some(1), + 60 => Some(0), + 61 => Some(0), + 64 => Some(0), + 65 => Some(0), + 66 => Some(0), + 67 => Some(0), + 68 => Some(0), + 69 => Some(0), + 71 => Some(0), + 72 => Some(0), + 74 => Some(0), + 75 => Some(0), + 76 => Some(0), + _ => None, + } +} + +// ── Request framing ─────────────────────────────────────────────────────────── +// Wire format (Kafka protocol spec): +// [total_length: i32] big-endian, excludes self +// [api_key: i16] +// [api_version: i16] +// [correlation_id: i32] +// header v1: [client_id: NULLABLE_STRING] +// header v2: [client_id: COMPACT_NULLABLE_STRING] [request_header_tagged_fields] +// [payload: bytes] + +fn write_unsigned_varint(buf: &mut BytesMut, mut value: u64) { + loop { + let mut byte = (value & 0x7F) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + buf.put_u8(byte); + if value == 0 { + break; + } + } +} + +fn write_compact_nullable_string(buf: &mut BytesMut, value: Option<&str>) { + match value { + None => write_unsigned_varint(buf, 0), + Some(s) => { + write_unsigned_varint(buf, (s.len() + 1) as u64); + buf.put_slice(s.as_bytes()); + } + } +} + +fn frame_request( + api_key: i16, + api_version: i16, + correlation_id: i32, + client_id: &str, + payload: &[u8], + flexible: bool, +) -> Bytes { + let mut header = BytesMut::new(); + header.put_i16(api_key); + header.put_i16(api_version); + header.put_i32(correlation_id); + if flexible { + write_compact_nullable_string(&mut header, Some(client_id)); + header.put_u8(0); // empty request-header tagged fields + } else { + header.put_i16(i16::try_from(client_id.len()).expect("client_id fits i16")); + header.put_slice(client_id.as_bytes()); + } + + let blen = header.len() + payload.len(); + let mut buf = BytesMut::with_capacity(4 + blen); + buf.put_i32(i32::try_from(blen).expect("frame fits i32")); + buf.put_slice(&header); + buf.put_slice(payload); + buf.freeze() +} + +// ── Payload builders ────────────────────────────────────────────────────────── +// Build the API-specific encoded body for a given api_key and version. +// All required fields contain realistic non-zero values. +// Returns raw bytes WITHOUT the framing header. +fn build_payload(api_key: i16, version: i16) -> Result { + let mut buf = BytesMut::new(); + match api_key { + 18 => { + let mut r = ApiVersionsRequest::default(); + if version >= 3 { + r.client_software_name = StrBytes::from_static_str("kafka-message-gen"); + r.client_software_version = StrBytes::from_static_str("0.1.0"); + } + r.encode(&mut buf, version).context("ApiVersions")?; + } + 3 => { + let mut r = MetadataRequest::default(); + if version >= 1 { + r.topics = None; + } + if version >= 4 { + r.allow_auto_topic_creation = true; + } + if version >= 8 { + r.include_cluster_authorized_operations = false; + r.include_topic_authorized_operations = false; + } + r.encode(&mut buf, version).context("Metadata")?; + } + 0 => { + use kafka_protocol::messages::produce_request::*; + use kafka_protocol::records::{ + Compression, Record, RecordBatchEncoder, RecordEncodeOptions, TimestampType, + }; + let rec = Record { + transactional: false, + control: false, + partition_leader_epoch: 0, + producer_id: -1, + producer_epoch: -1, + timestamp_type: TimestampType::Creation, + offset: 0, + sequence: 0, + timestamp: 1_700_000_000_000, + key: Some(Bytes::from_static(b"test-key")), + value: Some(Bytes::from_static(b"test-value")), + headers: indexmap::IndexMap::new(), + }; + let mut rb = BytesMut::new(); + RecordBatchEncoder::encode( + &mut rb, + [rec].iter(), + &RecordEncodeOptions { + version: 2, + compression: Compression::None, + }, + ) + .context("RecordBatch encode")?; + let pd = TopicProduceData::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partition_data(vec![ + PartitionProduceData::default() + .with_index(0) + .with_records(Some(rb.freeze())), + ]); + let mut r = ProduceRequest::default() + .with_acks(-1) + .with_timeout_ms(5000) + .with_topic_data(vec![pd]); + if version >= 3 { + r.transactional_id = None; + } + r.encode(&mut buf, version).context("Produce")?; + } + 1 => { + use kafka_protocol::messages::fetch_request::*; + let fp = FetchPartition::default() + .with_partition(0) + .with_fetch_offset(0) + .with_partition_max_bytes(1_048_576); + let ft = FetchTopic::default() + .with_topic(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![fp]); + let mut r = FetchRequest::default() + .with_replica_id(BrokerId(-1)) + .with_max_wait_ms(500) + .with_min_bytes(1) + .with_topics(vec![ft]); + if version >= 3 { + r.max_bytes = 52_428_800; + } + if version >= 4 { + r.isolation_level = 0; + } + if version >= 7 { + r.session_id = 0; + r.session_epoch = -1; + } + r.encode(&mut buf, version).context("Fetch")?; + } + 2 => { + use kafka_protocol::messages::list_offsets_request::*; + let p = ListOffsetsPartition::default() + .with_partition_index(0) + .with_timestamp(-1); + let t = ListOffsetsTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + ListOffsetsRequest::default() + .with_replica_id(BrokerId(-1)) + .with_isolation_level(0) + .with_topics(vec![t]) + .encode(&mut buf, version) + .context("ListOffsets")?; + } + 8 => { + use kafka_protocol::messages::offset_commit_request::*; + let p = OffsetCommitRequestPartition::default() + .with_partition_index(0) + .with_committed_offset(42) + .with_committed_metadata(Some(StrBytes::from_static_str(""))); + let t = OffsetCommitRequestTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + OffsetCommitRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_topics(vec![t]) + .encode(&mut buf, version) + .context("OffsetCommit")?; + } + 9 => { + OffsetFetchRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .encode(&mut buf, version) + .context("OffsetFetch")?; + } + 10 => { + FindCoordinatorRequest::default() + .with_key(StrBytes::from_static_str("test-group")) + .with_key_type(0) + .encode(&mut buf, version) + .context("FindCoordinator")?; + } + 11 => { + use kafka_protocol::messages::join_group_request::*; + let p = JoinGroupRequestProtocol::default() + .with_name(StrBytes::from_static_str("range")) + .with_metadata(Bytes::from_static(b"\x00\x00\x00\x01\x00\x0atest-topic")); + JoinGroupRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_session_timeout_ms(30_000) + .with_rebalance_timeout_ms(300_000) + .with_member_id(StrBytes::from_static_str("")) + .with_protocol_type(StrBytes::from_static_str("consumer")) + .with_protocols(vec![p]) + .encode(&mut buf, version) + .context("JoinGroup")?; + } + 12 => { + HeartbeatRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_generation_id(1) + .with_member_id(StrBytes::from_static_str("test-member-1")) + .encode(&mut buf, version) + .context("Heartbeat")?; + } + 13 => { + LeaveGroupRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_member_id(StrBytes::from_static_str("test-member-1")) + .encode(&mut buf, version) + .context("LeaveGroup")?; + } + 14 => { + SyncGroupRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_generation_id(1) + .with_member_id(StrBytes::from_static_str("test-member-1")) + .with_protocol_type(Some(StrBytes::from_static_str("consumer"))) + .with_protocol_name(Some(StrBytes::from_static_str("range"))) + .encode(&mut buf, version) + .context("SyncGroup")?; + } + 15 => { + DescribeGroupsRequest::default() + .with_groups(vec![GroupId::from(StrBytes::from_static_str("test-group"))]) + .with_include_authorized_operations(false) + .encode(&mut buf, version) + .context("DescribeGroups")?; + } + 16 => { + ListGroupsRequest::default() + .encode(&mut buf, version) + .context("ListGroups")?; + } + 17 => { + SaslHandshakeRequest::default() + .with_mechanism(StrBytes::from_static_str("PLAIN")) + .encode(&mut buf, version) + .context("SaslHandshake")?; + } + 19 => { + use kafka_protocol::messages::create_topics_request::*; + let t = CreatableTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str( + "iggy-test-topic", + ))) + .with_num_partitions(1) + .with_replication_factor(1); + CreateTopicsRequest::default() + .with_topics(vec![t]) + .with_timeout_ms(30_000) + .with_validate_only(false) + .encode(&mut buf, version) + .context("CreateTopics")?; + } + 20 => { + use kafka_protocol::messages::delete_topics_request::*; + let r = if version >= 6 { + DeleteTopicsRequest::default() + .with_topics(vec![DeleteTopicState::default().with_name(Some( + TopicName::from(StrBytes::from_static_str("iggy-test-topic")), + ))]) + .with_timeout_ms(30_000) + } else { + DeleteTopicsRequest::default() + .with_topic_names(vec![TopicName::from(StrBytes::from_static_str( + "iggy-test-topic", + ))]) + .with_timeout_ms(30_000) + }; + r.encode(&mut buf, version).context("DeleteTopics")?; + } + 21 => { + use kafka_protocol::messages::delete_records_request::*; + let p = DeleteRecordsPartition::default() + .with_partition_index(0) + .with_offset(0); + let t = DeleteRecordsTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + DeleteRecordsRequest::default() + .with_topics(vec![t]) + .with_timeout_ms(30_000) + .encode(&mut buf, version) + .context("DeleteRecords")?; + } + 22 => { + InitProducerIdRequest::default() + .with_transactional_id(None) + .with_transaction_timeout_ms(60_000) + .encode(&mut buf, version) + .context("InitProducerId")?; + } + 24 => { + use kafka_protocol::messages::add_partitions_to_txn_request::*; + let t = AddPartitionsToTxnTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![0i32]); + AddPartitionsToTxnRequest::default() + .with_v3_and_below_transactional_id(TransactionalId(StrBytes::from_static_str( + "test-txn", + ))) + .with_v3_and_below_producer_id(ProducerId(100)) + .with_v3_and_below_producer_epoch(1) + .with_v3_and_below_topics(vec![t]) + .encode(&mut buf, version) + .context("AddPartitionsToTxn")?; + } + 25 => { + AddOffsetsToTxnRequest::default() + .with_transactional_id(TransactionalId(StrBytes::from_static_str("test-txn"))) + .with_producer_id(ProducerId(100)) + .with_producer_epoch(1) + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .encode(&mut buf, version) + .context("AddOffsetsToTxn")?; + } + 26 => { + EndTxnRequest::default() + .with_transactional_id(TransactionalId(StrBytes::from_static_str("test-txn"))) + .with_producer_id(ProducerId(100)) + .with_producer_epoch(1) + .with_committed(true) + .encode(&mut buf, version) + .context("EndTxn")?; + } + 28 => { + use kafka_protocol::messages::txn_offset_commit_request::*; + let p = TxnOffsetCommitRequestPartition::default() + .with_partition_index(0) + .with_committed_offset(42) + .with_committed_metadata(Some(StrBytes::from_static_str(""))); + let t = TxnOffsetCommitRequestTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + TxnOffsetCommitRequest::default() + .with_transactional_id(TransactionalId(StrBytes::from_static_str("test-txn"))) + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_producer_id(ProducerId(100)) + .with_producer_epoch(1) + .with_topics(vec![t]) + .encode(&mut buf, version) + .context("TxnOffsetCommit")?; + } + 32 => { + use kafka_protocol::messages::describe_configs_request::*; + let r = DescribeConfigsResource::default() + .with_resource_type(2) + .with_resource_name(StrBytes::from_static_str("test-topic")); + DescribeConfigsRequest::default() + .with_resources(vec![r]) + .encode(&mut buf, version) + .context("DescribeConfigs")?; + } + 36 => { + SaslAuthenticateRequest::default() + .with_auth_bytes(Bytes::from_static(b"\x00iggy\x00secret")) + .encode(&mut buf, version) + .context("SaslAuthenticate")?; + } + other => { + warn!("api_key={other}: no explicit builder — empty payload (framing test)"); + } + } + Ok(buf.freeze()) +} + +// Build a complete framed Kafka request message ready for TCP transmission. +fn build_framed(api_key: i16, version: i16, corr: i32) -> Result { + let payload = build_payload(api_key, version)?; + let flexible = first_flexible_version(api_key) + .map(|fv| version >= fv) + .unwrap_or(false); + Ok(frame_request( + api_key, + version, + corr, + "kafka-message-gen", + &payload, + flexible, + )) +} + +// ── Commands ────────────────────────────────────────────────────────────────── + +fn cmd_list() { + println!( + "{:<6} {:<42} {:<10} {:<10} {:<8}", + "Key", "Name", "MinVer", "MaxVer", "Count" + ); + println!("{}", "─".repeat(78)); + for &(k, n, min, max) in API_REGISTRY { + println!( + "{:<6} {:<42} {:<10} {:<10} {:<8}", + k, + n, + min, + max, + max - min + 1 + ); + } + let total: i16 = API_REGISTRY + .iter() + .map(|&(_, _, min, max)| max - min + 1) + .sum(); + println!("{}", "─".repeat(78)); + println!( + "Total: {} API keys | {} versioned messages", + API_REGISTRY.len(), + total + ); +} + +async fn cmd_generate( + out: PathBuf, + fk: Option, + fv: Option, + hex_dump: bool, +) -> Result<()> { + tokio::fs::create_dir_all(&out).await?; + let (mut n, mut corr) = (0usize, 1i32); + for &(ak, name, min, max) in API_REGISTRY { + if fk.is_some_and(|k| k != ak) { + continue; + } + for v in min..=max { + if fv.is_some_and(|fv| fv != v) { + continue; + } + match build_framed(ak, v, corr) { + Ok(msg) => { + let fname = format!("{:03}_{}_v{}.bin", ak, name, v); + tokio::fs::write(out.join(&fname), &msg).await?; + if hex_dump { + println!("── {} v{} ({} bytes) ──", name, v, msg.len()); + println!("{}", hex::encode(&msg)); + println!(); + } else { + info!(" {} ({} bytes)", fname, msg.len()); + } + n += 1; + corr += 1; + } + Err(e) => warn!("SKIP {} v{}: {e}", name, v), + } + } + } + println!("\n✓ Generated {n} messages → {}/", out.display()); + println!( + " Quick test: cat {}/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd", + out.display() + ); + Ok(()) +} + +async fn connect(host: &str) -> Result { + TcpStream::connect(host) + .await + .with_context(|| format!("Cannot connect to {host}")) +} + +async fn read_kafka_response(stream: &mut TcpStream) -> std::io::Result> { + let mut lb = [0u8; 4]; + stream.read_exact(&mut lb).await?; + let frame_len = i32::from_be_bytes(lb); + if frame_len <= 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid response frame length: {frame_len}"), + )); + } + let mut body = vec![ + 0u8; + usize::try_from(frame_len).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "response frame length does not fit usize", + ) + })? + ]; + stream.read_exact(&mut body).await?; + Ok(body) +} + +async fn run_send( + host: &str, + registry: &[(i16, &str, i16, i16)], + filter_keys: &[i16], + fv: Option, + toms: u64, + fail_fast: bool, + quiet: bool, +) -> Result<(usize, usize)> { + let mut stream = connect(host).await?; + info!("Connected to {host}"); + let (mut ok, mut fail, mut corr) = (0usize, 0usize, 1i32); + 'outer: for &(ak, name, min, max) in registry { + if !filter_keys.is_empty() && !filter_keys.contains(&ak) { + continue; + } + for v in min..=max { + if fv.is_some_and(|wanted| wanted != v) { + continue; + } + let msg = match build_framed(ak, v, corr) { + Ok(m) => m, + Err(e) => { + warn!("Build {} v{}: {e}", name, v); + fail += 1; + if fail_fast { + break 'outer; + } + continue; + } + }; + if let Err(e) = stream.write_all(&msg).await { + println!("✗ {name} v{v} → write error: {e}"); + fail += 1; + stream = connect(host).await?; + if fail_fast { + break 'outer; + } + corr += 1; + continue; + } + + let res = tokio::time::timeout( + std::time::Duration::from_millis(toms), + read_kafka_response(&mut stream), + ) + .await; + + match res { + Ok(Ok(r)) => { + let summary = response::analyze_response(ak, v, corr, &r); + summary.print(name, v, quiet); + ok += 1; + } + Ok(Err(e)) => { + println!("✗ {name} v{v} → IO error: {e}"); + fail += 1; + stream = connect(host).await?; + if fail_fast { + break 'outer; + } + } + Err(_) => { + println!("✗ {name} v{v} → timeout ({toms}ms)"); + fail += 1; + stream = connect(host).await?; + if fail_fast { + break 'outer; + } + } + } + corr += 1; + } + } + Ok((ok, fail)) +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_target(false) + .with_level(true) + .init(); + let cli = Cli::parse(); + match cli.command { + Command::List => cmd_list(), + Command::Generate { + output, + api_key, + version, + hex, + } => cmd_generate(output, api_key, version, hex).await?, + Command::Send { + host, + api_key, + version, + timeout_ms, + quiet, + } => { + let filter_keys: Vec = api_key.into_iter().collect(); + let (ok, fail) = run_send( + &host, + API_REGISTRY, + &filter_keys, + version, + timeout_ms, + false, + quiet, + ) + .await?; + println!("\nResult: {ok} OK {fail} failed"); + } + Command::Verify { + host, + api_key, + version, + timeout_ms, + fail_fast, + all_apis, + quiet, + } => { + let registry = if all_apis { + API_REGISTRY + } else { + GATEWAY_REGISTRY + }; + let (ok, fail) = run_send( + &host, registry, &api_key, version, timeout_ms, fail_fast, quiet, + ) + .await?; + println!("\n=== Verify: {ok} passed {fail} failed ==="); + if fail > 0 { + std::process::exit(1); + } + } + } + Ok(()) +} diff --git a/gateways/kafka/tools/kafka-tool/src/response.rs b/gateways/kafka/tools/kafka-tool/src/response.rs new file mode 100644 index 0000000000..a55f5b62a8 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/src/response.rs @@ -0,0 +1,420 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka response frame parsing and human-readable summaries for `send` / `verify`. + +use bytes::Bytes; +use kafka_protocol::messages::{ + ApiVersionsResponse, CreateTopicsResponse, FetchResponse, ListOffsetsResponse, + MetadataResponse, ProduceResponse, +}; +use kafka_protocol::protocol::Decodable; + +/// Parsed view of one length-prefixed Kafka response payload (excluding the 4-byte frame length). +pub struct ResponseSummary { + pub frame_bytes: usize, + pub correlation_id: i32, + pub response_header_version: i16, + pub correlation_match: bool, + /// Highest-severity non-zero error found, or `0` when all decoded codes are zero. + pub primary_error_code: i16, + pub details: Vec, + pub decode_note: Option, +} + +impl ResponseSummary { + #[must_use] + pub fn has_nonzero_error(&self) -> bool { + self.primary_error_code != 0 + } + + pub fn print(&self, api_name: &str, version: i16, quiet: bool) { + let sym = if self.has_nonzero_error() { + "⚠" + } else { + "✓" + }; + let ec_label = format_error_code(self.primary_error_code); + let corr = if self.correlation_match { + format!("{}", self.correlation_id) + } else { + format!("{} (expected correlation mismatch)", self.correlation_id) + }; + + if quiet { + println!( + "{sym} {api_name} v{version} → {}B ec={} ({ec_label})", + self.frame_bytes, self.primary_error_code + ); + return; + } + + println!( + "{sym} {api_name} v{version} frame={}B correlation={corr} resp_hdr=v{} primary_ec={} ({ec_label})", + self.frame_bytes, self.response_header_version, self.primary_error_code + ); + for line in &self.details { + println!(" {line}"); + } + if let Some(note) = &self.decode_note { + println!(" note: {note}"); + } + } +} + +/// Analyze a response payload for the given request `(api_key, api_version)`. +pub fn analyze_response( + api_key: i16, + api_version: i16, + request_correlation_id: i32, + payload: &[u8], +) -> ResponseSummary { + let frame_bytes = payload.len(); + if payload.len() < 4 { + return ResponseSummary { + frame_bytes, + correlation_id: 0, + response_header_version: 0, + correlation_match: false, + primary_error_code: -1, + details: vec!["payload shorter than correlation_id".into()], + decode_note: Some("truncated response".into()), + }; + } + + let correlation_id = i32::from_be_bytes(payload[0..4].try_into().expect("4 bytes")); + let resp_hdr_ver = response_header_version(api_key, api_version); + let body_start = if resp_hdr_ver >= 1 { + 5 // correlation_id + empty tagged fields (0x00) + } else { + 4 + }; + + if payload.len() < body_start { + return ResponseSummary { + frame_bytes, + correlation_id, + response_header_version: resp_hdr_ver, + correlation_match: correlation_id == request_correlation_id, + primary_error_code: -1, + details: vec![format!( + "truncated after correlation (need {body_start} bytes)" + )], + decode_note: None, + }; + } + + let body = &payload[body_start..]; + let mut details = Vec::new(); + let mut codes = Vec::new(); + let mut decode_note = None; + + if body.len() == 2 { + let ec = i16::from_be_bytes(body.try_into().expect("2 bytes")); + codes.push(ec); + details.push(format!( + "error-only body: error_code={ec} ({})", + format_error_code(ec) + )); + } else { + match decode_body(api_key, api_version, body, &mut details, &mut codes) { + Ok(()) => {} + Err(e) => { + decode_note = Some(format!("schema decode failed: {e:#}")); + details.push(format!("raw_body_hex={}", hex::encode(body))); + } + } + } + + let primary_error_code = codes.iter().copied().filter(|&c| c != 0).max().unwrap_or(0); + + ResponseSummary { + frame_bytes, + correlation_id, + response_header_version: resp_hdr_ver, + correlation_match: correlation_id == request_correlation_id, + primary_error_code, + details, + decode_note, + } +} + +fn optional_topic_name(name: &Option) -> String { + name.as_ref() + .map(|n| n.0.as_str().to_string()) + .unwrap_or_else(|| "".into()) +} + +fn topic_name(name: &kafka_protocol::messages::TopicName) -> String { + name.0.as_str().to_string() +} + +fn decode_body( + api_key: i16, + api_version: i16, + body: &[u8], + details: &mut Vec, + codes: &mut Vec, +) -> anyhow::Result<()> { + let mut buf = Bytes::copy_from_slice(body); + match api_key { + 18 => { + let resp = ApiVersionsResponse::decode(&mut buf, api_version)?; + codes.push(resp.error_code); + details.push(format!( + "top_level.error_code={} ({})", + resp.error_code, + format_error_code(resp.error_code) + )); + details.push(format!("api_keys={}", resp.api_keys.len())); + if api_version >= 1 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + for (i, k) in resp.api_keys.iter().enumerate().take(8) { + details.push(format!( + "api_keys[{i}]: key={} min={} max={}", + k.api_key, k.min_version, k.max_version + )); + } + if resp.api_keys.len() > 8 { + details.push(format!("… {} more api_keys", resp.api_keys.len() - 8)); + } + } + 3 => { + let resp = MetadataResponse::decode(&mut buf, api_version)?; + if api_version >= 3 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + details.push(format!("brokers={}", resp.brokers.len())); + if let Some(b) = resp.brokers.first() { + details.push(format!( + "brokers[0]: id={} host={} port={}", + b.node_id.0, b.host, b.port + )); + } + details.push(format!("topics={}", resp.topics.len())); + for (i, t) in resp.topics.iter().enumerate().take(4) { + codes.push(t.error_code); + let name = optional_topic_name(&t.name); + details.push(format!( + "topics[{i}]: name={name} ec={} ({}) partitions={}", + t.error_code, + format_error_code(t.error_code), + t.partitions.len() + )); + } + if resp.topics.len() > 4 { + details.push(format!("… {} more topics", resp.topics.len() - 4)); + } + } + 0 => { + let resp = ProduceResponse::decode(&mut buf, api_version)?; + if api_version >= 1 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + details.push(format!("topics={}", resp.responses.len())); + for (ti, topic) in resp.responses.iter().enumerate().take(4) { + let name = topic_name(&topic.name); + details.push(format!( + "topics[{ti}]: name={name} partitions={}", + topic.partition_responses.len() + )); + for (pi, p) in topic.partition_responses.iter().enumerate().take(4) { + codes.push(p.error_code); + details.push(format!( + " partitions[{pi}]: index={} ec={} ({}) offset={}", + p.index, + p.error_code, + format_error_code(p.error_code), + p.base_offset + )); + } + } + } + 1 => { + let resp = FetchResponse::decode(&mut buf, api_version)?; + if api_version >= 1 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + if api_version >= 7 { + codes.push(resp.error_code); + details.push(format!( + "top_level.error_code={} ({}) session_id={}", + resp.error_code, + format_error_code(resp.error_code), + resp.session_id + )); + } + details.push(format!("topics={}", resp.responses.len())); + for (ti, topic) in resp.responses.iter().enumerate().take(4) { + let name = topic_name(&topic.topic); + details.push(format!( + "topics[{ti}]: name={name} partitions={}", + topic.partitions.len() + )); + for (pi, p) in topic.partitions.iter().enumerate().take(4) { + codes.push(p.error_code); + details.push(format!( + " partitions[{pi}]: index={} ec={} ({}) hw={}", + p.partition_index, + p.error_code, + format_error_code(p.error_code), + p.high_watermark + )); + } + } + } + 2 => { + let resp = ListOffsetsResponse::decode(&mut buf, api_version)?; + if api_version >= 2 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + details.push(format!("topics={}", resp.topics.len())); + for (ti, topic) in resp.topics.iter().enumerate().take(4) { + let name = topic_name(&topic.name); + details.push(format!( + "topics[{ti}]: name={name} partitions={}", + topic.partitions.len() + )); + for (pi, p) in topic.partitions.iter().enumerate().take(4) { + codes.push(p.error_code); + details.push(format!( + " partitions[{pi}]: index={} ec={} ({}) offset={}", + p.partition_index, + p.error_code, + format_error_code(p.error_code), + p.offset + )); + } + } + } + 19 => { + let resp = CreateTopicsResponse::decode(&mut buf, api_version)?; + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + details.push(format!("topics={}", resp.topics.len())); + for (i, t) in resp.topics.iter().enumerate().take(4) { + codes.push(t.error_code); + let name = topic_name(&t.name); + details.push(format!( + "topics[{i}]: name={name} ec={} ({})", + t.error_code, + format_error_code(t.error_code) + )); + } + } + other => { + details.push(format!("no schema decoder for api_key={other}")); + if body.len() >= 2 { + let ec = i16::from_be_bytes(body[0..2].try_into().expect("2 bytes")); + codes.push(ec); + details.push(format!( + "body[0..2] as i16={ec} ({}) — may not be top-level error_code", + format_error_code(ec) + )); + } + } + } + Ok(()) +} + +fn format_error_code(code: i16) -> &'static str { + match code { + 0 => "NONE", + 1 => "OFFSET_OUT_OF_RANGE", + 2 => "CORRUPT_MESSAGE", + 3 => "UNKNOWN_TOPIC_OR_PARTITION", + 35 => "UNSUPPORTED_VERSION", + 36 => "TOPIC_ALREADY_EXISTS", + 37 => "INVALID_PARTITIONS", + 42 => "INVALID_REQUEST", + -1 => "UNKNOWN", + _ => "OTHER", + } +} + +fn request_header_version(api_key: i16, api_version: i16) -> i16 { + let flex_from = first_flexible_version(api_key); + match flex_from { + Some(fv) if api_version >= fv => 2, + _ => 1, + } +} + +fn response_header_version(api_key: i16, api_version: i16) -> i16 { + if api_key == 18 { + return 0; + } + if request_header_version(api_key, api_version) >= 2 { + 1 + } else { + 0 + } +} + +/// First flexible protocol version per API key (matches gateway `header.rs` / kafka-tool framing). +fn first_flexible_version(api_key: i16) -> Option { + match api_key { + 0 => Some(9), + 1 => Some(12), + 2 => Some(6), + 3 => Some(9), + 8 => Some(8), + 9 => Some(6), + 10 => Some(3), + 11 => Some(6), + 12 => Some(4), + 13 => Some(4), + 14 => Some(4), + 15 => Some(5), + 16 => Some(3), + 17 => None, + 18 => Some(3), + 19 => Some(5), + 20 => Some(4), + 21 => Some(2), + 22 => Some(2), + 23 => Some(4), + 24 => Some(3), + 25 => Some(3), + 26 => Some(3), + 27 => Some(1), + 28 => Some(3), + 29 => Some(2), + 30 => Some(2), + 31 => Some(2), + 32 => Some(4), + 33 => Some(2), + 34 => Some(2), + 35 => Some(2), + 36 => Some(2), + 37 => Some(2), + 38 => Some(2), + 39 => Some(2), + 40 => Some(2), + 41 => Some(2), + 42 => Some(2), + 43 => Some(2), + 44 => Some(1), + 45 | 46 => Some(0), + 47 => None, + 48 | 49 => Some(1), + 50 | 51 | 55 | 56 => Some(0), + 57 => Some(1), + 60 | 61 | 64 | 65 | 66 | 67 | 68 | 69 | 71 | 72 | 74 | 75 | 76 => Some(0), + _ => None, + } +}