diff --git a/AGENTS.md b/AGENTS.md index e1cd5f1..eaf9738 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,216 +1,251 @@ -# CLAUDE.md - ClickHouse Format Explorer - -## Project Overview - -A tool for visualizing ClickHouse RowBinary and Native wire format data. Features an interactive hex viewer with AST-based type visualization, similar to ImHex. Available as a web app (Docker) or an Electron desktop app that connects to your existing ClickHouse server. - -**Current scope**: RowBinaryWithNamesAndTypes and Native formats. - -## Tech Stack - -- **Frontend**: React 18 + TypeScript + Vite -- **State**: Zustand -- **UI**: react-window (virtualized hex viewer), react-resizable-panels (split panes) -- **Desktop**: Electron (optional, connects to user's ClickHouse) -- **Testing**: Vitest + testcontainers (integration), Playwright (Electron e2e) -- **Deployment**: Docker (bundles ClickHouse + nginx) or Electron desktop app - -## Commands - -```bash -npm run dev # Start web dev server (requires ClickHouse at localhost:8123) -npm run build # Build web app for production -npm run test # Run integration tests (uses testcontainers) -npm run lint # ESLint check -npm run test:e2e # Build Electron + run Playwright e2e tests - -# Electron desktop app -npm run electron:dev # Dev mode with hot reload -npm run electron:build # Package desktop installer for current platform - -# Docker (self-contained with bundled ClickHouse) -docker build -t rowbinary-explorer . -docker run -d -p 8080:80 rowbinary-explorer -``` - -## Directory Structure - -``` -src/ -├── components/ # React components -│ ├── App.tsx # Main layout with resizable panels -│ ├── QueryInput.tsx # SQL query input + run button + connection settings -│ ├── HexViewer/ # Virtualized hex viewer with highlighting -│ └── AstTree/ # Collapsible AST tree view -├── core/ -│ ├── types/ -│ │ ├── ast.ts # AstNode, ByteRange, ParsedData interfaces -│ │ └── clickhouse-types.ts # ClickHouseType discriminated union -│ ├── decoder/ -│ │ ├── rowbinary-decoder.ts # RowBinaryWithNamesAndTypes decoder -│ │ ├── native-decoder.ts # Native format decoder -│ │ ├── reader.ts # BinaryReader with byte-range tracking -│ │ ├── leb128.ts # LEB128 varint decoder -│ │ ├── test-helpers.ts # Shared test utilities -│ │ ├── smoke-cases.ts # Smoke test case definitions -│ │ └── validation-cases.ts # Validation test case definitions -│ ├── parser/ -│ │ ├── type-lexer.ts # Tokenizer for type strings -│ │ └── type-parser.ts # Parser: string -> ClickHouseType -│ └── clickhouse/ -│ └── client.ts # HTTP client (fetch for web, IPC for Electron) -├── store/ -│ └── store.ts # Zustand store (query, parsed data, UI state) -└── styles/ # CSS files -electron/ -├── main.ts # Electron main process (window, IPC handlers) -└── preload.ts # Preload script (contextBridge → electronAPI) -e2e/ -└── electron.spec.ts # Playwright Electron e2e tests -docs/ -├── rowbinaryspec.md # RowBinary wire format specification -├── nativespec.md # Native wire format specification -└── jsonspec.md # JSON type specification -docker/ -├── nginx.conf # Proxies /clickhouse to ClickHouse server -├── users.xml # Read-only ClickHouse user -└── supervisord.conf # Runs nginx + ClickHouse together -``` - -## Wire Format Docs - - * RowBinary: docs/rowbinaryspec.md - * Native: docs/nativespec.md - * JSON: docs/jsonspec.md - -## Key Concepts - -### AstNode -Every decoded value is represented as an `AstNode` (`src/core/types/ast.ts:12`): -- `id` - Unique identifier for selection/highlighting -- `type` - ClickHouse type name string -- `byteRange` - `{start, end}` byte offsets (exclusive end) -- `value` - Decoded JavaScript value -- `displayValue` - Human-readable string -- `children` - Child nodes for composite types (Array, Tuple, etc.) - -### ClickHouseType -A discriminated union representing all ClickHouse types (`src/core/types/clickhouse-types.ts:4`): -- Primitives: `UInt8`-`UInt256`, `Int8`-`Int256`, `Float32/64`, `String`, etc. -- Composites: `Array`, `Tuple`, `Map`, `Nullable`, `LowCardinality` -- Advanced: `Variant`, `Dynamic`, `JSON` -- Geo: `Point`, `Ring`, `Polygon`, `MultiPolygon`, `LineString`, `MultiLineString`, `Geometry` -- Intervals: `IntervalSecond`, `IntervalMinute`, `IntervalHour`, `IntervalDay`, `IntervalWeek`, `IntervalMonth`, `IntervalQuarter`, `IntervalYear` (stored as Int64) -- Other: `Enum8/16`, `Nested`, `QBit`, `AggregateFunction` - -### Decoding Flow -1. User enters SQL query, clicks "Run Query" -2. `ClickHouseClient` (`src/core/clickhouse/client.ts`) sends query: - - **Web mode**: `fetch()` via Vite proxy or nginx - - **Electron mode**: IPC to main process → `fetch()` to ClickHouse (no CORS) -3. Decoder parses the binary response: - - **RowBinary** (`rowbinary-decoder.ts`): Row-oriented, header + rows - - **Native** (`native-decoder.ts`): Column-oriented with blocks -4. Type strings parsed via `parseType()` into `ClickHouseType` -5. Each decoded value returns an `AstNode` with byte tracking -6. UI renders hex view (left) and AST tree (right) - -### Electron Architecture -``` -Renderer (React) Main Process (Node.js) - │ │ - ├─ window.electronAPI │ - │ .executeQuery(opts) ────────►├─ fetch(clickhouseUrl + query) - │ │ → ArrayBuffer - │◄── IPC response ──────────────┤ - │ │ - ├─ Uint8Array → decoders │ - └─ render hex view + AST tree │ -``` - -- Runtime detection: `window.electronAPI` exists → IPC path, otherwise → `fetch()` -- `vite-plugin-electron` activates only when `ELECTRON=true` env var is set -- Connection config in `config.json` (project root in dev, next to executable in prod) -- Experimental ClickHouse settings (Variant, Dynamic, JSON, etc.) sent as query params - -### Interactive Highlighting -- Click a node in AST tree → highlights corresponding bytes in hex view -- Click a byte in hex view → selects the deepest AST node containing that byte -- State managed in Zustand store: `activeNodeId`, `hoveredNodeId` - -## Adding a New ClickHouse Type - -1. Add type variant to `ClickHouseType` in `src/core/types/clickhouse-types.ts` -2. Add `typeToString()` case for serialization back to string -3. Add `getTypeColor()` case for UI coloring -4. Add parser case in `src/core/parser/type-parser.ts` -5. Add decoder method in `RowBinaryDecoder` (`src/core/decoder/rowbinary-decoder.ts`): - - Add case in `decodeValue()` switch - - Implement `decode{TypeName}()` method returning `AstNode` -6. Add decoder method in `NativeDecoder` (`src/core/decoder/native-decoder.ts`): - - Add case in `decodeValue()` switch - - For columnar types, may need `decode{TypeName}Column()` method -7. If type has binary type index (for Dynamic), add to `decodeDynamicType()` -8. Add test cases to `smoke-cases.ts` and `validation-cases.ts` - -## Important Implementation Details - -- **LEB128**: Variable-length integers used for string lengths, array sizes, column counts -- **UUID byte order**: ClickHouse uses a special byte ordering (see `decodeUUID()` at `decoder.ts:629`) -- **IPv4**: Stored as little-endian UInt32, displayed in reverse order -- **Dynamic type**: Uses BinaryTypeIndex encoding; type is encoded in the data itself -- **LowCardinality**: Does not affect wire format in RowBinary (transparent wrapper) -- **Nested**: Encoded as parallel arrays, one per field - -## Testing - -### Integration Tests (Vitest + testcontainers) - -Tests use testcontainers to spin up a real ClickHouse instance: -```bash -npm run test # Runs all integration tests -``` - -Tests are organized into three categories with shared test case definitions: - -1. **Smoke Tests** (`smoke.integration.test.ts`) - - Verify parsing succeeds without value validation - - Test cases defined in `smoke-cases.ts` - - Parametrized for both RowBinary and Native formats - -2. **Validation Tests** (`validation.integration.test.ts`) - - Verify decoded values and AST structure - - Test cases defined in `validation-cases.ts` with format-specific callbacks - - Check values, children counts, byte ranges, metadata - -3. **Coverage Tests** (`coverage.integration.test.ts`) - - Analyze byte coverage of AST leaf nodes - - Report uncovered byte ranges - -### Electron e2e Tests (Playwright) - -```bash -npm run test:e2e # Builds Electron app + runs Playwright tests -``` - -Tests in `e2e/electron.spec.ts` launch the actual Electron app and verify: -- App window opens and UI renders -- Host input is visible (Electron mode) and Share button is hidden -- Connection settings can be edited -- Upload button is present and functional - -### Test Case Interface -```typescript -interface ValidationTestCase { - name: string; - query: string; - settings?: Record; - rowBinaryValidator?: (result: DecodedResult) => void; - nativeValidator?: (result: DecodedResult) => void; -} -``` - -### Adding New Test Cases -1. Add query to `smoke-cases.ts` for basic parsing verification -2. Add to `validation-cases.ts` with validator callbacks for detailed checks -3. Use `bothFormats(validator)` helper when validation logic is identical +# CLAUDE.md - ClickHouse Format Explorer + +## Project Overview + +A tool for visualizing ClickHouse RowBinary and Native wire format data. Features an interactive hex viewer with AST-based type visualization, similar to ImHex. Available as a web app (Docker) or an Electron desktop app that connects to your existing ClickHouse server. + +**Current scope**: RowBinaryWithNamesAndTypes and Native formats (including Native protocol revision selection via `client_protocol_version`), plus the **Native TCP protocol** — a full packet-stream capture decoded from a proxy dump (handshake, Query/ClientInfo/Settings, Data/Totals/Extremes/Log/ProfileEvents blocks, Progress, ProfileInfo, Exception, etc.). + +### Native TCP protocol capture + +The `NativeProtocol` format decodes a whole connection's packet stream rather than a single HTTP format body. A small TCP proxy (`scripts/native-proxy.mjs`, mirrored for the app in `electron/native-capture.ts`) sits between `clickhouse-client` and the server, forwarding bytes and teeing both directions into a capture. On localhost clickhouse-client disables compression, so the capture is plaintext, uncompressed packets (TLS/compression are out of scope). + +- Capture a dump for tests/inspection: `npm run capture -- --query "SELECT 1" --out cap.chproto` (see `scripts/capture-native.mjs`). +- `.chproto` dump format and parsing: `scripts/native-proxy.mjs` (writer) and `src/core/decoder/protocol-dump.ts` (reader). +- Decoder: `src/core/decoder/protocol-decoder.ts` (`ProtocolDecoder`) reuses `NativeDecoder.decodeProtocolBlock()` for the Block inside Data-family packets. It derives the negotiated version from `min(client, server)` Hello and gates every field per `docs/full_native_protocol_spec.md`. +- Capture from the **web UI**: the browser POSTs SQL to a `/capture` endpoint that runs the proxy server-side and returns the `.chproto` dump (the browser can't open raw TCP itself). Served by: `npm run dev` / `vite preview` (Vite plugin in `vite.config.ts` → `scripts/capture-middleware.mjs`), and the **Docker** image (standalone `scripts/capture-server.mjs` under supervisord, proxied by nginx). Native-connection defaults come from env: `CH_NATIVE_HOST`/`CH_NATIVE_PORT`/`CH_USER`/`CH_PASSWORD`/`CLICKHOUSE_CLIENT`; `CAPTURE_EXPERIMENTAL_SETTINGS=0` stops sending experimental type settings per-query (for read-only users that reject them — rely on the profile instead). +- Capture flow: renderer → `clickhouse.captureProtocol()` → **Electron** IPC `capture-native-protocol`, or **web** `POST /capture` → proxy + clickhouse-client → `{c2s, s2c}` → `ProtocolDecoder`. You can also **load** a `.chproto` file via Upload in either mode. +- Regression fixtures + 100%-coverage tests: `src/core/decoder/fixtures/protocol/*.chproto`, `src/core/decoder/protocol-decoder.test.ts`. +- **Known spec gap**: newer servers (observed: 26.2, proto 54483) append an extra trailing version VarUInt to `ServerHello` after `cluster_function_protocol_version` that the spec/public source don't document; the decoder consumes any trailing VarUInt that can't begin a valid post-hello packet (`hello_tail_extra_version`). + +## Tech Stack + +- **Frontend**: React 18 + TypeScript + Vite +- **State**: Zustand +- **UI**: react-window (virtualized hex viewer), react-resizable-panels (split panes) +- **Desktop**: Electron (optional, connects to user's ClickHouse) +- **Testing**: Vitest + testcontainers (integration), Playwright (Electron e2e) +- **Deployment**: Docker (bundles ClickHouse + nginx) or Electron desktop app + +## Commands + +```bash +npm run dev # Start web dev server (requires ClickHouse at localhost:8123) +npm run build # Build web app for production +npm run test # Run integration tests (uses testcontainers) +npm run lint # ESLint check +npm run test:e2e # Build Electron + run Playwright e2e tests + +# Electron desktop app +npm run electron:dev # Dev mode with hot reload +npm run electron:build # Package desktop installer for current platform + +# Docker (self-contained with bundled ClickHouse) +docker build -t rowbinary-explorer . +docker run -d -p 8080:80 rowbinary-explorer # read-only (default) +docker run -d -p 8080:80 -e READONLY=0 rowbinary-explorer # allow INSERT/DDL (internal use) +``` + +The Docker image bundles ClickHouse + nginx + a Node **capture server** (supervisord runs all three). nginx proxies `/clickhouse` (HTTP formats) and `/capture` (native-protocol capture) to local backends. The `READONLY` env (default `1`) picks the ClickHouse user for *both* paths: `viewer` (read-only) or `writer` (can INSERT/DDL). Native-connection overrides: `CH_NATIVE_HOST`/`CH_NATIVE_PORT`/`CH_USER`/`CH_PASSWORD`. + +## Directory Structure + +``` +src/ +├── components/ # React components +│ ├── App.tsx # Main layout with resizable panels +│ ├── QueryInput.tsx # SQL query input + run button + connection settings +│ ├── HexViewer/ # Virtualized hex viewer with highlighting +│ └── AstTree/ # Collapsible AST tree view +├── core/ +│ ├── types/ +│ │ ├── ast.ts # AstNode, ByteRange, ParsedData interfaces +│ │ ├── clickhouse-types.ts # ClickHouseType discriminated union +│ │ └── native-protocol.ts # Native protocol preset definitions and helpers +│ ├── decoder/ +│ │ ├── rowbinary-decoder.ts # RowBinaryWithNamesAndTypes decoder +│ │ ├── native-decoder.ts # Native format decoder with protocol-aware parsing +│ │ ├── reader.ts # BinaryReader with byte-range tracking +│ │ ├── leb128.ts # LEB128 varint decoder +│ │ ├── test-helpers.ts # Shared test utilities +│ │ ├── smoke-cases.ts # Smoke test case definitions +│ │ └── validation-cases.ts # Validation test case definitions +│ ├── parser/ +│ │ ├── type-lexer.ts # Tokenizer for type strings +│ │ └── type-parser.ts # Parser: string -> ClickHouseType +│ └── clickhouse/ +│ ├── client.ts # HTTP client (fetch for web, IPC for Electron) +│ └── request-params.ts # Shared request parameter builder +├── store/ +│ └── store.ts # Zustand store (query, parsed data, UI state) +└── styles/ # CSS files +electron/ +├── main.ts # Electron main process (window, IPC handlers) +└── preload.ts # Preload script (contextBridge → electronAPI) +e2e/ +└── electron.spec.ts # Playwright Electron e2e tests +docs/ +├── rowbinaryspec.md # RowBinary wire format specification +├── nativespec.md # Native wire format specification +├── native-protocol-versions.md # Native protocol revision reference +└── jsonspec.md # JSON type specification +docker/ +├── nginx.conf.template # Proxies /clickhouse + /capture; proxy user templated from READONLY +├── users.xml # viewer (read-only) + writer (read-write) users/profiles +├── start-nginx.sh # Renders nginx.conf from the template (picks viewer/writer) +├── start-capture.sh # Starts the native-protocol capture server (picks viewer/writer) +└── supervisord.conf # Runs ClickHouse + capture server + nginx together +``` + +## Wire Format Docs + + * RowBinary: docs/rowbinaryspec.md + * Native: docs/nativespec.md + * Native revisions: docs/native-protocol-versions.md + * JSON: docs/jsonspec.md + +## Key Concepts + +### AstNode +Every decoded value is represented as an `AstNode` (`src/core/types/ast.ts:12`): +- `id` - Unique identifier for selection/highlighting +- `type` - ClickHouse type name string +- `byteRange` - `{start, end}` byte offsets (exclusive end) +- `value` - Decoded JavaScript value +- `displayValue` - Human-readable string +- `children` - Child nodes for composite types (Array, Tuple, etc.) + +### ClickHouseType +A discriminated union representing all ClickHouse types (`src/core/types/clickhouse-types.ts:4`): +- Primitives: `UInt8`-`UInt256`, `Int8`-`Int256`, `Float32/64`, `String`, etc. +- Composites: `Array`, `Tuple`, `Map`, `Nullable`, `LowCardinality` +- Advanced: `Variant`, `Dynamic`, `JSON` +- Geo: `Point`, `Ring`, `Polygon`, `MultiPolygon`, `LineString`, `MultiLineString`, `Geometry` +- Intervals: `IntervalSecond`, `IntervalMinute`, `IntervalHour`, `IntervalDay`, `IntervalWeek`, `IntervalMonth`, `IntervalQuarter`, `IntervalYear` (stored as Int64) +- Other: `Enum8/16`, `Nested`, `QBit`, `AggregateFunction` + +### Decoding Flow +1. User enters SQL query, clicks "Run Query" +2. `ClickHouseClient` (`src/core/clickhouse/client.ts`) sends query: + - **Web mode**: `fetch()` via Vite proxy or nginx + - **Electron mode**: IPC to main process → `fetch()` to ClickHouse (no CORS) +3. Decoder parses the binary response: + - **RowBinary** (`rowbinary-decoder.ts`): Row-oriented, header + rows + - **Native** (`native-decoder.ts`): Column-oriented with protocol-aware block parsing +4. Type strings parsed via `parseType()` into `ClickHouseType` +5. Each decoded value returns an `AstNode` with byte tracking +6. UI renders hex view (left) and AST tree (right) + +For `Native`, the UI can attach `client_protocol_version` to the request. That affects both ClickHouse output and the local decoder: +- `0`: legacy HTTP Native path, no explicit `client_protocol_version` +- `> 0`: revision-aware parsing with `BlockInfo` +- `54454+`: per-column serialization metadata +- `54473+`: Dynamic/JSON v2 Native structures +- `54482+`: replicated serialization kinds +- `54483+`: nullable sparse serialization + +### Electron Architecture +``` +Renderer (React) Main Process (Node.js) + │ │ + ├─ window.electronAPI │ + │ .executeQuery(opts) ────────►├─ fetch(clickhouseUrl + query) + │ │ → ArrayBuffer + │◄── IPC response ──────────────┤ + │ │ + ├─ Uint8Array → decoders │ + └─ render hex view + AST tree │ +``` + +- Runtime detection: `window.electronAPI` exists → IPC path, otherwise → `fetch()` +- `vite-plugin-electron` activates only when `ELECTRON=true` env var is set +- Connection config in `config.json` (project root in dev, next to executable in prod) +- Experimental ClickHouse settings (Variant, Dynamic, JSON, etc.) sent as query params + +### Interactive Highlighting +- Click a node in AST tree → highlights corresponding bytes in hex view +- Click a byte in hex view → selects the deepest AST node containing that byte +- State managed in Zustand store: `activeNodeId`, `hoveredNodeId` + +## Adding a New ClickHouse Type + +1. Add type variant to `ClickHouseType` in `src/core/types/clickhouse-types.ts` +2. Add `typeToString()` case for serialization back to string +3. Add `getTypeColor()` case for UI coloring +4. Add parser case in `src/core/parser/type-parser.ts` +5. Add decoder method in `RowBinaryDecoder` (`src/core/decoder/rowbinary-decoder.ts`): + - Add case in `decodeValue()` switch + - Implement `decode{TypeName}()` method returning `AstNode` +6. Add decoder method in `NativeDecoder` (`src/core/decoder/native-decoder.ts`): + - Add case in `decodeValue()` switch + - For columnar types, may need `decode{TypeName}Column()` method +7. If type has binary type index (for Dynamic), add to `decodeDynamicType()` +8. Add test cases to `smoke-cases.ts` and `validation-cases.ts` + +## Important Implementation Details + +- **LEB128**: Variable-length integers used for string lengths, array sizes, column counts +- **UUID byte order**: ClickHouse uses a special byte ordering (see `decodeUUID()` at `decoder.ts:629`) +- **IPv4**: Stored as little-endian UInt32, displayed in reverse order +- **Dynamic type**: Uses BinaryTypeIndex encoding; type is encoded in the data itself +- **LowCardinality**: Does not affect wire format in RowBinary (transparent wrapper) +- **Nested**: Encoded as parallel arrays, one per field + +## Testing + +### Integration Tests (Vitest + testcontainers) + +Tests use testcontainers to spin up a real ClickHouse instance: +```bash +npm run test # Runs all integration tests +``` + +Tests are organized into three categories with shared test case definitions: + +1. **Smoke Tests** (`smoke.integration.test.ts`) + - Verify parsing succeeds without value validation + - Test cases defined in `smoke-cases.ts` + - Parametrized for both RowBinary and Native formats + +2. **Validation Tests** (`validation.integration.test.ts`) + - Verify decoded values and AST structure + - Test cases defined in `validation-cases.ts` with format-specific callbacks + - Check values, children counts, byte ranges, metadata + +3. **Coverage Tests** (`coverage.integration.test.ts`) + - Analyze byte coverage of AST leaf nodes + - Report uncovered byte ranges + +4. **Native Protocol Matrix** (`native-protocol.integration.test.ts`) + - Runs the same Native queries across every exposed protocol preset + - Verifies revision gates such as LowCardinality negotiation, serialization metadata, Dynamic v1/v2, JSON v1/v2, and modern `BlockInfo` + - Avoids assuming ClickHouse will choose sparse/replicated encodings for ordinary HTTP queries unless the case is deterministic + +### Electron e2e Tests (Playwright) + +```bash +npm run test:e2e # Builds Electron app + runs Playwright tests +``` + +Tests in `e2e/electron.spec.ts` launch the actual Electron app and verify: +- App window opens and UI renders +- Host input is visible (Electron mode) and Share button is hidden +- Connection settings can be edited +- Upload button is present and functional + +### Test Case Interface +```typescript +interface ValidationTestCase { + name: string; + query: string; + settings?: Record; + rowBinaryValidator?: (result: DecodedResult) => void; + nativeValidator?: (result: DecodedResult) => void; +} +``` + +### Adding New Test Cases +1. Add query to `smoke-cases.ts` for basic parsing verification +2. Add to `validation-cases.ts` with validator callbacks for detailed checks +3. Use `bothFormats(validator)` helper when validation logic is identical +4. If the change is Native revision-specific, add or extend `native-protocol.test.ts` for synthetic coverage and `native-protocol.integration.test.ts` for real ClickHouse behavior across all presets diff --git a/Dockerfile b/Dockerfile index 1540f76..f31fe40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,20 +12,38 @@ FROM clickhouse/clickhouse-server:latest # Install nginx and supervisor RUN apt-get update && apt-get install -y nginx supervisor && rm -rf /var/lib/apt/lists/* +# Node runtime for the capture server (glibc binary copied from the official +# Debian-based node image; the capture scripts use only Node built-ins, so no +# node_modules are required). The clickhouse-server image is glibc-based too. +COPY --from=node:20-bookworm-slim /usr/local/bin/node /usr/local/bin/node + +# clickhouse-client used by the capture proxy. The single `clickhouse` binary +# dispatches on argv[0], so a clickhouse-client symlink runs in client mode. +RUN ln -sf /usr/bin/clickhouse /usr/local/bin/clickhouse-client + # Copy built frontend COPY --from=builder /app/dist /var/www/html -# Copy nginx config (proxies /clickhouse to ClickHouse) -COPY docker/nginx.conf /etc/nginx/nginx.conf +# Capture server + proxy harness (no dependencies beyond Node built-ins) +COPY scripts/native-proxy.mjs scripts/capture-middleware.mjs scripts/capture-server.mjs /app/scripts/ -# Copy ClickHouse user config (read-only viewer user) +# nginx config template (rendered at start with the proxy user) + start scripts +COPY docker/nginx.conf.template /etc/nginx/nginx.conf.template +COPY docker/start-nginx.sh docker/start-capture.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/start-nginx.sh /usr/local/bin/start-capture.sh + +# Copy ClickHouse user config (viewer = read-only, writer = read-write) COPY docker/users.xml /etc/clickhouse-server/users.d/viewer.xml -# Copy supervisor config (runs both nginx and ClickHouse) +# Copy supervisor config (runs ClickHouse + capture server + nginx) COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf -# Expose only web port (ClickHouse internal only) +# READONLY=1 (default) serves the read-only viewer user; set READONLY=0 for an +# internal deployment that should also be able to INSERT. +ENV READONLY=1 + +# Expose only web port (ClickHouse and capture server are internal only) EXPOSE 80 -# Start supervisor (manages nginx + ClickHouse) +# Start supervisor (manages ClickHouse + capture server + nginx) CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] diff --git a/docker/nginx.conf b/docker/nginx.conf.template similarity index 54% rename from docker/nginx.conf rename to docker/nginx.conf.template index aa6fe3e..5acf460 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf.template @@ -17,15 +17,23 @@ http { try_files $uri $uri/ /index.html; } - # Proxy /clickhouse to ClickHouse server + # Native TCP protocol capture: proxied to the local capture server, + # which drives clickhouse-client through the packet-capturing proxy. + location /capture { + proxy_pass http://127.0.0.1:8124; + proxy_set_header Host $host; + proxy_read_timeout 120s; + } + + # Proxy /clickhouse to ClickHouse server. __CH_PROXY_USER__ is rendered + # at container start from the READONLY env var (viewer or writer). location /clickhouse { rewrite ^/clickhouse(.*) $1 break; proxy_pass http://127.0.0.1:8123; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - # Use the read-only viewer user - proxy_set_header X-ClickHouse-User viewer; + proxy_set_header X-ClickHouse-User __CH_PROXY_USER__; proxy_set_header X-ClickHouse-Key ""; } } diff --git a/docker/start-capture.sh b/docker/start-capture.sh new file mode 100644 index 0000000..9bd8097 --- /dev/null +++ b/docker/start-capture.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Start the native-protocol capture server. The ClickHouse user it connects as +# follows the READONLY toggle (same as the HTTP proxy): +# +# READONLY=1 (default) -> viewer (read-only; experimental settings come from +# the profile, not per-query, so they are +# not re-sent as client settings) +# READONLY=0 -> writer (can INSERT / DDL) +# +# Any of CH_USER / CH_PASSWORD / CH_NATIVE_HOST / CH_NATIVE_PORT can override. +set -e + +if [ "${READONLY:-1}" = "0" ]; then + DEFAULT_USER=writer + : "${CAPTURE_EXPERIMENTAL_SETTINGS:=1}" +else + DEFAULT_USER=viewer + # A readonly user rejects per-query setting changes; rely on its profile. + : "${CAPTURE_EXPERIMENTAL_SETTINGS:=0}" +fi + +export CH_USER="${CH_USER:-$DEFAULT_USER}" +export CH_PASSWORD="${CH_PASSWORD:-}" +export CH_NATIVE_HOST="${CH_NATIVE_HOST:-127.0.0.1}" +export CH_NATIVE_PORT="${CH_NATIVE_PORT:-9000}" +export CLICKHOUSE_CLIENT="${CLICKHOUSE_CLIENT:-clickhouse-client}" +export CAPTURE_EXPERIMENTAL_SETTINGS +export CAPTURE_BIND="${CAPTURE_BIND:-127.0.0.1}" +export CAPTURE_PORT="${CAPTURE_PORT:-8124}" + +exec node /app/scripts/capture-server.mjs diff --git a/docker/start-nginx.sh b/docker/start-nginx.sh new file mode 100644 index 0000000..896a843 --- /dev/null +++ b/docker/start-nginx.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Render the nginx config, substituting the ClickHouse proxy user based on the +# READONLY toggle, then run nginx in the foreground. +# +# READONLY=1 (default) -> viewer (read-only) +# READONLY=0 -> writer (can INSERT / DDL) +set -e + +if [ "${READONLY:-1}" = "0" ]; then + CH_PROXY_USER=writer +else + CH_PROXY_USER=viewer +fi + +sed "s/__CH_PROXY_USER__/${CH_PROXY_USER}/g" \ + /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf + +exec /usr/sbin/nginx -g "daemon off;" diff --git a/docker/supervisord.conf b/docker/supervisord.conf index f568805..d06fc29 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -11,8 +11,17 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 +[program:capture] +command=/usr/local/bin/start-capture.sh +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + [program:nginx] -command=/usr/sbin/nginx -g "daemon off;" +command=/usr/local/bin/start-nginx.sh autostart=true autorestart=true stdout_logfile=/dev/stdout diff --git a/docker/users.xml b/docker/users.xml index 3f6f132..ea45ca0 100644 --- a/docker/users.xml +++ b/docker/users.xml @@ -1,4 +1,12 @@ + @@ -10,6 +18,15 @@ default 0 + + + + ::/0 + + readwrite + default + 0 + @@ -22,5 +39,14 @@ 1 1 + + + 1 + 1 + 1 + 1 + 1 + 1 + diff --git a/docs/full_native_protocol_spec.md b/docs/full_native_protocol_spec.md new file mode 100644 index 0000000..ce0a407 --- /dev/null +++ b/docs/full_native_protocol_spec.md @@ -0,0 +1,895 @@ +--- +description: 'Specification of the ClickHouse native TCP protocol: packet framing, the connection lifecycle, version negotiation, and the body of each message' +sidebar_label: 'Native Protocol' +slug: /interfaces/specs/NativeProtocol +title: 'Native Protocol' +doc_type: 'reference' +keywords: ['native protocol', 'TCP', 'wire protocol', 'handshake', 'packets', 'connection'] +--- + +The native protocol is the binary, connection-oriented protocol that ClickHouse clients and servers speak over TCP. It carries SQL queries, result data, `INSERT` payloads, execution telemetry, and error signals. It is the protocol behind the command-line client and the C++ and most third-party native drivers. + +This page covers the protocol itself: packet framing, the connection state machine, version negotiation, and the body of every non-`Block` message. The bytes inside `Data`-family packets (the `Block`, its columns, and the per-type encodings) are a separate concern, documented in the [Native Format](/interfaces/specs/NativeFormat) specification. + +A few properties hold throughout. The protocol is binary and positional: there are no field tags except inside `BlockInfo`, so a single misplaced byte desynchronizes everything that follows. It is stateful, and each TCP connection processes one query at a time — there is no multiplexing. Fixed-width integers are little-endian. + +## Overview {#overview} + +| Property | Value | +|------------------|-------| +| Transport | TCP, optionally wrapped in TLS | +| Byte order | Little-endian for fixed-width integers | +| Encoding | Binary and positional (no field tags except in `BlockInfo`) | +| Connection model | Stateful, one query at a time, no multiplexing | +| Versioning | Negotiated at handshake; individual features gated by version | +| Data format | The [Native Format](/interfaces/specs/NativeFormat) for all tabular data | + +Every message on the wire starts with a `VarUInt` packet type code, followed by a body whose shape depends on that code and on the negotiated protocol version. + +A connection runs through three phases — a one-time handshake, then any number of `Ping` or `Query` exchanges, then close: + +```mermaid +sequenceDiagram + autonumber + participant C as Client + participant S as Server + + C->>S: TCP connect + + rect rgb(220, 235, 255) + Note over C,S: Handshake + C->>S: ClientHello (name, version, db, user, password) + S->>C: ServerHello (server_name, version, [timezone, display_name, ...]) + Note over C,S: negotiated_version = min(client, server) + opt negotiated_version ≥ 54458 + C->>S: Addendum (quota_key) + end + end + + rect rgb(220, 245, 225) + Note over C,S: Query phase + C->>S: Query packet (ClientInfo, settings, params, SQL) + C->>S: External-table Data packets (0 or more) + C->>S: Empty Data marker — the "go" signal + loop until EndOfStream or Exception + S->>C: Data / Progress / Log / ProfileInfo / Totals / ... + end + S->>C: EndOfStream + end +``` + +The native TCP protocol always carries tabular data in the Native format, regardless of any `FORMAT` clause in the SQL. Re-formatting into `RowBinary`, `CSV`, `JSON`, and so on is the client's job, done after it decodes the Native blocks. (The HTTP interface is a different code path that *does* honour the `FORMAT` clause; HTTP is out of scope here.) + +## Security {#security} + +### Transport security (TLS) {#transport-security} + +TLS lives at the transport layer, below the protocol. When it is enabled the entire TCP stream is encrypted, and the protocol messages are byte-for-byte identical whether TLS is in use or not. + +### Authentication {#authentication} + +Authentication happens during the handshake, in the [`ClientHello`](#clienthello) message. The `user` and `password` fields travel as plaintext strings, so transport-level encryption (TLS) is what protects credentials in transit. + +SSH challenge-response authentication is available from protocol version 54466 onward — see [SSH challenge-response authentication](#ssh-authentication). + +### Inter-server secret {#inter-server-secret} + +For distributed query execution, servers authenticate to one another with a shared secret string carried in the [`Query`](#query) message (the `cluster_secret` field). This is gated by the `INTERSERVER_SECRET` feature (v54441). External clients always send an empty string here. + +## Versioning and feature gates {#versioning-and-feature-gates} + +### Version negotiation {#version-negotiation} + +Both client and server declare their maximum supported protocol version during the handshake. The **negotiated version** is the smaller of the two: + +```text +negotiated_version = min(client_version, server_version) +``` + +Every message after that uses the negotiated version to decide which fields are present on the wire. + +### Feature gates {#feature-gates} + +A feature is identified by the protocol version that introduced it, and is **active** when the negotiated version is greater than or equal to that number. + +:::warning +When a feature is active, its fields **must** be present on the wire. The protocol is strictly positional, so omitting a feature-gated field corrupts the byte stream for every field that follows. +::: + +### Feature table {#feature-table} + +| Feature | Version | Affects | Wire impact | +|---------------------------------|---------|------------------------|-------------| +| BLOCK_INFO | 51903 | Block | Adds the BlockInfo prefix (`is_overflows`, `bucket_number`) to every Block. | +| TIMEZONE | 54058 | ServerHello | Adds the `timezone` field to ServerHello. | +| QUOTA_KEY_IN_CLIENT_INFO | 54060 | ClientInfo | Adds the `quota_key` field to ClientInfo. | +| DISPLAY_NAME | 54372 | ServerHello | Adds the `display_name` field to ServerHello. | +| VERSION_PATCH | 54401 | ServerHello, ClientInfo | Adds the `version_patch` field to both. | +| SERVER_LOGS | 54406 | Log | Server emits Log packets when `send_logs_level` is set. | +| WRITE_CLIENT_INFO | 54420 | Query, Progress | Adds the ClientInfo block to Query; adds `wrote_rows` and `wrote_bytes` to Progress. | +| SETTINGS_SERIALIZED_AS_STRINGS | 54429 | Query | Encodes settings as string key-value pairs in the Query body. | +| INTERSERVER_SECRET | 54441 | Query | Adds the `cluster_secret` field to Query. | +| OPEN_TELEMETRY | 54442 | ClientInfo | Adds the OpenTelemetry trace context to ClientInfo. | +| DISTRIBUTED_DEPTH | 54448 | ClientInfo | Adds the `distributed_depth` field to ClientInfo. | +| INITIAL_QUERY_START_TIME | 54449 | ClientInfo | Adds the `initial_time` field (Int64, fixed-width). | +| PROFILE_EVENTS | 54451 | ProfileEvents | Server emits ProfileEvents packets during query execution. | +| PARALLEL_REPLICAS | 54453 | ClientInfo | Adds parallel-replica coordination fields to ClientInfo. | +| CUSTOM_SERIALIZATION | 54454 | Block (Column) | Adds the `has_custom_serialization` byte after each column's type string. | +| ADDENDUM | 54458 | Handshake | Client sends an addendum (`quota_key`) after the handshake exchange. | +| PARAMETERS | 54459 | Query | Adds the parameters list to the Query body. | +| SERVER_QUERY_TIME_IN_PROGRESS | 54460 | Progress | Adds the `elapsed_ns` field to Progress. | +| PASSWORD_COMPLEXITY_RULES | 54461 | ServerHello | Adds a list of password-policy regex patterns and human-readable messages to ServerHello. | +| INTERSERVER_SECRET_V2 | 54462 | ServerHello | Adds an 8-byte `UInt64` nonce to ServerHello. Used by inter-server query signing; external clients decode and ignore. | +| TOTAL_BYTES_IN_PROGRESS | 54463 | Progress | Adds the `total_bytes_to_read` (VarUInt) field to Progress, between `total_rows` and `wrote_rows`. | +| TIMEZONE_UPDATES | 54464 | TimezoneUpdate | Adds the `TimezoneUpdate` server packet (type 17). Body: single `String` carrying the new session timezone. Sent when `SET session_timezone` mutates the session-default tz mid-query. | +| SPARSE_SERIALIZATION | 54465 | Block (Column) | Server may set `has_custom_serialization = 1` and emit a sparse-encoded column. Wire format: 1-byte kind (0x01 = SPARSE), then VarUInt offset stream terminated by EOG, then the non-default values densely encoded in the inner type. See [kind_stack and sparse encoding](/interfaces/specs/NativeFormat#kind-stack-and-sparse-encoding). | +| SSH_AUTHENTICATION | 54466 | Auth flow | Adds SSH challenge-response authentication. Opt-in: client sends a `user` of the form `" SSH KEY AUTHENTICATION " + ` with empty password to trigger it. See [SSH challenge-response authentication](#ssh-authentication). | +| TABLE_READ_ONLY_CHECK | 54467 | TablesStatusResponse | Adds an `is_readonly` flag to each table's row in TablesStatusResponse. External clients that don't issue `TablesStatusRequest` see no wire change. | +| SYSTEM_KEYWORDS_TABLE | 54468 | system tables | Server populates `system.keywords` so the canonical `clickhouse-client` can autocomplete keywords. No native-protocol wire change. | +| ROWS_BEFORE_AGGREGATION | 54469 | ProfileInfo | Adds `applied_aggregation` (Bool) and `rows_before_aggregation` (VarUInt) to ProfileInfo, in that order at the tail. | +| CHUNKED_PROTOCOL | 54470 | Connection framing | Per-packet chunk framing wraps every packet body. Negotiated in Addendum. ServerHello carries the server's preference for each direction; Addendum carries the client's final choice. See [chunked framing](#chunked-framing). | +| VERSIONED_PARALLEL_REPLICAS_PROTOCOL | 54471 | ServerHello, Addendum | Both sides exchange a `VarUInt` parallel-replicas coordination protocol version. ServerHello's field is positioned **immediately after `protocol_version`** (before `timezone`). Addendum's field is appended after the chunked-protocol strings. Current value: `7` (`DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION`). | +| INTERSERVER_EXTERNALLY_GRANTED_ROLES | 54472 | Query | Adds a `String external_roles` field to the Query body, between the settings terminator and the interserver-secret hash. External clients send an empty role list (a single byte `0x00`, i.e. VarUInt 0 inside a String envelope). | +| V2_DYNAMIC_AND_JSON_SERIALIZATION | 54473 | Column body | Server may emit V2 serialization for `Dynamic` and `JSON` column types — gates which `state_prefix` version they use. See [versioned types](/interfaces/specs/NativeFormat#versioned-types). | +| SERVER_SETTINGS | 54474 | ServerHello | Server broadcasts its non-default settings as a list at the tail of ServerHello, after `nonce`. Format: `(key, flags, value)` triples terminated by an empty key — same as the Query packet's settings list. | +| QUERY_AND_LINE_NUMBERS | 54475 | ClientInfo | Adds `script_query_number` (VarUInt) and `script_line_number` (VarUInt) at the tail of ClientInfo. Used by clickhouse-client for multi-statement script error attribution; external clients send `0, 0`. | +| JWT_IN_INTERSERVER | 54476 | ClientInfo | Adds a JWT-presence UInt8 + optional `String jwt` at the tail of ClientInfo. External clients (no JWT) send byte `0x00`. (Spelled `DBMS_MIN_REVISON_WITH_JWT_IN_INTERSERVER` in C++ — note the typo in the constant name.) | +| QUERY_PLAN_SERIALIZATION | 54477 | ServerHello, QueryPlan packet | ServerHello appends `VarUInt query_plan_serialization_version` after server settings. Also introduces `ClientPacket::QueryPlan` (code `13`) for inter-server delivery of pre-built query plans — external clients never send. | +| PARALLEL_BLOCK_MARSHALLING | 54478 | Block (Column) | Server may wrap columns in `ColumnBLOB` (compressed inline) for parallel processing. Gated on the query having compression enabled AND `rows > 1`; otherwise the regular column wire format applies. Clients that never enable compression on outgoing Query packets see no wire change. | +| VERSIONED_CLUSTER_FUNCTION_PROTOCOL | 54479 | ServerHello | Adds `VarUInt cluster_function_protocol_version` at the tail of ServerHello. Used for `*Cluster` table functions (`s3Cluster`, etc.). External clients decode and ignore. | +| OUT_OF_ORDER_BUCKETS_IN_AGGREGATION | 54480 | BlockInfo | Adds field 3 (`out_of_order_buckets: Vec`) to BlockInfo's field-tagged stream. Decoded as `[VarUInt count][Int32]*count`. External clients don't emit this themselves; the decoder reads any non-empty list the server sends. | +| COMPRESSED_LOGS_PROFILE_EVENTS_COLUMNS | 54481 | Log, ProfileEvents | Server may wrap [`Log`](#log) and [`ProfileEvents`](#profileevents) packet bodies in the [compression frame](/interfaces/specs/NativeFormat#compression-frame). The wrap only activates when the query has `compression = true`. Clients that never enable compression on outgoing Query packets see no wire change. | +| REPLICATED_SERIALIZATION | 54482 | Block (Column) | Server may emit columns with kind_stack `0x04 = REPLICATED` — a dictionary-style compact form for repeated values — see [kind_stack and sparse encoding](/interfaces/specs/NativeFormat#kind-stack-and-sparse-encoding). Below this version the writer expanded such columns before sending. Decoded via index lookup (`elements[indexes[i]]` per row); leaf types plus `Nullable`/`Array`/`Tuple`/`Map`/`Nested`/`LowCardinality` inners supported. | +| NULLABLE_SPARSE_SERIALIZATION | 54483 | Block (Column) | Composes sparse serialization with `Nullable(T)`. Below this version the writer expanded sparse for Nullable columns before sending; at v54483+ the wire data is sparse-over-Nullable. See [kind_stack and sparse encoding](/interfaces/specs/NativeFormat#kind-stack-and-sparse-encoding). | +| PROGRESS_IN_ASYNC_INSERT | 54484 | Progress (INSERT) | On an **asynchronous** INSERT (`async_insert = 1`), once the insert is flushed the server sends an extra [`Progress`](#progress) packet, then the insert's `ProfileEvents`, before `EndOfStream`. Gated on the *negotiated* version ≥ 54484; below it the server omits this trailing Progress. The Progress wire format is unchanged — only the emission is new. In practice the increment carries the elapsed time; the written-row counters are reported via the accompanying ProfileEvents. A client that already drains interleaved Progress needs no format change, only to tolerate one more packet. | + +## Packet envelope {#packet-envelope} + +Every message on the wire shares the same outer structure, in both directions: + +```text +[VarUInt: packet_type_code] always encoded as VarUInt +[message body] format depends on packet_type_code +``` + +The full packet-type tables are in the [packet type reference](#packet-type-reference). + +The packet type is a `VarUInt`, not a fixed-width byte. For values below 128 a `VarUInt` produces the same single byte, but implementations must use `VarUInt` encoding so they stay compatible should future packet types reach 128 or above. + +The [message reference](#message-reference) documents only the **body** of each packet — the bytes after the packet type code. Field numbering starts at 1 with the first body field. + +### Chunked framing (v54470+) {#chunked-framing} + +When the `CHUNKED_PROTOCOL` feature is **negotiated** (see [the handshake](#handshake-phase)), every packet on the wire is wrapped in chunked framing. The wrapping is **per-direction**: client→server and server→client are negotiated separately and may end up in different modes (chunked versus unframed). + +Wire layout per packet: + +```text +... one or more chunks comprising the packet's body +[u32 LE = 0] zero-size terminator marking end of packet +``` + +Wire layout per chunk: + +```text +[u32 LE: chunk_size] chunk_size in [1, UINT32_MAX] +[chunk_size bytes] packet body bytes +``` + +A single packet may be split across several chunks if the writer's buffer fills mid-packet. The reader treats the trailing 4-byte zero as a transparent packet boundary — it consumes it but does not surface it to whatever is reading packet bodies. + +**Negotiation.** ServerHello and Addendum each carry two `String` fields, one per direction, with values drawn from `{"chunked", "notchunked", "chunked_optional", "notchunked_optional"}`: + +- `chunked` / `notchunked` are strict: that side requires exactly that mode. +- The `_optional` variants are flexible: they accept whichever mode the other side picks. + +The agreed value for each direction is computed pairwise (mirroring `Client/Connection.cpp::connect::is_chunked`): + +| Server pref | Client pref | Agreed | +|-------------|-------------|--------| +| `*_optional` | anything | follow CLIENT (its `starts_with("chunked")`) | +| anything | `*_optional` | follow SERVER | +| `chunked` strict | `chunked` strict | `chunked` | +| `notchunked` strict | `notchunked` strict | `notchunked` | +| strict mismatch | strict mismatch | **protocol error** — the connection MUST be torn down | + +On the client side, the client's SEND preference negotiates against the server's RECV preference, and vice versa. + +**Timing.** The negotiation strings travel on the unframed wire: ClientHello → ServerHello (server prefs) → Addendum (client's negotiated values). The framing flip applies to every byte sent *after* the Addendum is flushed. The Addendum itself, the ClientHello, and the ServerHello are always unframed. + +## Connection lifecycle {#connection-lifecycle} + +At any moment a connection is in exactly one of four states: `HANDSHAKE`, `READY`, `READING_RESPONSE`, or terminated. Because the protocol does not multiplex, a client that sends a new request before draining the previous response interleaves bytes on the wire and corrupts the stream. + +### States {#states} + +```text + [Connect] + | + v + HANDSHAKE ---- error ----> [Disconnect] + | + ok + | + v + READY <----------------------------------+ + | | + |--- Ping -------> Pong ----------->--| + | | + |--- Query ------> READING_RESPONSE ->| + | | + +-------------------------------------+ +``` + +| State | Description | +|--------------------|-------------| +| `HANDSHAKE` | Initial state after the TCP connection opens. Only [handshake](#handshake-phase) messages are valid. Transitions to `READY` on success or terminates on failure. | +| `READY` | Idle. The client may send [Ping](#ping-phase), [Query](#query-phase), or close. The connection may stay in `READY` indefinitely (subject to `idle_connection_timeout`, see [connection limits](#connection-limits)). | +| `READING_RESPONSE` | Entered when the client sends a Query. The client must fully drain the server's response stream before returning to `READY`. The only allowed client→server packet here is Cancel (not specified on this page). | +| Terminated | No longer usable. The client must open a new TCP connection and restart the handshake. | + +### Handshake phase {#handshake-phase} + +Authenticates and negotiates the protocol version. Happens exactly once per connection, before anything else. + +The TCP connection has just opened and no messages have been exchanged. The flow: + +```text +Client Server + |--- ClientHello ------------------>| + |<--- ServerHello ------------------| or Exception + | | + | (compute negotiated_version) | + | | + |--- Addendum --------------------->| only if negotiated_version ≥ 54458 +``` + +1. The client sends [`ClientHello`](#clienthello) with its maximum supported protocol version. +2. The client reads the response and dispatches by packet type: + + | Packet type | Action | + |-----------------------|--------| + | `Hello` (0) | Decode [`ServerHello`](#serverhello). Compute `negotiated_version = min(client_ver, server_ver)`. Proceed to step 3. | + | `Exception` (2) | Decode [`Exception`](#exception). Return as error and terminate the connection. | + | anything else | Protocol violation. Terminate the connection. | + +3. If `negotiated_version ≥ 54458` (the `ADDENDUM` feature), the client sends an [`Addendum`](#addendum). This decision is based on the **negotiated** version, not the client's declared version. + +On success the connection moves to `READY`; on any error it terminates. + +### Ping phase {#ping-phase} + +An application-level liveness check, independent of TCP keepalive. A successful Ping/Pong round-trip confirms the TCP connection is alive in both directions and the server is responsive. Ping is stateless and uncorrelated with any query, so multiple sequential Pings are independent. + +Starting from `READY`, the flow is: + +```text +Client Server + |--- Ping (0x04) ------------------>| + |<--- Pong (0x04) ------------------| +``` + +1. The client sends [`Ping`](#ping). +2. The client reads the response: + + | Packet type | Action | + |-----------------------|--------| + | `Pong` (4) | Liveness confirmed. Return to `READY`. | + | `Exception` (2) | Decode [`Exception`](#exception) and return as error. | + | anything else | Protocol violation. | + +### Query phase {#query-phase} + +The client submits a SQL statement; the server streams back result blocks and execution telemetry. The response is a sequence of packets terminated by exactly one `EndOfStream` or `Exception`. + +Starting from `READY`, the flow is: + +```text +Client Server + |--- Query ------------------------>| the Query message + |--- ExternalTable (data) --------->| optional, for temp tables + |--- Empty Data marker ------------>| required, end-of-client-data + | | + |<--- Data (header block) ----------| schema: N cols, 0 rows + |<--- Progress ---------------------| 0 or more, interleaved + |<--- Log --------------------------| 0 or more (if logs enabled) + |<--- Data (result block) ----------| 0 or more: N cols, M rows each + |<--- Totals / Extremes ------------| 0 or more (aggregation queries) + |<--- ProfileInfo / ProfileEvents --| 0 or more (profiling) + |<--- Data (empty block) -----------| boundary marker + |<--- Progress ---------------------| final updates + |<--- EndOfStream ------------------| authoritative end of query +``` + +On error at any point the server sends an `Exception`, which terminates the query. + +1. The client sends [`Query`](#query) with a unique `query_id` (typically a UUID). +2. The client sends any external tables, then the empty Data marker. The empty Data packet has `table_name = ""`, `num_columns = 0`, `num_rows = 0`. The server does not begin executing the query until it receives this marker. +3. The client moves to `READING_RESPONSE` and flushes its write buffer. +4. The client reads response packets in a loop, dispatching by type: + + | Packet type | Action | + |-----------------------|--------| + | `Data` (1) | Decode the block. The first Data is the schema header; later ones are result blocks (accumulate); an empty block is a boundary marker. `num_rows == 0` is **not** end-of-query. | + | `Progress` (3) | Execution metrics. Cumulative, not deltas. | + | `EndOfStream` (5) | Query complete. Exit the loop and return to `READY`. | + | `ProfileInfo` (6) | Post-execution profiling data. | + | `Totals` (7) | Aggregation totals block (same wire format as Data). | + | `Extremes` (8) | Min/max values block (same wire format as Data). | + | `Log` (10) | Server log line. | + | `TableColumns` (11) | Column-defaults metadata. | + | `ProfileEvents` (14) | Performance counters. | + | `Exception` (2) | Decode and return as error. Exit the loop and return to `READY`. | + | anything else | Unexpected during the query phase. Terminate the connection. | + +On `EndOfStream` or a handled `Exception` the connection returns to `READY`. A protocol violation or I/O error terminates it. + +:::note +The `num_rows == 0` case trips up new implementations. A zero-row block is a boundary marker or schema header, not an end-of-stream signal. Only `EndOfStream` or `Exception` ends the response. +::: + +### INSERT phase {#insert-phase} + +The INSERT phase is the [Query phase](#query-phase) with two extra exchanges. The client submits an `INSERT` statement; the server replies with a **schema block** describing the target table; the client streams Data packets with the rows, then the empty Data marker; the server finishes with `EndOfStream` or `Exception`. + +Starting from `READY`, the SQL is an `INSERT` of the form `INSERT INTO [()] VALUES` — with no inline `VALUES (...)` literal, since the row data flows through Data packets. The flow: + +```text +Client Server +[Query packet — INSERT body] → +[ExternalTable*, then empty terminator] → + ← [optional metadata: TableColumns, Progress, ...] + ← [Data packet: schema block (rows = 0)] +[Data packet: rows N] → +[Data packet: rows M] → (additional blocks, optional) +[Data packet: empty block (rows 0)] → (end-of-input terminator) + ← [optional Progress, ProfileInfo, Log, ProfileEvents] + ← [EndOfStream] +``` + +1. The client sends [`Query`](#query) with `body` set to the INSERT SQL. +2. The client sends any external tables (rare for INSERT) followed by the empty terminator. +3. The client drains metadata packets (TableColumns, Progress, ProfileInfo, Log, ProfileEvents) until it reads the schema Data packet — a Block with 0 rows but full column structure (names and types). The schema block is the contract: the rows the client sends next must match these column shapes. +4. The client sends data block(s). For each block it writes `VarUInt(ClientPacket::Data = 2)`, then `String("")` for the empty external-table name, then the Block. Column types must align with the schema block's columns by position. +5. The client sends the end-of-input terminator: a Data packet with an empty Block (0 columns, 0 rows). +6. The client drains the response stream until `EndOfStream` (success) or `Exception` (failure). + +**Asynchronous INSERT (v54484+).** When the query carries `async_insert = 1`, the server queues the rows and flushes them as part of a batch. At negotiated version ≥ 54484 (`PROGRESS_IN_ASYNC_INSERT`), once the flush completes the server emits an extra [`Progress`](#progress) packet, immediately followed by the insert's `ProfileEvents`, then `EndOfStream`. Below 54484 the server skips that trailing Progress. The packet is an ordinary `Progress`; because the server resets the query pipeline before folding in the write counts, the increment in practice carries only the elapsed time, and the written-row and byte stats reach the client via the accompanying `ProfileEvents`. A client that already drains interleaved Progress in step 6 needs only to accept one more packet. + +The connection returns to `READY` on `EndOfStream` or a handled `Exception`. Protocol violations and I/O errors terminate it. + +## Message reference {#message-reference} + +Fields are listed in wire order. The `Type` column uses: + +- `VarUInt` — variable-length unsigned integer (see [VarUInt](/interfaces/specs/NativeFormat#varuint)). +- `String` — VarUInt-prefixed bytes (see [String](/interfaces/specs/NativeFormat#string)). +- `UInt8`, `Int32`, and so on — fixed-width little-endian integers. +- `Bool` — a single byte, `0x00` or `0x01`. + +The `Role` column says who uses each field: + +- **client** — set by external clients. +- **inter-server** — meaningful only for server-to-server communication; external clients write a default value. +- **universal** — used by both. + +These tables document only the body of each packet, after the packet type code. + +### ClientHello (packet type 0) {#clienthello} + +Client → Server. The first message after the TCP connection opens. + +| # | Field | Type | Role | Description | +|---|------------------|---------|-----------|-------------| +| 1 | client_name | String | universal | Client identifier (e.g., `"clickhouse-client"`) | +| 2 | version_major | VarUInt | universal | Client major version | +| 3 | version_minor | VarUInt | universal | Client minor version | +| 4 | protocol_version | VarUInt | universal | Client's max supported protocol version | +| 5 | database | String | universal | Default database name | +| 6 | user | String | universal | Username for authentication | +| 7 | password | String | universal | Password (plaintext) | + +### ServerHello (packet type 0) {#serverhello} + +Server → Client. The reply to ClientHello on successful authentication. + +| # | Field | Type | Role | Condition | Description | +|---|------------------|---------|-----------|------------------------|-------------| +| 1 | server_name | String | universal | always | Server identifier | +| 2 | version_major | VarUInt | universal | always | Server major version | +| 3 | version_minor | VarUInt | universal | always | Server minor version | +| 4 | protocol_version | VarUInt | universal | always | Server's protocol version | +| 4a | parallel_replicas_protocol_version | VarUInt | universal | VERSIONED_PARALLEL_REPLICAS_PROTOCOL (v54471) | Server's parallel-replicas coordination protocol version. **Wire position: immediately after `protocol_version`**, before `timezone`. Current: `7`. | +| 5 | timezone | String | universal | TIMEZONE (v54058) | Server timezone (e.g., `"UTC"`) | +| 6 | display_name | String | universal | DISPLAY_NAME (v54372) | Human-readable server name | +| 7 | version_patch | VarUInt | universal | VERSION_PATCH (v54401) | Server patch version | +| 8 | proto_send_chunked_srv | String | universal | CHUNKED_PROTOCOL (v54470) | Server's preferred outbound chunking. One of `"chunked"`, `"notchunked"`, `"chunked_optional"`, `"notchunked_optional"`. See [chunked framing](#chunked-framing). **Sits BEFORE `password_complexity_rules` on the wire even though its version gate is higher.** | +| 9 | proto_recv_chunked_srv | String | universal | CHUNKED_PROTOCOL (v54470) | Server's preferred inbound chunking. Same value set as field 8. | +| 10 | password_complexity_rules | Rule[] | universal | PASSWORD_COMPLEXITY_RULES (v54461) | Server's password policy. `VarUInt count` followed by `count × Rule`. See below. | +| 11 | nonce | UInt64 | inter-server | INTERSERVER_SECRET_V2 (v54462) | 8-byte LE random nonce. The server's inter-server query-signing scheme uses it. External clients MUST decode it (to keep the stream aligned) and SHOULD ignore the value. | +| 12 | server_settings | Setting[] | universal | SERVER_SETTINGS (v54474) | Server's non-default settings broadcast. Format: zero or more `(String key, VarUInt flags, String value)` triples, terminated by an empty key. Same as the [Query packet's settings list](#setting). | +| 13 | query_plan_serialization_version | VarUInt | universal | QUERY_PLAN_SERIALIZATION (v54477) | Server's supported query-plan serialization version. External clients decode and ignore. | +| 14 | cluster_function_protocol_version | VarUInt | universal | VERSIONED_CLUSTER_FUNCTION_PROTOCOL (v54479) | Server's `*Cluster` table-function protocol version. External clients decode and ignore. | + +**Rule** — an element of `password_complexity_rules`: + +| # | Field | Type | Description | +|---|---------|--------|-------------| +| 1 | pattern | String | Regular-expression pattern that a compliant password must match. | +| 2 | message | String | Human-readable explanation shown when a password fails this rule. | + +The list reflects the server operator's password-policy configuration and is purely advisory — the server does not enforce these rules during the handshake. A client that exposes password change/set functionality may use the rules to flag errors before round-tripping a non-compliant password to the server. + +:::note +To bound resource use against a hostile or misconfigured server, cap the decoded `count` at 256 entries and each `pattern` and `message` String at 4096 bytes. A `count` of `0` (no following pairs) is the common case for servers with no password policy configured. +::: + +### Addendum (no packet type) {#addendum} + +Client → Server, gated by `ADDENDUM` (v54458). Sent immediately after the handshake exchange completes. It is not a distinct packet type — the fields go on the wire raw, with no packet type byte prefix. + +| # | Field | Type | Role | Condition | Description | +|---|-------------------|--------|--------------|----------------------------|-------------| +| 1 | quota_key | String | inter-server | always | Resource quota identifier. External clients send empty string. | +| 2 | proto_send_chunked | String | universal | CHUNKED_PROTOCOL (v54470) | Client's negotiated outbound chunking: `"chunked"` or `"notchunked"`. Computed against `proto_recv_chunked_srv` from ServerHello. | +| 3 | proto_recv_chunked | String | universal | CHUNKED_PROTOCOL (v54470) | Client's negotiated inbound chunking. Computed against `proto_send_chunked_srv`. | +| 4 | parallel_replicas_protocol_version | VarUInt | universal | VERSIONED_PARALLEL_REPLICAS_PROTOCOL (v54471) | Client's supported parallel-replicas coordination protocol version. External clients not participating in distributed queries SHOULD still send a valid version (current `7`) so the server's compatibility check succeeds. | + +The chunked-framing flip applies *after* this Addendum is flushed — the Addendum itself is unframed. + +### Ping (packet type 4) {#ping} + +Client → Server. No body — the packet is a single byte `0x04` on the wire. + +### Pong (packet type 4) {#pong} + +Server → Client. No body — the packet is a single byte `0x04` on the wire. + +### Exception (packet type 2) {#exception} + +Server → Client. Sent when the server hits an error during any phase. + +| # | Field | Type | Role | Description | +|---|-------------|--------|-----------|-------------| +| 1 | code | Int32 | universal | Error code | +| 2 | name | String | universal | Exception class (e.g., `"DB::Exception"`) | +| 3 | message | String | universal | Human-readable error message | +| 4 | stack_trace | String | universal | Server-side stack trace | +| 5 | has_nested | Bool | universal | If true, another Exception follows immediately | + +When `has_nested` is true, another Exception structure follows (without a packet type prefix), forming a chain of nested exceptions. + +### Query (packet type 1) {#query} + +Client → Server. + +| # | Field | Type | Role | Condition | Description | +|---|----------------|-------------|--------------|------------------------------------------|-------------| +| 1 | query_id | String | universal | always | Unique query identifier (UUID) | +| 2 | client_info | ClientInfo | universal | WRITE_CLIENT_INFO (v54420) | See [ClientInfo](#clientinfo) | +| 3 | settings | Setting[] | universal | SETTINGS_SERIALIZED_AS_STRINGS (v54429) | See [Setting](#setting). Terminated by empty key. | +| 3a | external_roles | String | universal | INTERSERVER_EXTERNALLY_GRANTED_ROLES (v54472) | Serialized list of externally-granted role names. Empty list = byte `0x00` (VarUInt 0) wrapped in a String envelope (`[VarUInt 1][0x00]` on the wire). External clients always send empty. | +| 4 | cluster_secret | String | inter-server | INTERSERVER_SECRET (v54441) | Cluster auth. External clients send empty string. | +| 5 | stage | VarUInt | universal | always | 0 = FetchColumns, 1 = WithMergeableState, 2 = Complete | +| 6 | compression | VarUInt | universal | always | 0 = disabled, 1 = enabled | +| 7 | query_body | String | universal | always | SQL text | +| 8 | parameters | Parameter[] | client | PARAMETERS (v54459) | See [Parameter](#parameter). Terminated by empty key. | + +### ClientInfo (embedded in Query) {#clientinfo} + +Client → Server, embedded in the Query body (field 2). Gated by `WRITE_CLIENT_INFO` (v54420). + +| # | Field | Type | Role | Condition | Description | +|----|------------------------------|---------|--------------|----------------------------------------|-------------| +| 1 | query_kind | UInt8 | universal | always | 0 = NoQuery, 1 = InitialQuery, 2 = SecondaryQuery. External clients send `1`. | +| 2 | initial_user | String | universal | always | User who initiated the query | +| 3 | initial_query_id | String | universal | always | Original query ID | +| 4 | initial_address | String | universal | always | Originating client socket address in `host:port` format | +| 5 | initial_time | Int64 | client | INITIAL_QUERY_START_TIME (v54449) | Query start time (microseconds). Fixed-width 8 bytes, not VarUInt | +| 6 | query_interface | UInt8 | universal | always | 1 = TCP, 2 = HTTP | +| 7 | os_user | String | client | if interface = TCP | OS username | +| 8 | client_hostname | String | client | if interface = TCP | Client machine hostname | +| 9 | client_name | String | client | if interface = TCP | Client application name | +| 10 | version_major | VarUInt | universal | if interface = TCP | Client major version | +| 11 | version_minor | VarUInt | universal | if interface = TCP | Client minor version | +| 12 | protocol_version | VarUInt | universal | if interface = TCP | Negotiated protocol version | +| 13 | quota_key | String | inter-server | QUOTA_KEY_IN_CLIENT_INFO (v54060) | Resource quota key. External clients send empty string. | +| 14 | distributed_depth | VarUInt | inter-server | DISTRIBUTED_DEPTH (v54448) | Distributed query nesting depth. External clients send `0`. | +| 15 | version_patch | VarUInt | universal | VERSION_PATCH (v54401), TCP only | Client patch version | +| 16 | open_telemetry | (below) | client | OPEN_TELEMETRY (v54442) | Trace context. Clients without tracing send `0`. | +| 17 | collaborate_with_initiator | VarUInt | inter-server | PARALLEL_REPLICAS (v54453) | Bool as VarUInt. External clients send `0`. | +| 18 | count_participating_replicas | VarUInt | inter-server | PARALLEL_REPLICAS (v54453) | External clients send `0`. | +| 19 | number_of_current_replica | VarUInt | inter-server | PARALLEL_REPLICAS (v54453) | External clients send `0`. | +| 20 | script_query_number | VarUInt | client | QUERY_AND_LINE_NUMBERS (v54475) | 1-indexed statement position in a multi-statement script. External clients send `0`. | +| 21 | script_line_number | VarUInt | client | QUERY_AND_LINE_NUMBERS (v54475) | 1-indexed line number within the source script. External clients send `0`. | +| 22 | jwt_present | UInt8 | inter-server | JWT_IN_INTERSERVER (v54476) | `0` = no JWT; `1` = JWT follows. External clients without JWT auth send `0`. | +| 23 | jwt | String | inter-server | JWT_IN_INTERSERVER (v54476), if jwt_present=1 | JWT bearer token, only present when field 22 = `1`. | + +OpenTelemetry encoding (field 16): + +```text +[UInt8: has_trace] 0 = no trace data follows, 1 = trace data follows +If has_trace == 1: + [16 bytes: trace_id] byte-swapped per-8-bytes + [8 bytes: span_id] byte-swapped + [String: trace_state] W3C trace state + [UInt8: trace_flags] W3C trace flags +``` + +### Setting {#setting} + +Encoded inline in the Query body's settings list (the [Query](#query) packet, field 3). The list is terminated by a Setting with an empty key — a single `VarUInt 0`, with no flags or value following. + +| # | Field | Type | Role | Description | +|---|-------|---------|-----------|-------------| +| 1 | key | String | universal | Setting name. Empty = end of list. | +| 2 | flags | VarUInt | universal | Bit flags: `0x01` = Important, `0x02` = Custom, `0x04` = Obsolete | +| 3 | value | String | universal | Setting value as string | + +Fields 2 and 3 are absent when `key` is empty. + +### Parameter {#parameter} + +Query parameters, for parameterized queries like `SELECT {x:UInt64}`. Encoded identically to a [Setting](#setting) with the `Custom` flag (`0x02`) set, and terminated by an empty key in the same way. + +| # | Field | Type | Role | Description | +|---|-------|---------|--------|-------------| +| 1 | key | String | client | Parameter name. Empty = end of list. | +| 2 | flags | VarUInt | client | Always `0x02` (Custom) | +| 3 | value | String | client | Parameter value as string. See the note below on quoting. | + +:::note +The parameter value is the SQL representation of the value, not a raw literal. String-typed parameters must be passed already single-quoted (for example, the value for `{name:String}` is `'Alice'`, not `Alice`); otherwise the server's value parser rejects them. +::: + +### Data (packet type 1 server→client, packet type 2 client→server) {#data} + +Both directions. Carries result blocks, INSERT data, external tables, and end-of-data markers. + +The wire format is symmetric — both directions include a `table_name` prefix before the Block. Only the packet type byte differs. + +```text +[VarUInt: packet_type] 1 (server→client) or 2 (client→server) +[String: table_name] External table name; empty in most cases +[Block] See the Native Format spec for the Block layout +``` + +| Field | Type | Role | Description | +|------------|--------|-----------|-------------| +| table_name | String | universal | External table name. Client: empty = end-of-data marker. Server: always empty for query results. | +| Block body | — | — | See [Block & column structure](/interfaces/specs/NativeFormat#block-and-column-structure). | + +The block variants and what they mean are documented under [Block variants](/interfaces/specs/NativeFormat#block-variants). + +### Progress (packet type 3) {#progress} + +Server → Client. Sent periodically during query execution. All fields are VarUInt, and each packet carries cumulative totals (not deltas) since the start of the query. + +| # | Field | Type | Role | Condition | Description | +|---|-------------|---------|-----------|----------------------------------------|-------------| +| 1 | rows | VarUInt | universal | always | Rows processed so far | +| 2 | bytes | VarUInt | universal | always | Bytes processed so far | +| 3 | total_rows | VarUInt | universal | always | Estimated total rows (may be 0) | +| 4 | total_bytes | VarUInt | universal | TOTAL_BYTES_IN_PROGRESS (v54463) | Estimated total bytes (may be 0). Sits BETWEEN `total_rows` and `wrote_rows` on the wire. | +| 5 | wrote_rows | VarUInt | universal | WRITE_CLIENT_INFO (v54420) | Rows written (for INSERT) | +| 6 | wrote_bytes | VarUInt | universal | WRITE_CLIENT_INFO (v54420) | Bytes written (for INSERT) | +| 7 | elapsed_ns | VarUInt | universal | SERVER_QUERY_TIME_IN_PROGRESS (v54460) | Elapsed nanoseconds since query start | + +### ProfileInfo (packet type 6) {#profileinfo} + +Server → Client. Sent once per query, near the end of execution. + +| # | Field | Type | Role | Condition | Description | +|---|-------------------------------|---------|-----------|------------------------------------|-------------| +| 1 | rows | VarUInt | universal | always | Total rows processed | +| 2 | blocks | VarUInt | universal | always | Total blocks processed | +| 3 | bytes | VarUInt | universal | always | Total bytes processed | +| 4 | applied_limit | Bool | universal | always | Whether a LIMIT clause was applied | +| 5 | rows_before_limit | VarUInt | universal | always | Row count before LIMIT | +| 6 | calculated_rows_before_limit | Bool | universal | always | Whether `rows_before_limit` was computed | +| 7 | applied_aggregation | Bool | universal | ROWS_BEFORE_AGGREGATION (v54469) | Whether GROUP BY was applied | +| 8 | rows_before_aggregation | VarUInt | universal | ROWS_BEFORE_AGGREGATION (v54469) | Row count before aggregation | + +### Totals (packet type 7) {#totals} + +Server → Client. Sent for queries with `WITH TOTALS`. Wire format is identical to [Data](#data): a `table_name` string (always empty) followed by a Block. Only the packet type byte differs. + +```text +[VarUInt: 7] packet type +[String: table_name] always empty +[Block] see the Native Format spec +``` + +### Extremes (packet type 8) {#extremes} + +Server → Client. Sent when the `extremes` setting is enabled. Wire format is identical to [Data](#data). The block has exactly 2 rows: row 0 holds the minimum of each column, row 1 holds the maximum. + +```text +[VarUInt: 8] packet type +[String: table_name] always empty +[Block] num_rows = 2 +``` + +### Log (packet type 10) {#log} + +Server → Client. Sent when the query has an active logs queue (the `send_logs_level` setting; see [log streaming](#log-streaming)). + +Same envelope and body format as [Data](#data). The block has a fixed `num_columns = 8` and a predefined schema. Each log line is one row across all 8 columns, and a single Log packet may carry many rows. + +```text +[VarUInt: 10] packet type +[String: table_name] always empty +[Block] num_columns = 8, num_rows = number of log lines +``` + +The 8 columns, in this exact order: + +| # | Name | Type | Description | +|---|-------------------------|----------|-------------| +| 1 | event_time | DateTime | Event timestamp (seconds since epoch) | +| 2 | event_time_microseconds | UInt32 | Microseconds component | +| 3 | host_name | String | Server hostname emitting the log | +| 4 | query_id | String | Query ID the log belongs to | +| 5 | thread_id | UInt64 | OS thread ID | +| 6 | priority | Int8 | Log level (Poco priority: 1 = Fatal, … 8 = Trace) | +| 7 | source | String | Logger name | +| 8 | text | String | Log message text | + +### ProfileEvents (packet type 14) {#profileevents} + +Server → Client. Carries per-query performance counters. + +Same envelope and body format as [Data](#data). The block has a fixed `num_columns = 6` and a predefined schema. Each event is one row. + +```text +[VarUInt: 14] packet type +[String: table_name] always empty +[Block] num_columns = 6, num_rows = number of events +``` + +The 6 columns: + +| # | Name | Type | Description | +|---|--------------|----------|-------------| +| 1 | host_name | String | Server hostname | +| 2 | current_time | DateTime | Event timestamp | +| 3 | thread_id | UInt64 | Thread ID | +| 4 | type | Int8 | Event type: 1 = Increment (counter), 2 = Gauge | +| 5 | name | String | Event name (e.g., `"Query"`, `"NetworkReceiveBytes"`) | +| 6 | value | Int64 or UInt64 | Counter value or gauge reading | + +:::note +The `value` column's element type is not fixed across packets — older servers emit `UInt64`, newer ones `Int64`. Read the column's type string from the block header rather than assuming one width. +::: + +### TableColumns (packet type 11) {#tablecolumns} + +Server → Client. Sent when the client needs column-default metadata, typically before an INSERT that omits some columns. + +| # | Field | Type | Role | Description | +|---|---------------------|--------|-----------|-------------| +| 1 | external_table | String | universal | External table name. Empty = main table. | +| 2 | columns_description | String | universal | Textual column definitions, e.g., `"id Int32, name String DEFAULT ''"`. Free-form text — parse as a string. | + +### TimezoneUpdate (packet type 17) {#timezoneupdate} + +Server → Client, gated by `TIMEZONE_UPDATES` (v54464). Sent when the session-default timezone changes mid-query — for example, a `SET session_timezone = '...'` runs as part of the query and the server wants the client to use the new default when formatting subsequent `DateTime` values. + +| # | Field | Type | Role | Description | +|---|----------|--------|-----------|-------------| +| 1 | timezone | String | universal | The new session-default timezone (e.g., `"UTC"`, `"Europe/Berlin"`). | + +The packet may arrive at any point in the query response stream, between Data, Progress, or Log packets. A decoder that ignores `TimezoneUpdate` MUST still consume the trailing `String` to keep the wire aligned. + +### SSH challenge-response authentication (packet types 11, 12, 18) {#ssh-authentication} + +Gated by `SSH_AUTHENTICATION` (v54466), and opt-in only. A connection enters the SSH flow when ClientHello sends `user = " SSH KEY AUTHENTICATION " + ` (with the leading and trailing spaces) and `password = ""`. The server reads the prefix, strips it to recover the real user, and switches to challenge-response. + +| Packet | Code | Direction | Body | +|--------|------|-----------|------| +| SSHChallengeRequest | 11 | Client → Server | (no body) | +| SSHChallenge | 18 | Server → Client | `String challenge` — the bytes to sign | +| SSHChallengeResponse | 12 | Client → Server | `String signature` — the SSH-signed challenge | + +The flow runs in place of password authentication, after ClientHello: + +1. The client sends ClientHello with the SSH marker prefix. +2. The server replies with ServerHello as usual. +3. The client sends `SSHChallengeRequest` (packet 11). +4. The server replies with `SSHChallenge` carrying random bytes (packet 18). +5. The client signs the bytes with its SSH private key and sends `SSHChallengeResponse` (packet 12) with the signature. +6. The server verifies the signature against the user's registered public key, then continues as if password auth had succeeded (or returns an Exception on failure). + +External clients that don't use SSH auth never see packets 11, 12, or 18 — they stay off the wire unless the user explicitly opts in via the username prefix. + +## Packet type reference {#packet-type-reference} + +### Client → Server {#client-to-server} + +| Code | Name | Body format | Description | +|------|---------------------------|---------------------|-------------| +| 0 | Hello | [ClientHello](#clienthello) | Handshake initiation | +| 1 | Query | [Query](#query) | Query execution request | +| 2 | Data | [Data](#data) | Data block (INSERT data, external tables, end-of-data marker) | +| 3 | Cancel | (no body) | Cancel running query | +| 4 | Ping | [Ping](#ping) | Liveness check | +| 5 | TablesStatusRequest | not specified | Table status check | +| 6 | KeepAlive | not specified | Connection keepalive | +| 7 | Scalar | not specified | Scalar data block | +| 8 | IgnoredPartUUIDs | not specified | Parts to exclude from query | +| 9 | ReadTaskResponse | not specified | S3 cluster read response | +| 10 | MergeTreeReadTaskResponse | not specified | Parallel read task response | +| 11 | SSHChallengeRequest | [SSH auth](#ssh-authentication) | SSH auth challenge request | +| 12 | SSHChallengeResponse | [SSH auth](#ssh-authentication) | SSH auth challenge response | +| 13 | QueryPlan | not specified | Query plan | + +### Server → Client {#server-to-client} + +| Code | Name | Body format | Description | +|------|--------------------------------|---------------------|-------------| +| 0 | Hello | [ServerHello](#serverhello) | Handshake response | +| 1 | Data | [Data](#data) | Result data block | +| 2 | Exception | [Exception](#exception) | Error | +| 3 | Progress | [Progress](#progress) | Query execution progress | +| 4 | Pong | [Pong](#pong) | Liveness response | +| 5 | EndOfStream | (no body) | Query complete | +| 6 | ProfileInfo | [ProfileInfo](#profileinfo) | Post-execution profiling data | +| 7 | Totals | [Totals](#totals) | GROUP BY WITH TOTALS row | +| 8 | Extremes | [Extremes](#extremes) | Min/max values (2-row block) | +| 9 | TablesStatusResponse | not specified | Table status response | +| 10 | Log | [Log](#log) | Query execution log lines | +| 11 | TableColumns | [TableColumns](#tablecolumns) | Column descriptions for defaults | +| 12 | PartUUIDs | not specified | Unique part IDs | +| 13 | ReadTaskRequest | not specified | Cluster read task request | +| 14 | ProfileEvents | [ProfileEvents](#profileevents) | Performance counters | +| 15 | MergeTreeAllRangesAnnouncement | not specified | Parallel read initialization | +| 16 | MergeTreeReadTaskRequest | not specified | Parallel read task assignment | +| 17 | TimezoneUpdate | [TimezoneUpdate](#timezoneupdate) | Server timezone update | +| 18 | SSHChallenge | [SSH auth](#ssh-authentication) | SSH auth challenge | + +## Configuration {#configuration} + +This section covers the tunables that shape native protocol connections: + +- [Transport-layer settings](#transport-layer-settings) — TCP socket options and timeouts, which affect how the TCP connection itself behaves. +- [Application-layer settings](#application-layer-settings) — per-query tunables carried in the [Query packet's settings list](#setting), which affect what the server sends on the wire or how it is framed. +- [Settings out of scope](#settings-out-of-scope) — settings often confused with protocol settings but which actually control SQL execution or storage. + +The defaults below reflect a recent server release; they may differ across versions and deployments. + +### Transport-layer settings {#transport-layer-settings} + +#### Socket options {#socket-options} + +| Option | Default | Side | Description | +|----------------------|----------------------------------|------------|-------------| +| `TCP_NODELAY` | on | both | Nagle's algorithm disabled. Small packets are sent immediately. | +| `SO_KEEPALIVE` | on (client), OS default (server) | asymmetric | Kernel-level TCP keepalive probes. Client explicitly enables this when `tcp_keep_alive_timeout > 0`. Server inherits the OS default. | +| `SO_RCVBUF` / `SO_SNDBUF` | OS defaults | — | Socket buffer sizes. Not tuned by the protocol. | + +#### Timeouts {#timeouts} + +| Setting | Default | Unit | Side | Description | +|------------------------------------------|---------|--------------|--------|-------------| +| `connect_timeout` | 10 | seconds | client | Timeout for establishing the initial TCP connection. | +| `handshake_timeout_ms` | 10000 | milliseconds | client | Timeout for receiving ServerHello during the handshake. | +| `send_timeout` | 300 | seconds | both | If no bytes can be written within this interval, the connection throws. | +| `receive_timeout` | 300 | seconds | both | If no bytes can be read within this interval, the connection throws. | +| `tcp_keep_alive_timeout` | 290 | seconds | client | Idle duration before the OS sends the first TCP keepalive probe. | +| `receive_data_timeout_ms` | 2000 | milliseconds | client | Timeout for receiving the first Data packet from a replica. | +| `connect_timeout_with_failover_ms` | 1000 | milliseconds | client | Per-attempt connect timeout when iterating replicas. | +| `connect_timeout_with_failover_secure_ms`| 1000 | milliseconds | client | Per-attempt connect timeout when iterating replicas over TLS. | +| `hedged_connection_timeout_ms` | 50 | milliseconds | client | Per-attempt connect timeout for hedged requests. | +| `poll_interval` | 10 | seconds | server | Granularity of the server's idle-connection and shutdown check loop. | + +The timeouts nest like this: + +```text +tcp_keep_alive_timeout (290s) + < receive_timeout (300s) + < idle_connection_timeout (3600s) + < tcp_close_connection_after_queries_seconds (0 = unlimited by default) +``` + +OS keepalive fires first and may detect dead peers silently at the kernel level. The application receive timeout is the next line of defence. The idle timeout is the last resort that reaps long-unused connections. + +#### Connection limits {#connection-limits} + +| Setting | Default | Unit | Side | Description | +|-----------------------------------------------|---------------|---------|--------|-------------| +| `max_connections` | 4096 | count | server | Maximum concurrent TCP connections. | +| `idle_connection_timeout` | 3600 | seconds | server | Maximum time an idle connection may remain open. | +| `tcp_close_connection_after_queries_num` | 0 (unlimited) | count | server | Maximum number of queries per connection before a forced close. | +| `tcp_close_connection_after_queries_seconds` | 0 (unlimited) | seconds | server | Maximum total connection lifetime regardless of activity. | + +A connection that issues queries regularly can live indefinitely. Only idle connections are reaped after an hour, and there is no default maximum lifetime. + +### Application-layer settings {#application-layer-settings} + +These settings travel per-query in the [Query packet's settings list](#setting). They change what the server sends on the wire, or how it is framed. + +#### Compression {#compression-settings} + +| Setting | Default | Unit | Description | +|----------------------------------|----------|--------|-------------| +| `network_compression_method` | `"LZ4"` | string | Compression codec used when the Query packet's `compression` flag is set. Values: `"LZ4"`, `"LZ4HC"`, `"ZSTD"`, `"NONE"`. | +| `network_zstd_compression_level` | 1 | 1–15 | ZSTD level when `network_compression_method == "ZSTD"`. | + +The `compression` flag in the [Query packet](#query) (field 6) toggles compression on and off; these settings select which codec is used when it is on. + +#### Log streaming {#log-streaming} + +| Setting | Default | Unit | Description | +|--------------------------|---------------|--------|-------------| +| `send_logs_level` | `"fatal"` | string | Minimum log level. Values: `"none"`, `"fatal"`, `"error"`, `"warning"`, `"information"`, `"debug"`, `"trace"`. | +| `send_logs_source_regexp`| `""` | string | Regex filter on the logger source. Empty = all sources pass. | + +Setting `send_logs_level` to anything other than `"none"` makes the server emit [Log](#log) packets during query execution. + +#### Progress reporting {#progress-reporting} + +| Setting | Default | Unit | Description | +|---------------------|---------|--------------|-------------| +| `interactive_delay` | 100000 | microseconds | Target minimum interval between consecutive Progress packets. | + +This is a target minimum, not a strict maximum: the server may send Progress packets less often when the query is not producing work fast enough. + +#### Result envelope {#result-envelope} + +| Setting | Default | Unit | Description | +|------------------------|---------------|--------------------|-------------| +| `extremes` | false | bool | When true, the server sends an [Extremes](#extremes) packet with min/max values per column. | +| `max_result_rows` | 0 (unlimited) | count | Cap on rows transmitted. Behaviour controlled by `result_overflow_mode`. | +| `max_result_bytes` | 0 (unlimited) | uncompressed bytes | Cap on uncompressed byte volume. Behaviour controlled by `result_overflow_mode`. | +| `result_overflow_mode` | `"throw"` | string | `"throw"` ends the stream with Exception; `"break"` sends partial results followed by EndOfStream. | + +#### Async INSERT {#async-insert} + +| Setting | Default | Unit | Description | +|----------------------------------|---------|---------|-------------| +| `async_insert` | true | bool | When true, INSERT data is queued server-side and batched. | +| `wait_for_async_insert` | true | bool | When true (with `async_insert` on), the server holds the response until queued data is flushed. | +| `wait_for_async_insert_timeout` | 120 | seconds | Maximum time the server waits for a flush before returning. | + +#### Distributed tracing {#distributed-tracing} + +| Setting | Default | Unit | Description | +|-----------------------------------------|---------|-------------------|-------------| +| `opentelemetry_start_trace_probability` | 0.0 | 0–1 probability | Server-side probability of attaching OpenTelemetry context to response telemetry. | + +### Settings out of scope {#settings-out-of-scope} + +These settings are sometimes mistaken for protocol-level settings, but they control SQL execution, storage, or CPU use rather than wire behaviour. A protocol implementation does not need to handle them specially. + +- `max_threads` — parallelism within query execution. +- `max_memory_usage` — per-query memory cap. +- `max_block_size`, `preferred_block_size_bytes` — internal block sizing during query processing; wire blocks are independent of these. +- `compile_expressions` — JIT compilation; CPU only. +- `async_insert_max_data_size` — server-side queue buffer. +- All settings prefixed `input_format_*` and `output_format_*` — these apply to non-native formats over HTTP, not the native protocol. + +## Glossary {#glossary} + +**Cancel** — a client-initiated packet (type 3) that aborts a running query. Not specified in detail on this page. + +**End-of-client-data marker** — an empty Data packet (0 columns, 0 rows) the client sends after the Query packet (and any external tables) to signal "no more input". The server does not begin executing the query until it receives this marker. + +**Feature** — a wire-format change introduced in a specific protocol version. Active when the negotiated version is at or above the feature's version. See [versioning and feature gates](#versioning-and-feature-gates). + +**Inter-server** — a role label for a field that is only meaningful in server-to-server distributed queries. External clients write a default value (usually empty string, 0, or false). + +**Negotiated version** — `min(client_version, server_version)`, computed during the handshake. Determines which features are active for the lifetime of the connection. + +**Packet** — a wire message: a VarUInt packet type code followed by a body whose format depends on the type. See [packet envelope](#packet-envelope). + +**Packet type code** — the leading VarUInt of a packet that identifies its format. Values 0–18 are currently assigned. See the [packet type reference](#packet-type-reference). + +**Response stream** — the sequence of packets the server emits during a query. Open-ended in length, terminated by exactly one `EndOfStream` (success) or `Exception` (failure). See the [query phase](#query-phase). + +**Schema block** — the header block (a Block with columns but 0 rows) that the server sends during the INSERT phase to announce the expected column shapes before the client sends data. + +**Settings list** — a sequence of `(key, flags, value)` tuples in the Query body, terminated by an empty key. Carries per-query application-layer configuration. See [Setting](#setting). + +**Stage** — a VarUInt field in the [Query](#query) packet (field 5) controlling how far the server executes the query: `0` = FetchColumns, `1` = WithMergeableState, `2` = Complete. External clients typically send `2`. + +**Terminator** — a packet that ends a stream. The Query response ends on `EndOfStream` (success) or `Exception` (failure). The client's input stream ends on the empty Data marker. \ No newline at end of file diff --git a/docs/full_native_spec.md b/docs/full_native_spec.md new file mode 100644 index 0000000..dc2f1ff --- /dev/null +++ b/docs/full_native_spec.md @@ -0,0 +1,1161 @@ +--- +description: 'Specification of the ClickHouse Native columnar format: wire primitives, the Block and Column structure, every data type encoding, and the compression frame' +sidebar_label: 'Native Format' +slug: /interfaces/specs/NativeFormat +title: 'Native Format' +doc_type: 'reference' +keywords: ['native format', 'columnar', 'block', 'wire format', 'serialization', 'compression'] +--- + +The Native format is the columnar wire format ClickHouse uses to move tabular data. It shows up in several places: + +- the body of `Data`, `Totals`, `Extremes`, `Log`, `ProfileEvents`, and `TableColumns` packets in the [native TCP protocol](/interfaces/specs/NativeProtocol); +- the output of `SELECT ... FORMAT Native` over HTTP; +- file exports written with `INTO OUTFILE ... FORMAT Native`; +- inter-server replication payloads. + +This page describes the bytes inside a Block — the columnar payload — and the per-column type encodings that build it. Packet framing, connection state, and version negotiation belong to the [native protocol specification](/interfaces/specs/NativeProtocol). + +All multi-byte integer fields are little-endian. Signed integers use two's complement. + +:::tip +For a user-facing introduction to the `Native` format (with `curl` examples), see the [Native format page](/interfaces/formats/Native). This specification is the lower-level wire reference. +::: + +## Overview {#overview} + +Everything that carries rows on the wire is a **Block**: a self-describing chunk of rows stored column by column. All values of column 1 come first, then all of column 2, and so on. A Block carries only the columns the query references, never the full table. + +A column's `data` is laid out according to the *family* its type belongs to. The families, in increasing decoder complexity, are: + +```mermaid +flowchart TD + B[Block] + B --> BI[BlockInfo] + B --> NC[num_columns] + B --> NR[num_rows] + B --> Cs["columns[ ]"] + + Cs --> Col[Column] + Col --> Cname[name] + Col --> Ctype[type] + Col --> Chcs[has_custom_serialization] + Col --> Cdata["data — layout depends on type family"] + + Cdata --> Fixed["Fixed-width
bytes_per_value × num_rows"] + Cdata --> Comp["Composite
recursive, shape from type string"] + Cdata --> Ver["Versioned / stateful
version prefix + cross-block state"] + + Fixed --> FixedEx["Int*, UInt*, Float*, Decimal*
Date, DateTime, DateTime64
UUID, IPv4, IPv6, FixedString(N)"] + Comp --> CompEx["Nullable(T), Array(T)
Tuple(...), Map(K, V), Nested(...)"] + Ver --> VerEx["LowCardinality(T), JSON
Variant(...), Dynamic"] +``` + +- **Fixed-width** types lay `data` out as `bytes_per_value × num_rows` raw bytes, with no per-row framing. +- **Composite** types (`Nullable`, `Array`, `Tuple`, `Map`, `Nested`) have a recursive shape fully derivable from the type string, with no version prefix and no cross-block state. +- **Versioned / stateful** types (`LowCardinality`, `JSON`, `Variant`, `Dynamic`) begin with a serialization-version prefix and may carry state *across* blocks within one query response. + +## Wire primitives {#wire-primitives} + +The Native format builds on four primitive encodings. + +| Primitive | Size | Description | +|-----------------|----------|-------------| +| VarUInt | 1–10 B | LEB-128 variable-length unsigned integer | +| Fixed-width int | 1, 2, 4, 8, 16, 32 B | Little-endian, two's complement for signed | +| String | variable | VarUInt length prefix + raw bytes | +| Bool | 1 B | `0x00` = false, non-zero = true | + +### VarUInt {#varuint} + +A variable-length unsigned integer using LEB-128 encoding. Each byte carries 7 data bits in positions 0–6 and 1 continuation bit in position 7. The continuation bit is `1` when more bytes follow and `0` on the final byte. + +| Value range | Bytes | +|------------------------|-------| +| 0 – 127 | 1 | +| 128 – 16383 | 2 | +| 16384 – 2097151 | 3 | +| up to full UInt64 | up to 10 | + +Encoding the value `300`: + +```text +300 = 0b100101100 + +Byte 0: 0xAC = 0b10101100 (data: 0101100, continuation: 1) +Byte 1: 0x02 = 0b00000010 (data: 0000010, continuation: 0) +``` + +Decoding the bytes `0xAC 0x02`: + +```text +Byte 0: data = 0x2C, continuation = 1 → accumulator = 0x2C, shift = 7 +Byte 1: data = 0x02, continuation = 0 → accumulator = (0x02 << 7) | 0x2C = 300 +``` + +### Fixed-width integers {#fixed-width-integers} + +| Type | Bytes | Encoding | +|--------|-------|-----------------------------------------------| +| UInt8 | 1 | Raw byte | +| UInt16 | 2 | Little-endian | +| UInt32 | 4 | Little-endian | +| UInt64 | 8 | Little-endian | +| UInt128| 16 | Little-endian | +| UInt256| 32 | Little-endian | +| Int8 | 1 | Raw byte, two's complement | +| Int16 | 2 | Little-endian, two's complement | +| Int32 | 4 | Little-endian, two's complement | +| Int64 | 8 | Little-endian, two's complement | +| Int128 | 16 | Little-endian, two's complement | +| Int256 | 32 | Little-endian, two's complement | +| Float32| 4 | IEEE 754 single-precision, little-endian | +| Float64| 8 | IEEE 754 double-precision, little-endian | + +For example, the UInt32 value `1` encodes as `01 00 00 00`, and the Int32 value `-1` as `FF FF FF FF`. + +### String {#string} + +A length-prefixed byte sequence: + +```text +[VarUInt: byte_length] [byte_length bytes: raw value] +``` + +The byte sequence need not be valid UTF-8. An empty string encodes as a single `0x00` byte, and strings may contain any byte values, including embedded NUL. The string `"ab"` encodes as `02 61 62`; to decode, read the VarUInt length (`2`), then read that many bytes. + +### Bool {#bool} + +A single byte. `0x00` is false; any non-zero value is true (canonically `0x01`). + +## Block and column structure {#block-and-column-structure} + +### Block wire layout {#block-wire-layout} + +```text +[BlockInfo] metadata (when the BLOCK_INFO feature is active) +[VarUInt: num_columns] number of columns in this block +[VarUInt: num_rows] number of rows in this block +[Column × num_columns] column entries, omitted when num_columns = 0 +``` + +When the enclosing protocol predates `BLOCK_INFO` (version 51903), the BlockInfo prefix is omitted and everything else is identical. + +### BlockInfo {#blockinfo} + +BlockInfo uses **field-tagged encoding** for forward compatibility. Each field is preceded by a VarUInt field ID, and a field ID of `0` terminates the structure. A decoder skips unknown field IDs by reading the value according to the type implied by the ID. + +| Field ID | Field | Type | Description | +|----------|---------------|-------|------------------------------------------------| +| 1 | is_overflows | UInt8 | Overflow block from GROUP BY. `0` for non-overflow blocks. | +| 2 | bucket_number | Int32 | Aggregation bucket. `-1` for non-bucketed blocks. | +| 0 | (terminator) | — | End of BlockInfo. Always required. | + +Wire layout: + +```text +[VarUInt: 1] [UInt8: is_overflows] +[VarUInt: 2] [Int32: bucket_number] +[VarUInt: 0] +``` + +### Column wire layout {#column-wire-layout} + +A Column appears `num_columns` times within a Block. + +| # | Field | Type | Condition | Description | +|---|--------------------------|---------|--------------------------|-------------| +| 1 | name | String | always | Column name | +| 2 | type | String | always | ClickHouse type string (e.g., `"UInt64"`, `"Array(String)"`) | +| 3 | has_custom_serialization | UInt8 | feature `CUSTOM_SERIALIZATION` (v54454) | `0` = default, `1` = custom (kind_stack follows) | +| 4 | kind_stack | bytes | when field 3 = `1` | One UInt8 enum byte (see below) describing the non-default serialization (sparse, etc.). For the `COMBINATION` value, followed by a VarUInt count plus that many additional kind bytes. | +| 5 | data | bytes | always | Column values for all `num_rows` rows. Layout per type — see [data types](#data-types). For sparse columns, see below. | + +A decoder dispatches on the `type` string. Type strings often carry parameters in parentheses; the decoder strips the `(...)` suffix to find the base type and then parses the parameters for size, scale, or inner-type decisions. Parsing a parameter list with nested types (a `Tuple` inside an `Array`, say) needs a depth-aware comma splitter that tracks parenthesis nesting rather than a naive split on `,`. + +#### kind_stack and sparse encoding {#kind-stack-and-sparse-encoding} + +The `kind_stack` byte enumerates a non-default per-column serialization, mirroring `KindStackBinarySerializationType` in `ClickHouse/src/DataTypes/Serializations/SerializationInfo.cpp`: + +| Byte | Name | Meaning | Wire impact on `data` | +|------|------|---------|------------------------| +| `0x00` | DEFAULT | Default serialization | Identical to `has_custom = 0` | +| `0x01` | SPARSE | Sparse serialization (v54465+) | Offset stream + non-default values; see below | +| `0x02` | DETACHED | Internal storage form (not used over the wire) | — | +| `0x03` | DETACHED_OVER_SPARSE | Detached over sparse (not used over the wire) | — | +| `0x04` | REPLICATED | Dictionary form for repeated values (v54482+) | Index stream + dense element values; see below | +| `0x05` | COMBINATION | Multi-kind stack | Followed by VarUInt `count` and `count` further kind bytes | + +**Sparse wire format.** When `kind_stack = 0x01`, the column `data` is two streams written back-to-back in the single shared TCP stream: + +1. **Offset stream** — a sequence of `VarUInt`s. Each value `v` is either: + - `v` with the high bit at position 62 clear: `(v & 0x3FFFFFFFFFFFFFFF)` = the number of default positions before the next explicit non-default value. That non-default position is `cursor + group_size`, where `cursor` is the running position; afterwards `cursor` advances by `group_size + 1`. + - `v` with bit 62 set (`END_OF_GRANULE_FLAG`): the value with the flag cleared = the number of trailing default positions after the last non-default. This marks the end of the offset stream for the block. +2. **Values stream** — `count` non-default values densely encoded in the inner type, where `count` is the number of non-EOG VarUInts read above. + +A decoder reconstructs a dense column of `num_rows` entries by filling every non-explicit position with the inner type's zero value (`0` for integers and floats, `""` for `String`, `0` days for `Date`, and so on). + +**Replicated wire format.** When `kind_stack = 0x04`, the column `data` is a dictionary: a list of distinct element values plus a per-row index into that list (the same lookup shape as `LowCardinality`). + +```text +[VarUInt num_rows] +[UInt8 size_of_indexes_type] width of each index: 1, 2, 4, or 8 bytes +[indexes: num_rows × size_of_indexes_type bytes] +[VarUInt num_elements] +[elements: num_elements dense inner-type values] +``` + +A decoder reconstructs a dense column by selecting `elements[indexes[i]]` for each output row `i`. Composite inner types recurse: the element list is materialized in the inner type, then indexed. Supported inner types include the leaf types, `Nullable(T)`, `Array(T)`, `Tuple(...)`, `Map(K, V)`, `Nested(...)` (each field expanded like an `Array`), and `LowCardinality(T)` (the shared dictionary is kept; only the per-element keys are indexed). + +### Block variants {#block-variants} + +All Data-family packets share the same Block wire format. The variants differ only in their column and row counts: + +| Variant | num_columns | num_rows | Purpose | +|---------------|-------------|----------|---------| +| Header block | N > 0 | 0 | Announces the result schema (column names + types). | +| Result block | N > 0 | M > 0 | Actual result rows. | +| Empty block | 0 | 0 | Sentinel — end-of-input on the client side; boundary marker on the server side. | + +### Byte-level examples {#byte-level-examples} + +An empty block (with BlockInfo), 8 bytes total: + +```text +01 00 BlockInfo: field_id=1, is_overflows=0 +02 FF FF FF FF BlockInfo: field_id=2, bucket_number=-1 +00 BlockInfo terminator +00 num_columns = 0 +00 num_rows = 0 +``` + +A header block for `SELECT 1` announces one column named `"1"` of type `UInt8`, with zero rows. At protocol ≥ 54454 the `has_custom_serialization` byte is included: + +```text +01 00 BlockInfo: is_overflows = 0 +02 FF FF FF FF BlockInfo: bucket_number = -1 +00 BlockInfo terminator +01 num_columns = 1 +00 num_rows = 0 +01 "1" Column[0].name = "1" +05 "UInt8" Column[0].type = "UInt8" +00 Column[0].has_custom_serialization = 0 + Column[0].data: no bytes (num_rows = 0) +``` + +The result block for the same query, with one row: + +```text +01 00 BlockInfo: is_overflows = 0 +02 FF FF FF FF BlockInfo: bucket_number = -1 +00 BlockInfo terminator +01 num_columns = 1 +01 num_rows = 1 +01 "1" Column[0].name = "1" +05 "UInt8" Column[0].type = "UInt8" +00 Column[0].has_custom_serialization = 0 +01 Column[0].data: one UInt8 byte = 1 +``` + +## Data types {#data-types} + +This section documents every type the Native format can encode within a column's `data`. The four families, in increasing decoder complexity: + +| Family | Section | Streams per column | Cross-block state | +|----------------------------------|---------|--------------------|-------------------| +| Fixed-width | [Fixed-width types](#fixed-width-types) | One | None | +| Variable-length | [Variable-length types](#variable-length-types) | One | None | +| Composite (fixed shape) | [Composite types](#composite-types) | Multiple | None | +| Versioned / stateful | [Versioned types](#versioned-types) | Multiple | Per-column state prefix; possibly cross-block state | + +### Fixed-width types {#fixed-width-types} + +Each value occupies a constant number of bytes. A column of `M` rows occupies exactly `bytes_per_row × M` bytes on the wire, concatenated with no separators or padding. + +| Type string | Bytes per value | Logical value | Wire encoding | +|---------------------|-----------------|---------------------------------------------------|---------------| +| `UInt8` | 1 | Unsigned 8-bit integer | Raw byte | +| `UInt16` | 2 | Unsigned 16-bit integer | Little-endian | +| `UInt32` | 4 | Unsigned 32-bit integer | Little-endian | +| `UInt64` | 8 | Unsigned 64-bit integer | Little-endian | +| `UInt128` | 16 | Unsigned 128-bit integer | Little-endian | +| `UInt256` | 32 | Unsigned 256-bit integer | Little-endian | +| `Int8` | 1 | Signed 8-bit, two's complement | Raw byte | +| `Int16` | 2 | Signed 16-bit, two's complement | Little-endian | +| `Int32` | 4 | Signed 32-bit, two's complement | Little-endian | +| `Int64` | 8 | Signed 64-bit, two's complement | Little-endian | +| `Int128` | 16 | Signed 128-bit, two's complement | Little-endian | +| `Int256` | 32 | Signed 256-bit, two's complement | Little-endian | +| `Float32` | 4 | IEEE 754 single-precision | Little-endian | +| `Float64` | 8 | IEEE 754 double-precision | Little-endian | +| `Bool` | 1 | `0x00` = false, `0x01` = true | Raw byte | +| `Date` | 2 | Days since `1970-01-01` | Little-endian UInt16 | +| `Date32` | 4 | Days since `1970-01-01` (signed; pre-1970 ok) | Little-endian Int32 | +| `DateTime` | 4 | Unix timestamp in seconds | Little-endian UInt32 | +| `DateTime(tz)` | 4 | Same as `DateTime`; timezone is metadata | Little-endian UInt32 | +| `DateTime64(s)` | 8 | Ticks at scale `s` (10^-s seconds since epoch) | Little-endian Int64 | +| `DateTime64(s, tz)` | 8 | Same as `DateTime64(s)`; timezone is metadata | Little-endian Int64 | +| `UUID` | 16 | 128-bit identifier | Two byte-swapped LE UInt64 halves (see [UUID](#uuid)) | +| `IPv4` | 4 | IPv4 address | Little-endian UInt32 | +| `IPv6` | 16 | IPv6 address | Network byte order, no swap | +| `Enum8` | 1 | Signed 8-bit (variant index) | Raw byte | +| `Enum16` | 2 | Signed 16-bit (variant index) | Little-endian | +| `Decimal(P, S)` | 4 / 8 / 16 / 32 | `value × 10^S` as a signed integer; width depends on P (≤9 → 4 B, ≤18 → 8 B, ≤38 → 16 B, ≤76 → 32 B) | Little-endian signed integer | + +#### Integer types {#integer-types} + +`UInt8`–`UInt256` and `Int8`–`Int256` are direct binary encodings of integer values. A decoder reads `bytes_per_row × num_rows` bytes and interprets them according to the type. + +A `UInt32` column holding `[1, 256, 65536]`: + +```text +01 00 00 00 row 0: 1 +00 01 00 00 row 1: 256 +00 00 01 00 row 2: 65536 +``` + +An `Int32` column holding `[-1, 42]`: + +```text +FF FF FF FF row 0: -1 +2A 00 00 00 row 1: 42 +``` + +#### Float32 and Float64 {#float32-and-float64} + +Standard IEEE 754 binary floats: 4 bytes single-precision (`binary32`) and 8 bytes double-precision (`binary64`), each little-endian. NaN, ±Infinity, ±0.0, and subnormals all round-trip without normalization. + +`Float32` value `1.5` (`0x3FC00000`): + +```text +00 00 C0 3F little-endian IEEE 754 +``` + +`Float64` value `1.5` (`0x3FF8000000000000`): + +```text +00 00 00 00 00 00 F8 3F little-endian IEEE 754 +``` + +#### Bool {#bool-type} + +Wire-compatible with `UInt8`: 1 byte per row, `0x00` = false, `0x01` = true. The type string on the wire is literally `Bool` (not `UInt8`), so a decoder dispatching on the type string must recognize it separately. + +A `Bool` column `[true, false, true]`: + +```text +01 00 01 +``` + +#### Date and Date32 {#date-and-date32} + +Both encode dates as integer day counts relative to the Unix epoch `1970-01-01`. Neither carries a time component. + +| Type | Bytes | Encoding | Range | +|----------|-------|-----------------------|---------------------------------| +| `Date` | 2 | Little-endian UInt16 | `1970-01-01` to `2149-06-06` | +| `Date32` | 4 | Little-endian Int32 | wide signed range, pre-1970 ok | + +`Date` value `1970-01-02` (1 day): + +```text +01 00 UInt16 LE = 1 +``` + +`Date32` value `1900-01-01` (-25567 days): + +```text +21 9C FF FF Int32 LE = -25567 +``` + +#### DateTime {#datetime} + +Wire-compatible with `UInt32`: a Unix timestamp in seconds, 4 bytes little-endian. The type may appear as `DateTime` or `DateTime('Timezone')`; the timezone affects display only and is not part of the wire value. Two `DateTime` columns with different timezone parameters produce identical bytes for the same instant. A decoder strips the `(...)` parameter suffix and processes the column as `UInt32`. + +`DateTime('UTC')` value `2024-03-15 14:30:00 UTC` (timestamp `1710513000`): + +```text +A8 84 F4 65 UInt32 LE = 1710513000 +``` + +#### DateTime64(scale[, timezone]) {#datetime64} + +8 bytes, little-endian Int64 representing ticks at scale `10^-scale` seconds since the Unix epoch. The `scale` parameter (0–9) lives in the type string and sets the time unit: + +| Scale | Tick size | Common name | +|-------|-----------------|-------------| +| 0 | 1 second | seconds | +| 3 | 1 millisecond | ms | +| 6 | 1 microsecond | µs | +| 9 | 1 nanosecond | ns | + +The type appears as `DateTime64(s)` (implicit server-default timezone) or `DateTime64(s, 'TimezoneName')` (explicit timezone, display only). Negative values represent ticks before the epoch. + +`DateTime64(3, 'UTC')` value `2024-01-15 12:30:45.123 UTC` (1705321845123 ms): + +```text +83 51 1A 0D 8D 01 00 00 Int64 LE = 1705321845123 +``` + +`DateTime64(0)` value `2024-01-15 12:30:45 UTC` (1705321845 s): + +```text +75 25 A5 65 00 00 00 00 Int64 LE = 1705321845 +``` + +#### UUID {#uuid} + +16 bytes per value. The wire encoding is **not** the canonical 16 big-endian bytes — each 8-byte half is byte-reversed independently. + +The logical model is a 128-bit identifier in canonical text form `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`, where the bytes are conventionally written big-endian. The wire model takes those 16 canonical bytes, splits them into two 8-byte halves, and writes each half little-endian: + +- Wire bytes 0..7 = canonical bytes 0..7 reversed. +- Wire bytes 8..15 = canonical bytes 8..15 reversed. + +UUID `550e8400-e29b-41d4-a716-446655440000`: + +```text +Canonical bytes (16): 55 0E 84 00 E2 9B 41 D4 A7 16 44 66 55 44 00 00 + +Wire bytes: +D4 41 9B E2 00 84 0E 55 high half byte-reversed +00 00 44 55 66 44 16 A7 low half byte-reversed +``` + +The nil UUID (all zeros) appears identically in both representations. + +#### IPv4 and IPv6 {#ipv4-and-ipv6} + +Two related but differently-encoded address types. + +`IPv4` is 4 bytes, encoded as a little-endian UInt32 holding the canonical 32-bit address (the value `(a << 24) | (b << 16) | (c << 8) | d` from `a.b.c.d`). The wire bytes are the network-order bytes reversed. + +`192.168.1.10` (canonical 32-bit value `0xC0A8010A`): + +```text +0A 01 A8 C0 Little-endian UInt32 +``` + +`IPv6` is 16 bytes, written **verbatim in network byte order** with no swap — the same byte order as `inet_pton(AF_INET6, ...)`. + +`2001:db8::1`: + +```text +20 01 0D B8 00 00 00 00 network bytes 0..7 +00 00 00 00 00 00 00 01 network bytes 8..15 +``` + +The asymmetry is deliberate: IPv4 is stored as a `u32` for arithmetic and compact range queries, while IPv6 keeps the network-order layout common to most networking APIs. + +#### Enum8 and Enum16 {#enum8-and-enum16} + +Wire-compatible with `Int8` and `Int16` respectively: 1 or 2 bytes per row, two's complement little-endian for the 16-bit variant. The full variant mapping lives in the type string: + +```text +Enum8('active' = 1, 'inactive' = 2, 'banned' = -1) +Enum16('a' = 1, 'b' = 30000) +``` + +A decoder may strip the `(...)` parameter suffix and dispatch as `Int8` / `Int16`. Clients that need the human-readable label parse the type string. + +An `Enum8('active' = 1, 'inactive' = 2)` column `[active, inactive, active]`: + +```text +01 02 01 +``` + +An `Enum16(...)` value `30000`: + +```text +30 75 Int16 LE = 30000 +``` + +#### Decimal(P, S) {#decimal} + +A signed integer scaled by a power of 10. The integer's byte width is implied by the **precision** `P`; the **scale** `S` is the negative exponent (the number of digits after the decimal point). Both live in the type string. + +| Precision (P) | Backing integer | Bytes | +|-----------------|-----------------|-------| +| 1 ≤ P ≤ 9 | Int32 | 4 | +| 10 ≤ P ≤ 18 | Int64 | 8 | +| 19 ≤ P ≤ 38 | Int128 | 16 | +| 39 ≤ P ≤ 76 | Int256 | 32 | + +The wire encoding is the backing integer in little-endian two's complement, and the logical decimal value is `wire_integer × 10^(-S)`. + +ClickHouse always emits `Decimal(P, S)` regardless of how the type was declared. `Decimal32(S)`, `Decimal64(S)`, and so on all normalize to `Decimal(P, S)` on the wire (with `P` set to the natural maximum for the width: 9, 18, 38, 76). A decoder that recognizes only `Decimal(P, S)` covers every spelling the server emits. + +`Decimal(9, 4)` value `123.4567` → backing integer `1234567`: + +```text +87 D6 12 00 Int32 LE = 1234567 +``` + +`Decimal(18, 1)` value `-1.5` → backing integer `-15`: + +```text +F1 FF FF FF FF FF FF FF Int64 LE = -15 +``` + +`Decimal(38, 4)` value `123.4567` (16 bytes total): + +```text +87 D6 12 00 00 00 00 00 00 00 00 00 00 00 00 00 +``` + +#### Nothing {#nothing} + +The `Nothing` type carries no values. In practice it appears only as the inner type of `Nullable(Nothing)` — what the server returns for an expression like `SELECT NULL` whose only valid value is the absence of one. It is conceptually a unit type. + +On the wire it occupies exactly **one placeholder byte per row**. The server emits the ASCII character `'0'` (`0x30`), but the deserializer ignores the bytes — the content is undefined and decoders must not rely on any specific value. The number of bytes written is `num_rows × 1`, so the column header's `num_rows` fully determines how much to consume. + +The byte-per-row keeps the Block invariant intact: every column spans a length derivable from `num_rows`, so decoders scan forward without per-cell length prefixes. The surrounding `Nullable` always reports every position as NULL, so the placeholders are never inspected. + +A `Nullable(Nothing)` column with 3 rows (all NULL): + +```text +01 01 01 null map: 1, 1, 1 (three NULLs) +30 30 30 Nothing placeholder bytes (one per row) +``` + +The null-map prefix is the standard `Nullable` framing (see [Nullable](#nullable)); the inner three bytes are the `Nothing` payload, which the decoder skips. + +### Variable-length types {#variable-length-types} + +Each value carries its own length on the wire. + +#### String {#string-type} + +Type string: `String`. A `String` column is a sequence of `num_rows` length-prefixed byte sequences: + +```text +[VarUInt: byte_length] [byte_length bytes: raw value] +[VarUInt: byte_length] [byte_length bytes: raw value] +... +``` + +There are no separators between rows beyond the length prefixes, and no row-level state. An empty string is a single `0x00` byte. ClickHouse `String` is byte-oriented rather than text-oriented: UTF-8 validity is not enforced, and a value may contain any bytes including embedded NUL. A decoder that targets a UTF-8 string type either validates on read or exposes raw bytes to the caller. The total bytes consumed by the column is `Σ (varuint_size(len_i) + len_i)` over all rows. + +A column of 3 strings `["ab", "", "c"]` (6 bytes total): + +```text +02 61 62 row 0: length 2, "ab" +00 row 1: length 0, empty +01 63 row 2: length 1, "c" +``` + +#### FixedString(N) {#fixedstring} + +Type string: `FixedString(N)`, where `N` is a positive integer (for example, `FixedString(16)`). The column is exactly `N × num_rows` raw bytes, with no length prefixes and no separators. A decoder parses `N` from the type string and consumes that many bytes per row. + +When the SQL inserts a value shorter than `N` bytes (for example, `CAST('abc' AS FixedString(5))`), the server right-pads with NUL bytes (`0x00`) to the declared length. These padding bytes are part of the stored value and are sent on the wire as-is; trimming is a client-side concern. Like `String`, `FixedString(N)` is byte-array-like rather than text-like — typically used for fixed-width identifiers, address bytes, or hash digests. + +Two `FixedString(3)` values `["abc", "de\0"]` (6 bytes total): + +```text +61 62 63 row 0: 3 bytes, "abc" +64 65 00 row 1: 3 bytes, "de" + NUL padding +``` + +The two string types compared: + +| Property | `String` | `FixedString(N)` | +|------------------------|-----------------------|-----------------------------| +| Per-row length prefix | Yes (VarUInt) | No | +| Row size | Variable | Exactly `N` bytes | +| Total column bytes | Variable | `N × num_rows` | +| NUL-byte padding | n/a | Right-padded by server | +| UTF-8 expected | Typically (not enforced) | No (treat as raw bytes) | +| Type parameter | None | Required integer `N` | + +### Composite types {#composite-types} + +Composite types wrap one or more inner types and share a common wire model: **multiple streams per column**. A single logical column is encoded as two or more independently-read byte sequences, concatenated. + +They share three structural properties: + +- **Fixed shape per schema.** The structure is determined entirely by the type string at decode time. `Array(UInt32)` always has the same stream layout, block to block. +- **No version prefix.** The stream layout is stable across ClickHouse releases. +- **No cross-block state.** Each block is fully self-describing. + +Composites are recursive — an inner type may itself be a composite. + +#### Nullable(T) {#nullable} + +Type string: `Nullable(InnerType)`. Examples: `Nullable(UInt32)`, `Nullable(String)`, `Nullable(FixedString(16))`, `Nullable(DateTime('UTC'))`. + +The wire layout is two concatenated streams, null-map first: + +```text +[null-map stream] num_rows × UInt8 +[values stream] inner type's encoding for num_rows values +``` + +The null-map is exactly `num_rows` bytes, one per row: + +| Byte value | Meaning | +|------------|---------| +| `0x00` | Value is present at this row. | +| non-zero (canonical `0x01`) | Value is NULL. The corresponding bytes in the values stream are a placeholder. | + +The values stream contains the inner type's standard encoding for **all** `num_rows` rows, including the null positions. A decoder must still read the placeholder bytes at null positions to advance the stream, but must consult the null-map before interpreting any individual value. Senders may write any bytes at null positions, so decoders must not rely on a specific placeholder value. + +Placeholder values by inner type family: + +| Inner type family | Placeholder at null position | +|------------------------------------------------------|------------------------------| +| Fixed-width (UInt/Int/Float/DateTime/UUID/etc.) | Zero-initialized bytes of the type's width | +| `String` | Empty string — single `0x00` byte | +| `FixedString(N)` | `N` zero bytes | +| `Array(T)` | Empty array — offsets advance by zero | +| `Tuple(T1, T2, ...)` | Each element uses its own placeholder | + +`Nullable(T)` may appear inside `Array`, `Tuple`, `Map`, and `Nested` — `Array(Nullable(T))` and `Tuple(Nullable(T1), T2)` are common. Nullability does not compose with itself: `Nullable(Nullable(T))` is rejected by the server. + +A `Nullable(UInt8)` with three rows `[5, NULL, 9]` (6 bytes total): + +```text +00 01 00 null-map: present, null, present +05 00 09 values: 5, placeholder, 9 +``` + +A `Nullable(String)` with three rows `["hello", NULL, "world"]` (15 bytes total): + +```text +00 01 00 null-map +05 'h' 'e' 'l' 'l' 'o' row 0: "hello" +00 row 1: placeholder (empty string) +05 'w' 'o' 'r' 'l' 'd' row 2: "world" +``` + +#### Array(T) {#array} + +Type string: `Array(InnerType)`. Examples: `Array(UInt32)`, `Array(String)`, `Array(Nullable(UInt32))`, `Array(Array(UInt8))`. + +The wire layout is two concatenated streams, offsets first: + +```text +[offsets stream] num_rows × UInt64 LE +[values stream] inner type's encoding for offsets[num_rows - 1] values +``` + +The offsets stream is exactly `num_rows` little-endian UInt64 values, each the **cumulative end position** in the values stream after that row's elements: + +- Element start index for row `N` = `offsets[N - 1]` (or `0` when `N == 0`). +- Element end index (exclusive) for row `N` = `offsets[N]`. +- Row `N`'s element count = `offsets[N] - offsets[N - 1]`. + +`offsets[num_rows - 1]` is therefore the total element count across all rows, and the values stream holds that many inner values concatenated end-to-end. + +Offsets are **monotonic non-decreasing**; equal consecutive offsets mean an empty row, and a decoder should reject non-monotonic offsets as corruption. An empty column (`num_rows == 0`) writes zero bytes — no offsets stream and no values stream. Inner types may be any type, including other composites: `Array(Array(T))`, `Array(Tuple(...))`, and `Array(Nullable(T))` are all legal. + +`Array(UInt32)` with rows `[[10, 20, 30], [], [40, 50]]` (44 bytes total): + +```text +Offsets (3 × UInt64 LE = 24 bytes): +03 00 00 00 00 00 00 00 offsets[0] = 3 +03 00 00 00 00 00 00 00 offsets[1] = 3 (empty row) +05 00 00 00 00 00 00 00 offsets[2] = 5 + +Values (5 × UInt32 LE = 20 bytes): +0A 00 00 00 10 +14 00 00 00 20 +1E 00 00 00 30 +28 00 00 00 40 +32 00 00 00 50 +``` + +`Array(String)` with rows `[["a", "bb"], []]` (20 bytes total): + +```text +Offsets (2 × UInt64 LE = 16 bytes): +02 00 00 00 00 00 00 00 offsets[0] = 2 +02 00 00 00 00 00 00 00 offsets[1] = 2 (empty row) + +Values (2 strings, 4 bytes total): +01 'a' row's first string: "a" +02 'b' 'b' row's second string: "bb" +``` + +`Array(Array(UInt32))` with rows `[[[1,2]], [], [[3], [4,5]]]` nests the same shape: + +- Outer offsets: `[1, 1, 3]` — row 0 has 1 inner array, row 1 has 0, row 2 has 2. +- The middle `Array(UInt32)` decodes 3 rows with offsets `[2, 3, 5]`. +- The innermost `UInt32` decodes 5 values: `[1, 2, 3, 4, 5]`. + +That comes to 24 (outer offsets) + 24 (middle offsets) + 20 (values) = 68 bytes. + +#### Tuple(T1, T2, ...) {#tuple} + +Type string: `Tuple(T1, T2, ..., Tn)`. Examples: `Tuple(UInt32, String)`, `Tuple(Int32)`, `Tuple(Array(UInt32), String)`, `Tuple(UInt8, Tuple(Int32, String))`. ClickHouse also supports **named tuples** via `Tuple(a UInt32, b String)`; names are metadata only and do not affect the wire format. + +The wire layout is *N* concatenated streams, one per element type, in declaration order: + +```text +[stream for T1] inner T1's encoding for num_rows values +[stream for T2] inner T2's encoding for num_rows values + ... +[stream for Tn] inner Tn's encoding for num_rows values +``` + +Each stream encodes exactly `num_rows` values. There is no length prefix, no offsets stream, and no separators between streams. A tuple has at least one element (`n >= 1`); empty tuples are rejected at type-parse time. An empty column (`num_rows == 0`) writes zero bytes per stream. Element types may be any type, including other composites — `Tuple(Tuple(...), ...)`, `Tuple(Array(...), ...)`, and `Tuple(Nullable(T1), T2)` are all legal. + +`Tuple(UInt8, UInt8)` with 3 rows `(1,4), (2,5), (3,6)`: + +```text +Element 0 stream (3 × UInt8 = 3 bytes): +01 02 03 + +Element 1 stream (3 × UInt8 = 3 bytes): +04 05 06 +``` + +The layout is **not** row-major: reading the raw bytes back yields `[1, 2, 3]` for element 0 and `[4, 5, 6]` for element 1. + +`Tuple(UInt32, String)` with 2 rows `(10, "a")`, `(20, "bb")` (13 bytes total): + +```text +Element 0 stream (2 × UInt32 LE = 8 bytes): +0A 00 00 00 10 +14 00 00 00 20 + +Element 1 stream (2 strings, 5 bytes total): +01 'a' "a" +02 'b' 'b' "bb" +``` + +#### Map(K, V) {#map} + +Type string: `Map(KeyType, ValueType)`. Examples: `Map(String, UInt32)`, `Map(String, Array(UInt32))`, `Map(UInt8, Tuple(Int32, String))`, `Map(Array(String), Int8)`. The wire format places no restriction on either type — both `K` and `V` may be any supported type, including composites. (ClickHouse's SQL-level rules around accepted key types have varied across releases; consult the SQL documentation for the targeted server version.) + +The wire layout is byte-identical to `Array(Tuple(K, V))`: + +```text +[offsets stream] num_rows × UInt64 LE ← from Array +[keys stream] K's encoding for total_pairs values ┐ from Tuple's +[values stream] V's encoding for total_pairs values ┘ per-element streams +``` + +where `total_pairs = offsets[num_rows - 1]` (or `0` when `num_rows == 0`). The offsets stream has the same semantics as [Array](#array). Keys are positionally aligned with values: pair `i` is `(keys[i], values[i])`. + +ClickHouse's in-memory representation of a Map column is an array of tuples; the type system surfaces it as a distinct type for SQL ergonomics (`m['key']`, `mapKeys`, `mapValues`). The wire format is a direct serialization of that storage, so `Map` and `Array(Tuple(K, V))` are byte-for-byte interchangeable. + +Offsets are monotonic non-decreasing, and both the keys and values streams contain exactly `total_pairs` values. An empty column writes zero bytes. Within a single row keys are typically unique, but this is a semantic rule, not a wire-enforced one: the wire format lets duplicate keys round-trip, and server-side semantics resolve duplicates only when a Map-aware function consumes the row. + +`Map(UInt8, UInt8)` with 2 rows `{1:10, 2:20}`, `{3:30}` (22 bytes total): + +```text +Offsets (2 × UInt64 LE = 16 bytes): +02 00 00 00 00 00 00 00 offsets[0] = 2 +03 00 00 00 00 00 00 00 offsets[1] = 3 + +Keys (3 × UInt8 = 3 bytes): +01 02 03 keys: 1, 2, 3 + +Values (3 × UInt8 = 3 bytes): +0A 14 1E values: 10, 20, 30 +``` + +Keys and values are stored as separate streams, not interleaved — pair `i` is reconstructed by reading `keys[i]` and `values[i]` together. + +`Map(String, UInt32)` with 1 row `{'a':1, 'b':2}` (20 bytes total): + +```text +Offsets (1 × UInt64 LE = 8 bytes): +02 00 00 00 00 00 00 00 offsets[0] = 2 + +Keys (2 strings, 4 bytes total): +01 'a' "a" +01 'b' "b" + +Values (2 × UInt32 LE = 8 bytes): +01 00 00 00 1 +02 00 00 00 2 +``` + +#### Nested(name1 T1, name2 T2, ...) {#nested} + +The on-wire representation of `Nested` depends on the server-side `flatten_nested` setting, which gives two distinct cases. + +**Case A: `flatten_nested = 1` (server default).** When the table was created under default settings, `Nested` is **not a wire type**. The server stores and presents the column as N parallel `Array(T_i)` columns with **dotted names** (`outer.field1`, `outer.field2`, and so on). For the format layer there is nothing new — every dotted column is a regular [Array](#array): + +```text +DESCRIBE TABLE t -- t has column n Nested(a UInt8, b String) +id UInt8 +n.a Array(UInt8) +n.b Array(String) +``` + +**Case B: `flatten_nested = 0`.** When the table was created with `flatten_nested = 0`, the column appears on the wire as a single column with type string `Nested(name1 T1, name2 T2, ...)`, and its layout after the type string is **byte-identical to `Array(Tuple(T1, T2, ..., Tn))`**: + +```text +Nested(a UInt8, b String) bytes (after type string): + 02 00 00 00 00 00 00 00 offsets[0] = 2 + 03 00 00 00 00 00 00 00 offsets[1] = 3 + 0A 14 1E UInt8 stream + 01 'x' 01 'y' 01 'z' String stream + +Array(Tuple(a UInt8, b String)) bytes (after type string): + 02 00 00 00 00 00 00 00 offsets[0] = 2 + 03 00 00 00 00 00 00 00 offsets[1] = 3 + 0A 14 1E UInt8 stream + 01 'x' 01 'y' 01 'z' String stream +``` + +The only difference is the type-string text: `Nested` preserves the field names (`a`, `b`), which `Array(Tuple)` does not carry as named slots. + +The Case B type string is a comma-separated list of (name, type) pairs. The first whitespace separates a name from its type; the type itself may contain further whitespace, commas, and parens, so parsing needs the same depth-aware splitter used for `Tuple`. The wire layout: + +```text +[offsets stream] num_rows × UInt64 LE ← from Array +[field1 stream] T1's encoding for total_elements values ┐ from Tuple's +[field2 stream] T2's encoding for total_elements values │ per-element + ... │ streams +[fieldn stream] Tn's encoding for total_elements values ┘ +``` + +where `total_elements = offsets[num_rows - 1]` (or `0` when `num_rows == 0`). Offsets are monotonic non-decreasing, and every field stream holds exactly `total_elements` values. The server enforces at INSERT time that, within a single row, all fields carry the same number of elements. An empty column writes zero bytes. + +`Nested(a UInt8, b String)` with 2 rows `[(10,'x'),(20,'y')]` and `[(30,'z')]` (25 bytes after the type string): + +```text +Offsets (2 × UInt64 LE = 16 bytes): +02 00 00 00 00 00 00 00 offsets[0] = 2 +03 00 00 00 00 00 00 00 offsets[1] = 3 + +Field 'a' stream (3 × UInt8 = 3 bytes): +0A 14 1E 10, 20, 30 + +Field 'b' stream (3 strings, 6 bytes): +01 'x' 01 'y' 01 'z' "x", "y", "z" +``` + +### Versioned types {#versioned-types} + +Versioned types carry an on-wire serialization-version prefix that declares which variant of the encoding follows. They may also use multiple streams (like the composites) and may maintain cross-block state. + +These types are considerably more involved than the fixed-shape composites, and a client targeting simple analytical queries can defer them. + +#### Serialization version: concept {#serialization-version-concept} + +A **serialization version** is a per-type, per-column on-wire version number that declares which variant of a type's encoding the sender is using. It is the first thing in the column's state prefix, so the decoder reads it and dispatches to the right parser for the rest of the column. + +It is distinct from the protocol version: + +| Dimension | Protocol version | Serialization version (this section) | +|------------------------|----------------------------|--------------------------------------| +| Scope | Connection-wide | Per-type, per-column | +| Negotiated | Yes, at handshake | No — sender writes, receiver reads | +| Controls | Which packet-level features are active | Which wire variant of one type | +| Mandatory to read | Yes | Yes, for each versioned column | + +Most versioned types write the version as a little-endian UInt64 immediately before any other state-prefix data; a few use VarUInt or UInt8. A decoder reads the version first and rejects unknown values — a higher version implies a newer sender format the decoder does not understand, and mis-parsing it corrupts every subsequent byte. + +#### Serialization version reference {#serialization-version-reference} + +| Type | Field width | Value | Name | Meaning | +|---|---|---|---|---| +| **Object** (base for JSON) | UInt64 LE | `0` | `V1` | Original encoding. Includes `max_dynamic_paths` parameter and a list of dynamic paths. | +| | | `1` | `STRING` | Native-format compatibility mode — Object transmitted as a single `String` column containing JSON text. | +| | | `2` | `V2` | V1 layout minus the `max_dynamic_paths` parameter. | +| | | `3` | `FLATTENED` | Native-format compatibility mode — flattened path representation. | +| | | `4` | `V3` | V2 plus a shared-data serialization version sub-field and a statistics flag. | +| **Object shared data** (sub-stream used in Object `V3`) | VarUInt | `0` | `MAP` | Shared data encoded as `Map(String, String)`. | +| | | `1` | `MAP_WITH_BUCKETS` | Same as `MAP` but split into N buckets for scan efficiency. | +| | | `2` | `ADVANCED` | Compact granule format with separate streams for paths / marks / metadata. | +| **Dynamic** | UInt64 LE | `1` | `V1` | Original encoding. Includes `max_dynamic_types` and a list of runtime variant types. | +| | | `2` | `V2` | V1 minus the `max_dynamic_types` parameter. | +| | | `3` | `FLATTENED` | Native-format compatibility mode. | +| | | `4` | `V3` | V2 plus binary-encoded variant type names and empty-statistics support. | +| **Variant** discriminators mode | UInt64 LE | `0` | `BASIC` | Every row's discriminator is written literally. | +| | | `1` | `COMPACT` | If all rows in a granule share one discriminator, only a single value + granule marker is written. | +| **Variant** granule format (when mode is `COMPACT`) | UInt8 | `0` | `PLAIN` | Granule has heterogeneous discriminators. | +| | | `1` | `COMPACT` | Granule has one discriminator for all rows. | +| **LowCardinality** key serialization | Int64 | `1` | `sharedDictionariesWithAdditionalKeys` | Only version currently defined. | +| **JSON-as-String** fallback (when `output_format_native_write_json_as_string` is enabled) | UInt64 LE | `1` | `JSONStringSerializationVersion` | JSON column arrives as a `String` column preceded by this prefix. | + +A few things worth noting about the table: + +- **The values are not contiguous.** `Dynamic` uses `1`, `2`, `3`, `4` with `V3` at `4` and `FLATTENED` at `3`. A higher number is not necessarily newer. +- **Some values are native-format-only.** `Object::STRING`, `Object::FLATTENED`, and `Dynamic::FLATTENED` exist for native-protocol compatibility with clients that do not implement full Object/Dynamic. They do not appear in MergeTree on-disk storage. +- **`V3` is primarily on-disk.** Clients consuming the native TCP protocol typically see `FLATTENED` (value `3`) rather than `V3` (value `4`). + +#### LowCardinality(T) {#lowcardinality} + +The simplest versioned type. It replaces a column of `N` inner values with a small dictionary of unique values plus `N` indices into that dictionary. + +Type string: `LowCardinality(InnerType)`. Examples: `LowCardinality(String)`, `LowCardinality(FixedString(4))`, `LowCardinality(Nullable(String))`. + +```text +[8 bytes: Int64 LE state prefix = 1] ← once per column per query + only emitted before the first block with rows > 0 +[per block with rows > 0]: + [8 bytes: UInt64 LE metadata] ← key type code (low byte) + flag bits + [8 bytes: UInt64 LE dict_size] ← number of dict entries (incl. placeholder slot) + [N bytes: dict values] ← inner type's encoding for dict_size values + [8 bytes: UInt64 LE keys_count] ← always equal to this block's row count + [K bytes: keys] ← (1 << key_type_code) bytes per key +``` + +The state prefix (Int64 LE = 1) is the single defined version, `sharedDictionariesWithAdditionalKeys`; other values are reserved. + +The per-block metadata UInt64 is a bitfield: + +| Bit range | Meaning | +|--------------|---------| +| 0..7 | Key type code: `0` = UInt8, `1` = UInt16, `2` = UInt32, `3` = UInt64. The smallest type that can index `dict_size` entries is chosen. | +| 9 (`0x200`) | `HasAdditionalKeysBit` — set when the block contains new dict entries. | +| 10 (`0x400`) | `NeedUpdateDictionary` — set when the dict in this block extends the global dict. | +| 11 (`0x800`) | `NeedGlobalDictionaryBit` — set when this block references entries from a global dict shared across blocks. | + +For a typical query response with a single data block per column, the metadata is `0x600` (HasAdditionalKeys + NeedUpdateDictionary). + +The dict values are `dict_size` values encoded using the inner type T, with `dict[0]` an empty/default placeholder by convention. For `LowCardinality(Nullable(T))` the wire encodes the dict as plain T (no null-map stream): `dict[1]` is the null marker and real values start at `dict[2..]`. The keys are indices into the dict, with `keys.len()` equal to the block's row count; each index is `1 << key_type_code` bytes (1, 2, 4, or 8), and logical row `N` is reconstructed as `dict[keys[N]]`. + +The state prefix is read once per column per query, before the first block whose row count is greater than zero — header blocks (rows = 0) and empty blocks emit nothing. Within a block, `keys_count` equals the row count, `dict_size` equals the number of values in the dict stream, and each key fits in `1 << key_type_code` bytes. + +:::note +The dictionary can grow **across blocks** within a single query response: a later block may set `NeedUpdateDictionary` and append entries that earlier-block keys do not reference, but that later keys do. A decoder that discards the dictionary after each block cannot decode a multi-block `LowCardinality` column — the dictionary state must persist for the lifetime of the column within the response. +::: + +`LowCardinality(String)` with values `['a', 'b', 'a', 'c', 'b']`: + +```text +01 00 00 00 00 00 00 00 state prefix Int64 = 1 +00 06 00 00 00 00 00 00 metadata UInt64 = 0x600 +04 00 00 00 00 00 00 00 dict_size = 4 +00 dict[0] = "" (placeholder) +01 'a' dict[1] = "a" +01 'b' dict[2] = "b" +01 'c' dict[3] = "c" +05 00 00 00 00 00 00 00 keys_count = 5 +01 02 01 03 02 keys (UInt8): 1, 2, 1, 3, 2 +``` + +Reconstructed: `dict[1], dict[2], dict[1], dict[3], dict[2]` = `["a", "b", "a", "c", "b"]`. + +#### JSON (Tier 1: String fallback) {#json-tier-1-string-fallback} + +ClickHouse's `JSON` type has several wire encodings (see [the serialization version reference](#serialization-version-reference)). Tier 1 is the simplest: when the per-query setting `output_format_native_write_json_as_string = 1` is set, the server flattens every JSON value to its serialized text and emits the column as a `String` with a state-prefix marker. + +Type string: `JSON`. + +```text +[8 bytes: Int64 LE state prefix = 1] ← JSONStringSerializationVersion +[per block with rows > 0]: + [N bytes: String column encoding for num_rows JSON text values] +``` + +The state prefix value is `1` (`JSONStringSerializationVersion`); other values (`0`, `3`, `4`) indicate FLATTENED / V3 formats. It is read once per column per query, before the first block with rows > 0, and the values stream is a standard [String](#string-type) column for `num_rows` rows. + +`JSON` value `'{"a":1}'` (one row): + +```text +01 00 00 00 00 00 00 00 state prefix Int64 = 1 +09 7B 22 61 22 3A 22 31 22 String: 9 bytes "{"a":"1"}" +7D +``` + +ClickHouse re-stringifies non-string JSON values when emitting in Tier 1 mode — the integer `1` becomes the JSON string `"1"`. Tier 1 is enough when the client receives JSON for opaque transit; faithful round-tripping of types requires the Tier 2 encoding below. + +#### Variant(T1, T2, ...) {#variant} + +A discriminated union: each row holds a value of exactly one of the variant types, or NULL. Every row carries a one-byte **global discriminator** selecting its type, and the per-type values are then stored densely, one contiguous run per variant type. + +Type string: `Variant(T1, T2, ...)`. The server canonicalizes the order (variant types are sorted by name), so the type string as received already lists the types in **global-discriminator order**: discriminator `0` selects the first listed type, `1` the second, and so on. `255` (`NULL_DISCRIMINATOR`) means the row is NULL. Variant elements are never `Nullable` — NULL is the discriminator's job. Examples: `Variant(String, UInt64)`, `Variant(Array(UInt8), String)`. + +The state prefix carries a `UInt64 LE` discriminators mode: `0` = BASIC (every row's discriminator written literally), `1` = COMPACT (run-length granule encoding). The server uses BASIC over the native protocol by default (`use_compact_variant_discriminators_serialization = false`); only BASIC is specified here. + +```text +[8 bytes: UInt64 LE discriminators mode = 0] ← state prefix, once per column per query + only emitted before the first block with rows > 0; + followed by each variant element's own state prefix + (empty for leaf types) +[per block with rows > 0]: + [num_rows bytes: UInt8 discriminators] ← one global discriminator per row; 255 = NULL + [for each variant type i, in declared order]: + [values for the rows whose discriminator == i] ← dense encoding in type i; count = #rows selecting i +``` + +To reconstruct, walk the discriminators left to right while keeping a per-type running counter. Row `r` with discriminator `d` (≠ 255) takes the value at index `counter[d]` from variant type `d`'s value run, then `counter[d]` is incremented. Rows with discriminator `255` are NULL and consume no value from any run, so the sum of the per-type counters equals the number of non-NULL rows. + +The state prefix (the mode `UInt64`) is read once per column per query, before the first block with rows > 0; header and empty blocks emit nothing. Each non-NULL discriminator is less than the number of variant types, and variant type `i` is decoded for exactly `count[i]` rows. + +:::note +Variant elements that are themselves stateful (`LowCardinality`, `Variant`, `Dynamic`, `JSON`) emit their own state prefix in the per-element state-prefix phase, after the mode `UInt64`. Leaf types and the plain composites (`Array`, `Tuple`, `Map` of leaf types) have empty state prefixes and compose freely. +::: + +`Variant(String, UInt64)` with values `[42, 'hi', NULL]` (canonical order sorts `String` before `UInt64`, so discriminator 0 = String, 1 = UInt64): + +```text +00 00 00 00 00 00 00 00 state prefix: UInt64 discriminators mode = 0 (BASIC) +01 00 FF discriminators (3 rows): 1 (UInt64), 0 (String), 255 (NULL) +02 68 69 String run (1 value): len=2 "hi" +2A 00 00 00 00 00 00 00 UInt64 run (1 value): 42 +``` + +Reconstructed: row 0 = UInt64 run[0] = `42`; row 1 = String run[0] = `"hi"`; row 2 = NULL. + +#### Dynamic {#dynamic} + +A column whose value type is discovered at runtime: each row holds a value of one of a runtime-determined set of types, or NULL. Unlike `Variant`, the type set is **not** in the column's type string — it is carried in the state prefix. + +Type string: `Dynamic` or `Dynamic(max_types=N)`. The `max_types` parameter bounds how many distinct types the column tracks but does not affect the wire format below. + +`Dynamic` has several encodings (`V1=1`, `V2=2`, `FLATTENED=3`, `V3=4`). Over the native protocol the server emits **V2 by default**, which carries per-variant statistics. The simpler **FLATTENED (version 3)** encoding is selected by sending the query setting `output_format_native_use_flattened_dynamic_and_json_serialization = 1`: + +```text +[8 bytes: UInt64 LE version = 3] ← state prefix, once per column per query + only emitted before the first block with rows > 0 +[VarUInt num_types] ← number of runtime types +[num_types × String] ← type names, in wire order +[per type: its own state prefix] ← empty for leaf types; + indexes-type prefix (empty, integer) +[per block with rows > 0]: + [num_rows × discriminator] ← width by num_types (UInt8 if ≤ 255, else UInt16/32/64); + NULL discriminator = num_types (one past the last type) + [for each type i, in wire order]: + [values for the rows whose discriminator == i] ← dense encoding in type i +``` + +The discriminator width is the smallest unsigned integer that can index `num_types` types plus the NULL slot — `UInt8` for `num_types ≤ 255`, then `UInt16`, `UInt32`, `UInt64` (matching `getSmallestIndexesType(num_types + 1)`). NULL is the discriminator value `num_types` itself, which differs from `Variant`, where NULL is the fixed value `255`. Reconstruction is the same dense walk as `Variant`: keep a per-type counter, and row `r` with discriminator `d` (≠ `num_types`) takes value `counter[d]` from type `d`'s run. + +The state prefix (version + type list) is read once per column per query, before the first block with rows > 0; header and empty blocks emit nothing. + +:::note +Runtime types whose serialization is stateful (`LowCardinality`, `Variant`, `Dynamic`, `JSON`) carry nested state prefixes after the type-name list. +::: + +`Dynamic` with runtime types `["UInt64", "String"]` and rows `[42, "hi", NULL]` (discriminator 0 = UInt64, 1 = String, 2 = NULL): + +```text +03 00 00 00 00 00 00 00 state prefix: UInt64 version = 3 (FLATTENED) +02 VarUInt num_types = 2 +06 55 49 6E 74 36 34 type[0] = "UInt64" +06 53 74 72 69 6E 67 type[1] = "String" +00 01 02 discriminators (3 rows): 0 (UInt64), 1 (String), 2 (NULL) +2A 00 00 00 00 00 00 00 UInt64 run (1 value): 42 +02 68 69 String run (1 value): len=2 "hi" +``` + +Reconstructed: row 0 = UInt64 run[0] = `42`; row 1 = String run[0] = `"hi"`; row 2 = NULL. + +#### JSON (Tier 2: FLATTENED Object) {#json-tier-2-flattened-object} + +The richer JSON encoding: instead of flattening every value to text (Tier 1), the column is split into one sub-column per JSON path. It is selected by **not** requesting the Tier 1 fallback (`output_format_native_write_json_as_string = 0`) while the flattened-serialization flag is on (`output_format_native_use_flattened_dynamic_and_json_serialization = 1`); the server then emits serialization **version 3**. + +There are two kinds of path: + +- **Typed paths** are declared in the type string, for example `JSON(a UInt32, b String)`, and decoded in their declared type. A path name containing dots is backtick-quoted in the type string. +- **Dynamic paths** are discovered at runtime and each decoded as a [Dynamic](#dynamic) column. + +In FLATTENED mode there is **no shared-data column** (that overflow store belongs to the non-flat V2/V3 Object encodings). Every path is a full column of `num_rows` values. + +```text +[8 bytes: UInt64 LE version = 3] ← state prefix, once per column per query +[VarUInt num_dynamic_paths] +[num_dynamic_paths × String] ← dynamic path names, in wire order +[per typed path: its column's state prefix] ← empty for leaf types +[per dynamic path: a Dynamic state prefix] ← version + type list (see Dynamic) +[per block with rows > 0]: + [for each typed path: its column's data] ← num_rows values in the declared type + [for each dynamic path: its Dynamic data] ← num_rows values (discriminators + runs) +``` + +Note the two-phase shape: **all** path state prefixes come first, then **all** path data. A dynamic path's `Dynamic` prefix (in the prefix phase) is therefore separated from its data (in the data phase). The state prefix is read once per column per query, before the first block with rows > 0, and every path column (typed or dynamic) holds exactly `num_rows` values. Row `r`'s object is assembled by reading each path's value at index `r`; a dynamic path whose `Dynamic` discriminator is NULL for that row contributes no key. + +`JSON` value `{"a": 1, "b": "hi"}` (one row, both paths dynamic): + +```text +03 00 00 00 00 00 00 00 version = 3 (Object) +02 num_dynamic_paths = 2 +01 61 path "a" +01 62 path "b" +03 00 00 00 00 00 00 00 01 06 55 49 6E 74 36 34 "a" Dynamic prefix: version 3, 1 type, "UInt64" +03 00 00 00 00 00 00 00 01 06 53 74 72 69 6E 67 "b" Dynamic prefix: version 3, 1 type, "String" +00 2A 00 00 00 00 00 00 00 "a" data: discriminator 0, UInt64 42 +00 02 68 69 "b" data: discriminator 0, String "hi" +``` + +#### JSON non-flat (V2/V3) {#json-non-flat} + +The non-flattened `Object` encodings (V2/V3) are used by MergeTree on-disk storage, and are emitted over the protocol when the flattened flag is off. They carry a shared-data column and per-variant statistics, and are not specified on this page. + +## Compression frame {#compression-frame} + +ClickHouse supports per-block compression for the column data carried inside `Data`, `Totals`, `Extremes`, `Log`, and `ProfileEvents` packets. Compression is opt-in, activated by the protocol-level `compression` flag in the [Query packet](/interfaces/specs/NativeProtocol#query). + +When compression is active, every Block body (the bytes after the `table_name` string of a Data-family packet) is wrapped in the frame below. The packet envelope itself — the packet type code, the `table_name` string, the BlockInfo prefix — is **not** compressed; only the columnar payload is. + +### Frame format {#frame-format} + +```text +[16 bytes: CityHash128 checksum over the 9-byte header + compressed body] +[1 byte: method] ← 0x82 = LZ4, 0x90 = ZSTD, 0x02 = NONE +[4 bytes: compressed_size LE u32] ← INCLUDES the 9-byte header, EXCLUDES the 16-byte checksum +[4 bytes: uncompressed_size LE u32] +[N bytes: compressed body] ← N = compressed_size - 9 +``` + +The total framed size is `16 + compressed_size` = `16 + 9 + body_size` = `25 + body_size`. + +### Method byte values {#method-byte-values} + +| Byte | Method | Body encoding | +|--------|--------|---------------| +| `0x02` | NONE | Body is the raw bytes (no compression). The frame is still emitted; the receiver verifies the checksum. | +| `0x82` | LZ4 | Body is the **LZ4 block format** — *not* the LZ4 frame format. No magic number. | +| `0x90` | ZSTD | Body is a raw zstd single-frame stream (the standard zstd magic number is part of the body). | + +### Checksum {#checksum} + +ClickHouse uses CityHash v1.0.2 (the historical variant), **not** modern Google CityHash; the two produce different outputs. + +The checksum is computed over the 9 header bytes (method + compressed_size + uncompressed_size) plus the N body bytes — everything between the checksum and the end of the frame. The first 8 bytes of the 16-byte CityHash128 output are the low half (LE), the next 8 bytes the high half (LE). A decoder recomputes the CityHash128 over the received header and body and compares against the leading 16 bytes; a mismatch is corruption and the decoder fails. + +### Per-block boundaries {#per-block-boundaries} + +The compressed payload of a Block is a **stream of one or more frames**, not necessarily a single frame. The sender writes the serialized block through a compressed buffer that emits a frame whenever its internal buffer fills (≈1 MB) and a final frame when the block is flushed. So a small block is one frame; a large block is several consecutive frames; and because the sender flushes at the end of each block, a frame boundary always coincides with a block end. + +A receiver streams the frames: read 16 + 9 bytes, read exactly `compressed_size - 9` body bytes, decompress to exactly `uncompressed_size` bytes, and serve those bytes to the block decoder; when the decoder needs more than the current frame holds, pull the next frame. Because the sender flushes per block, after a block is fully decoded the frame buffer is empty and the next block begins at a fresh frame. + +The packet envelope — the packet-type VarUInt and the `table_name` string — is written to the **raw** stream, outside the compressed payload. Only the block body (BlockInfo + columns) is framed. + +### Negotiation {#compression-negotiation} + +Compression is per-query, not per-connection. The Query packet's `compression: bool` field requests it for that single query. The server honours the request and emits compressed `Data`/`Totals`/`Extremes`/`Log`/`ProfileEvents` bodies for the lifetime of the query (`Log`/`ProfileEvents` only at v54481+). It also expects the client's *outgoing* Data blocks — external tables, the empty end-of-data marker, and INSERT rows — to be framed the same way. Subsequent queries on the same connection may differ. + +:::note +With compression on, the server may also route columns through the parallel block-marshalling / `ColumnBLOB` path (`PARALLEL_BLOCK_MARSHALLING`, v54478) for blocks with more than one row. An implementation that compresses INSERT data must be prepared to handle (or explicitly opt out of) that path to avoid a desynchronized stream. +::: + +## Glossary {#glossary} + +**Block** — the unit of data exchange in the Native format. A self-describing chunk of rows stored columnar. See [block and column structure](#block-and-column-structure). + +**BlockInfo** — the metadata header that precedes a Block when the protocol-level `BLOCK_INFO` feature is active (v51903+). Field-tagged for forward compatibility. See [BlockInfo](#blockinfo). + +**Column body** — the bytes of a Column that hold the actual values, after the column header (name, type, has_custom_serialization byte). Layout is type-specific. See [column wire layout](#column-wire-layout). + +**Composite type** — a type built from one or more inner types, encoded as multiple streams per column. The wire format is stable and unversioned. See [composite types](#composite-types). + +**Dictionary (LowCardinality)** — the array of unique values that a `LowCardinality(T)` column references via integer indices. See [LowCardinality](#lowcardinality). + +**Empty block** — a Block with `num_columns = 0` and `num_rows = 0`. Used as a sentinel: a client-side end-of-input marker and a server-side stream boundary marker. See [block variants](#block-variants). + +**Header block** — a Block with `num_columns > 0` and `num_rows = 0`, sent by the server as the first Data packet of a query response. Announces the result schema. See [block variants](#block-variants). + +**Inner type** — the type a composite wraps. `Array(UInt32)` has inner type `UInt32`; `Nullable(T)`'s inner type is `T`. + +**Offsets stream** — the cumulative-end-position UInt64 array that `Array`, `Map`, and `Nested` use to delimit per-row element boundaries. See [Array](#array). + +**Placeholder value** — the bytes written at null positions in a `Nullable(T)` column's values stream. The decoder reads them to advance the stream but ignores their content. See [Nullable](#nullable). + +**Result block** — a Block with `num_rows > 0` carrying actual query result rows. See [block variants](#block-variants). + +**Schema block** — a synonym for header block, used when describing the INSERT phase, where the schema block tells the client the expected column shapes. + +**Serialization version** — a per-type on-wire version number that versioned types use to declare which variant of the encoding follows. Distinct from the protocol version. See [serialization version: concept](#serialization-version-concept). + +**State prefix** — the bytes preceding the per-block payload of a versioned type. Carries the serialization version and (for LowCardinality) one-time-per-column metadata. Emitted once per column per query, before the first block with rows > 0. + +**Stream** — a contiguous run of bytes within a column body, encoding one logical sub-component (a null-map, an offsets array, a values stream). Multi-stream types concatenate two or more streams per column. diff --git a/electron/main.ts b/electron/main.ts index 903f57e..8f9111a 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import fs from 'node:fs'; import { appendClickHouseRequestParams } from '../src/core/clickhouse/request-params'; import { DEFAULT_NATIVE_PROTOCOL_VERSION } from '../src/core/types/native-protocol'; +import { captureNativeQuery } from './native-capture'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -16,9 +17,20 @@ const defaultConfigPath = path.join(configDir, 'config.default.json'); interface Config { host: string; + /** Native TCP connection used for NativeProtocol captures. */ + nativeHost?: string; + nativePort?: number; + user?: string; + password?: string; } -const DEFAULT_CONFIG: Config = { host: 'http://localhost:8123' }; +const DEFAULT_CONFIG: Config = { + host: 'http://localhost:8123', + nativeHost: 'localhost', + nativePort: 9000, + user: 'default', + password: '', +}; function loadConfig(): Config { for (const p of [configPath, defaultConfigPath]) { @@ -91,6 +103,25 @@ ipcMain.handle('execute-query', async (_event, options: { query: string; format: return await response.arrayBuffer(); }); +// IPC: Capture a query over the native TCP protocol via the proxy harness. +// Returns the two per-direction byte streams for the protocol decoder. +ipcMain.handle('capture-native-protocol', async (_event, options: { query: string }) => { + const config = loadConfig(); + const result = await captureNativeQuery({ + query: options.query, + host: config.nativeHost ?? 'localhost', + port: config.nativePort ?? 9000, + user: config.user, + password: config.password, + settings: CLICKHOUSE_SETTINGS, + }); + return { + c2s: new Uint8Array(result.c2s), + s2c: new Uint8Array(result.s2c), + meta: result.meta, + }; +}); + // IPC: Get config ipcMain.handle('get-config', () => { return loadConfig(); diff --git a/electron/native-capture.ts b/electron/native-capture.ts new file mode 100644 index 0000000..cb7ba2d --- /dev/null +++ b/electron/native-capture.ts @@ -0,0 +1,151 @@ +import net from 'node:net'; +import { spawn } from 'node:child_process'; +import { Buffer } from 'node:buffer'; + +/** + * TCP proxy + capture for the ClickHouse native protocol, used by the desktop + * app. It opens a one-shot proxy on localhost, drives `clickhouse-client` + * through it for a single query, and returns the two per-direction byte + * streams. Because both ends are localhost, clickhouse-client disables + * compression, so the capture is plaintext, uncompressed native-protocol + * packets — what ProtocolDecoder consumes. TLS/compression are out of scope. + * + * This mirrors scripts/native-proxy.mjs (the CLI / test-fixture harness); the + * two are kept intentionally small so they stay in sync. + */ + +interface Segment { + dir: 0 | 1; // 0 = client->server, 1 = server->client + data: Buffer; +} + +const DIR_C2S = 0; +const DIR_S2C = 1; + +export interface CaptureOptions { + query: string; + host?: string; + port?: number; + user?: string; + password?: string; + database?: string; + clientPath?: string; + settings?: Record; +} + +export interface CaptureResult { + c2s: Buffer; + s2c: Buffer; + meta: Record; +} + +function startProxy(targetHost: string, targetPort: number): Promise<{ + port: number; + done: Promise; + close: () => void; +}> { + const segments: Segment[] = []; + return new Promise((resolve, reject) => { + let resolveDone!: (value: Segment[]) => void; + let rejectDone!: (reason: Error) => void; + const done = new Promise((res, rej) => { + resolveDone = res; + rejectDone = rej; + }); + let handled = false; + + const server = net.createServer((client) => { + if (handled) { + client.destroy(); + return; + } + handled = true; + const upstream = net.connect(targetPort, targetHost); + let openEnds = 2; + const closeOne = () => { + openEnds -= 1; + if (openEnds === 0) { + server.close(); + resolveDone(segments); + } + }; + client.on('data', (chunk) => { + segments.push({ dir: DIR_C2S, data: Buffer.from(chunk) }); + upstream.write(chunk); + }); + upstream.on('data', (chunk) => { + segments.push({ dir: DIR_S2C, data: Buffer.from(chunk) }); + client.write(chunk); + }); + client.on('end', () => upstream.end()); + upstream.on('end', () => client.end()); + client.on('close', closeOne); + upstream.on('close', closeOne); + const fail = (err: Error) => { + client.destroy(); + upstream.destroy(); + try { server.close(); } catch { /* ignore */ } + rejectDone(err); + }; + client.on('error', fail); + upstream.on('error', fail); + }); + + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const addr = server.address(); + if (addr === null || typeof addr === 'string') { + reject(new Error('proxy failed to bind an ephemeral port')); + return; + } + resolve({ port: addr.port, done, close: () => server.close() }); + }); + }); +} + +function concat(segments: Segment[], dir: 0 | 1): Buffer { + return Buffer.concat(segments.filter((s) => s.dir === dir).map((s) => s.data)); +} + +export async function captureNativeQuery(opts: CaptureOptions): Promise { + const host = opts.host ?? '127.0.0.1'; + const port = opts.port ?? 9000; + const { port: proxyPort, done, close } = await startProxy(host, port); + + const args = ['--host', '127.0.0.1', '--port', String(proxyPort), '--query', opts.query]; + if (opts.user) args.push('--user', opts.user); + if (opts.password) args.push('--password', opts.password); + if (opts.database) args.push('--database', opts.database); + for (const [k, v] of Object.entries(opts.settings ?? {})) args.push(`--${k}=${v}`); + + const child = spawn(opts.clientPath ?? 'clickhouse-client', args, { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.on('data', (d: Buffer) => { stderr += d.toString(); }); + child.stdout.on('data', () => { /* drain */ }); + + const exit = new Promise((resolve, reject) => { + child.on('error', reject); + child.on('close', (code) => resolve(code)); + }); + + let segments: Segment[]; + try { + const code = await exit; + segments = await done; + if (code !== 0 && segments.every((s) => s.dir === DIR_C2S)) { + close(); + throw new Error(`clickhouse-client exited ${code}: ${stderr.trim()}`); + } + } catch (err) { + close(); + throw err; + } + + return { + c2s: concat(segments, DIR_C2S), + s2c: concat(segments, DIR_S2C), + meta: { query: opts.query, host, port, stderr: stderr.trim() }, + }; +} diff --git a/electron/preload.ts b/electron/preload.ts index a1fac7a..25c9aa3 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -3,6 +3,8 @@ import { contextBridge, ipcRenderer } from 'electron'; contextBridge.exposeInMainWorld('electronAPI', { executeQuery: (options: { query: string; format: string; nativeProtocolVersion?: number }): Promise => ipcRenderer.invoke('execute-query', options), + captureNativeProtocol: (options: { query: string }): Promise<{ c2s: Uint8Array; s2c: Uint8Array; meta?: Record }> => + ipcRenderer.invoke('capture-native-protocol', options), getConfig: (): Promise<{ host: string }> => ipcRenderer.invoke('get-config'), saveConfig: (config: { host: string }): Promise => diff --git a/package.json b/package.json index c2d8c90..343b103 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "preview": "vite preview", "test": "vitest run", "test:watch": "vitest", + "capture": "node scripts/capture-native.mjs", "electron:dev": "ELECTRON=true vite dev", "electron:build": "ELECTRON=true vite build && electron-builder", "test:e2e": "ELECTRON=true vite build && npx playwright test" diff --git a/scripts/capture-middleware.mjs b/scripts/capture-middleware.mjs new file mode 100644 index 0000000..8fdd914 --- /dev/null +++ b/scripts/capture-middleware.mjs @@ -0,0 +1,100 @@ +// @ts-check +/** + * HTTP middleware that runs a native-protocol capture and returns the `.chproto` + * dump. Used by the Vite dev/preview server so the **web** UI (which can't open + * a raw TCP socket or spawn a process) can still capture packet streams: the + * browser POSTs the SQL to `/capture`, this handler drives clickhouse-client + * through the proxy, and responds with the dump bytes for the client to decode. + * + * Connection defaults come from env vars so it works without config: + * CH_NATIVE_HOST (default localhost), CH_NATIVE_PORT (9000), + * CH_USER (default), CH_PASSWORD (empty), CLICKHOUSE_CLIENT (clickhouse-client) + */ + +import { captureQuery, encodeDump } from './native-proxy.mjs'; + +/** + * Experimental type settings so Variant/Dynamic/JSON/QBit queries work. Sent as + * per-query client settings. Disabled when CAPTURE_EXPERIMENTAL_SETTINGS=0 — a + * readonly ClickHouse user rejects per-query setting changes, so in that case + * the settings must come from the user's profile instead (see docker/users.xml). + */ +const EXPERIMENTAL_SETTINGS = { + allow_experimental_variant_type: '1', + allow_experimental_dynamic_type: '1', + allow_experimental_json_type: '1', + allow_suspicious_variant_types: '1', + allow_experimental_qbit_type: '1', + allow_suspicious_low_cardinality_types: '1', +}; + +function readBody(req) { + return new Promise((resolve, reject) => { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => resolve(body)); + req.on('error', reject); + }); +} + +/** + * Build a connect-style request handler for native-protocol captures. + * @param {{ host?: string, port?: number, user?: string, password?: string, clientPath?: string }} [opts] + */ +export function createCaptureHandler(opts = {}) { + const host = opts.host ?? process.env.CH_NATIVE_HOST ?? 'localhost'; + const port = Number(opts.port ?? process.env.CH_NATIVE_PORT ?? 9000); + const user = opts.user ?? process.env.CH_USER ?? 'default'; + const password = opts.password ?? process.env.CH_PASSWORD ?? ''; + const clientPath = opts.clientPath ?? process.env.CLICKHOUSE_CLIENT ?? 'clickhouse-client'; + const injectExperimental = (process.env.CAPTURE_EXPERIMENTAL_SETTINGS ?? '1') !== '0'; + const settings = opts.settings ?? (injectExperimental ? EXPERIMENTAL_SETTINGS : {}); + + return async (req, res) => { + if (req.method !== 'POST') { + res.statusCode = 405; + res.end('Use POST with the SQL query as the request body'); + return; + } + try { + const query = (await readBody(req)).trim(); + if (!query) { + res.statusCode = 400; + res.end('Empty query'); + return; + } + const capture = await captureQuery({ + query, + host, + port, + user, + password, + clientPath, + settings, + }); + const dump = encodeDump(capture); + res.statusCode = 200; + res.setHeader('Content-Type', 'application/octet-stream'); + res.setHeader('Content-Length', String(dump.length)); + res.end(dump); + } catch (err) { + res.statusCode = 500; + res.setHeader('Content-Type', 'text/plain'); + res.end(String(err && err.message ? err.message : err)); + } + }; +} + +/** Vite plugin: serve the capture handler at `/capture` in dev and preview. */ +export function captureServerPlugin() { + const handler = createCaptureHandler(); + return { + name: 'native-protocol-capture', + configureServer(server) { + server.middlewares.use('/capture', handler); + }, + configurePreviewServer(server) { + server.middlewares.use('/capture', handler); + }, + }; +} diff --git a/scripts/capture-native.mjs b/scripts/capture-native.mjs new file mode 100644 index 0000000..1c6d8c9 --- /dev/null +++ b/scripts/capture-native.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +// @ts-check +/** + * CLI: capture a single native-protocol query exchange to a .chproto dump. + * + * node scripts/capture-native.mjs --query "SELECT 1" --out capture.chproto + * + * Options: + * --query (required) SQL to run + * --out output dump path (default: capture.chproto) + * --host server host (default 127.0.0.1) + * --port

server native port (default 9000) + * --user / --password / --database + * --client path to clickhouse-client + * --setting k=v per-query setting (repeatable) + */ + +import fs from 'node:fs'; +import { captureQuery, encodeDump } from './native-proxy.mjs'; + +function parseArgs(argv) { + const out = { settings: {}, clientArgs: [] }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + const next = () => argv[++i]; + switch (a) { + case '--query': out.query = next(); break; + case '--out': out.out = next(); break; + case '--host': out.host = next(); break; + case '--port': out.port = Number(next()); break; + case '--user': out.user = next(); break; + case '--password': out.password = next(); break; + case '--database': out.database = next(); break; + case '--client': out.clientPath = next(); break; + case '--setting': { + const [k, ...rest] = next().split('='); + out.settings[k] = rest.join('='); + break; + } + default: out.clientArgs.push(a); + } + } + return out; +} + +async function main() { + const opts = parseArgs(process.argv.slice(2)); + if (!opts.query) { + console.error('error: --query is required'); + process.exit(2); + } + const outPath = opts.out ?? 'capture.chproto'; + const capture = await captureQuery(opts); + fs.writeFileSync(outPath, encodeDump(capture)); + const total = capture.c2s.length + capture.s2c.length; + console.error( + `captured ${capture.segments.length} segments → ${outPath} ` + + `(C2S ${capture.c2s.length}B, S2C ${capture.s2c.length}B, total ${total}B)`, + ); + if (capture.meta.stderr) console.error(`client stderr: ${capture.meta.stderr}`); +} + +main().catch((err) => { + console.error(err.message || err); + process.exit(1); +}); diff --git a/scripts/capture-server.mjs b/scripts/capture-server.mjs new file mode 100644 index 0000000..e8a38aa --- /dev/null +++ b/scripts/capture-server.mjs @@ -0,0 +1,39 @@ +#!/usr/bin/env node +// @ts-check +/** + * Standalone HTTP server exposing the native-protocol capture endpoint. Used in + * production (Docker) where there is no Vite dev server: nginx proxies + * `/capture` to this process, which drives clickhouse-client through the proxy + * and returns the `.chproto` dump for the browser to decode. + * + * Configuration (env): + * CAPTURE_PORT (default 8124), CAPTURE_BIND (default 127.0.0.1) + * CH_NATIVE_HOST, CH_NATIVE_PORT, CH_USER, CH_PASSWORD, CLICKHOUSE_CLIENT + * CAPTURE_EXPERIMENTAL_SETTINGS (default 1; set 0 for readonly users) + */ + +import http from 'node:http'; +import { createCaptureHandler } from './capture-middleware.mjs'; + +const port = Number(process.env.CAPTURE_PORT ?? 8124); +const bind = process.env.CAPTURE_BIND ?? '127.0.0.1'; +const handler = createCaptureHandler(); + +const server = http.createServer((req, res) => { + const url = (req.url ?? '').split('?')[0]; + if (url === '/health') { + res.statusCode = 200; + res.end('ok'); + return; + } + if (url !== '/capture' && url !== '/') { + res.statusCode = 404; + res.end('not found'); + return; + } + handler(req, res); +}); + +server.listen(port, bind, () => { + console.error(`native-protocol capture server listening on http://${bind}:${port}`); +}); diff --git a/scripts/native-proxy.mjs b/scripts/native-proxy.mjs new file mode 100644 index 0000000..913b307 --- /dev/null +++ b/scripts/native-proxy.mjs @@ -0,0 +1,249 @@ +// @ts-check +/** + * TCP proxy + dump harness for the ClickHouse native protocol. + * + * Sits between a native client (clickhouse-client) and a ClickHouse server, + * forwarding bytes in both directions and teeing every byte into a capture. + * Because the proxy listens on localhost and clickhouse-client disables + * compression for localhost connections, the captured stream is plaintext, + * uncompressed native-protocol packets — exactly what protocol-decoder.ts + * consumes. TLS and compression are intentionally out of scope. + * + * The capture is stored as two concatenated per-direction byte streams + * (client->server and server->client). Each direction is an independent, + * ordered native-protocol stream: a packet may be split across TCP segments, + * so the decoder must treat each direction as one contiguous buffer. The raw + * segment log (with direction + order) is kept too, for a faithful timeline. + * + * Dump file layout (.chproto): + * magic "CHPROTO1" (8 bytes) + * metaLen u32 LE length of the metadata JSON + * meta metaLen bytes UTF-8 JSON {query, host, port, ...} + * segments repeated until EOF: + * dir u8 0 = client->server, 1 = server->client + * len u32 LE segment byte length + * data len bytes raw segment bytes + */ + +import net from 'node:net'; +import { spawn } from 'node:child_process'; +import { Buffer } from 'node:buffer'; + +export const MAGIC = 'CHPROTO1'; +export const DIR_C2S = 0; +export const DIR_S2C = 1; + +/** + * @typedef {{ dir: 0 | 1, data: Buffer }} Segment + * @typedef {{ c2s: Buffer, s2c: Buffer, segments: Segment[], meta: Record }} Capture + */ + +/** + * Start a one-shot capturing TCP proxy. It accepts a single client connection, + * forwards it to (targetHost, targetPort), records every byte, and resolves + * once both ends have closed. + * + * @param {object} opts + * @param {string} opts.targetHost + * @param {number} opts.targetPort + * @param {string} [opts.listenHost] + * @returns {Promise<{ port: number, done: Promise, close: () => void }>} + */ +export function startProxy({ targetHost, targetPort, listenHost = '127.0.0.1' }) { + /** @type {Segment[]} */ + const segments = []; + + return new Promise((resolve, reject) => { + /** @type {(value: Segment[]) => void} */ + let resolveDone; + /** @type {(reason: Error) => void} */ + let rejectDone; + const done = new Promise((res, rej) => { + resolveDone = res; + rejectDone = rej; + }); + + let handled = false; + + const server = net.createServer((client) => { + if (handled) { + // Only the first connection is captured; ignore stragglers. + client.destroy(); + return; + } + handled = true; + + const upstream = net.connect(targetPort, targetHost); + let openEnds = 2; + const closeOne = () => { + openEnds -= 1; + if (openEnds === 0) { + server.close(); + resolveDone(segments); + } + }; + + client.on('data', (chunk) => { + segments.push({ dir: DIR_C2S, data: Buffer.from(chunk) }); + upstream.write(chunk); + }); + upstream.on('data', (chunk) => { + segments.push({ dir: DIR_S2C, data: Buffer.from(chunk) }); + client.write(chunk); + }); + + client.on('end', () => upstream.end()); + upstream.on('end', () => client.end()); + client.on('close', closeOne); + upstream.on('close', closeOne); + + const fail = (/** @type {Error} */ err) => { + client.destroy(); + upstream.destroy(); + try { server.close(); } catch { /* ignore */ } + rejectDone(err); + }; + client.on('error', fail); + upstream.on('error', fail); + }); + + server.on('error', reject); + server.listen(0, listenHost, () => { + const addr = server.address(); + if (addr === null || typeof addr === 'string') { + reject(new Error('proxy failed to bind an ephemeral port')); + return; + } + resolve({ port: addr.port, done, close: () => server.close() }); + }); + }); +} + +/** + * Split an ordered segment log into the two concatenated per-direction streams. + * @param {Segment[]} segments + */ +export function splitStreams(segments) { + const c2s = Buffer.concat(segments.filter((s) => s.dir === DIR_C2S).map((s) => s.data)); + const s2c = Buffer.concat(segments.filter((s) => s.dir === DIR_S2C).map((s) => s.data)); + return { c2s, s2c }; +} + +/** + * Capture a single query by driving clickhouse-client through the proxy. + * + * @param {object} opts + * @param {string} opts.query + * @param {string} [opts.host] ClickHouse server host (default 127.0.0.1) + * @param {number} [opts.port] ClickHouse native port (default 9000) + * @param {string} [opts.user] + * @param {string} [opts.password] + * @param {string} [opts.database] + * @param {string} [opts.clientPath] path to clickhouse-client (default "clickhouse-client") + * @param {string[]} [opts.clientArgs] extra args appended to clickhouse-client + * @param {Record} [opts.settings] per-query settings (--name=value) + * @returns {Promise} + */ +export async function captureQuery({ + query, + host = '127.0.0.1', + port = 9000, + user, + password, + database, + clientPath = 'clickhouse-client', + clientArgs = [], + settings = {}, +}) { + const { port: proxyPort, done, close } = await startProxy({ targetHost: host, targetPort: port }); + + const args = ['--host', '127.0.0.1', '--port', String(proxyPort), '--query', query]; + if (user) args.push('--user', user); + if (password) args.push('--password', password); + if (database) args.push('--database', database); + for (const [k, v] of Object.entries(settings)) args.push(`--${k}=${v}`); + args.push(...clientArgs); + + const child = spawn(clientPath, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + let stderr = ''; + child.stderr.on('data', (d) => { stderr += d.toString(); }); + // Drain stdout so the client isn't blocked on a full pipe. + child.stdout.on('data', () => {}); + + const exit = new Promise((resolve, reject) => { + child.on('error', reject); + child.on('close', (code) => resolve(code)); + }); + + let segments; + try { + const code = await exit; + segments = await done; + if (code !== 0 && segments.every((s) => s.dir === DIR_C2S)) { + // Client failed before the server answered anything useful. + close(); + throw new Error(`clickhouse-client exited ${code}: ${stderr.trim()}`); + } + } catch (err) { + close(); + throw err; + } + + const { c2s, s2c } = splitStreams(segments); + return { + c2s, + s2c, + segments, + meta: { query, host, port, user, database, settings, stderr: stderr.trim() }, + }; +} + +/** + * Serialize a capture to the .chproto dump format. + * @param {Capture} capture + * @returns {Buffer} + */ +export function encodeDump(capture) { + const metaJson = Buffer.from(JSON.stringify(capture.meta), 'utf-8'); + const head = Buffer.alloc(MAGIC.length + 4); + head.write(MAGIC, 0, 'ascii'); + head.writeUInt32LE(metaJson.length, MAGIC.length); + + const parts = [head, metaJson]; + for (const seg of capture.segments) { + const segHead = Buffer.alloc(5); + segHead.writeUInt8(seg.dir, 0); + segHead.writeUInt32LE(seg.data.length, 1); + parts.push(segHead, seg.data); + } + return Buffer.concat(parts); +} + +/** + * Parse a .chproto dump back into a capture. + * @param {Buffer} buf + * @returns {Capture} + */ +export function decodeDump(buf) { + if (buf.subarray(0, MAGIC.length).toString('ascii') !== MAGIC) { + throw new Error('not a CHPROTO dump (bad magic)'); + } + let pos = MAGIC.length; + const metaLen = buf.readUInt32LE(pos); + pos += 4; + const meta = JSON.parse(buf.subarray(pos, pos + metaLen).toString('utf-8')); + pos += metaLen; + + /** @type {Segment[]} */ + const segments = []; + while (pos < buf.length) { + const dir = /** @type {0 | 1} */ (buf.readUInt8(pos)); + pos += 1; + const len = buf.readUInt32LE(pos); + pos += 4; + segments.push({ dir, data: Buffer.from(buf.subarray(pos, pos + len)) }); + pos += len; + } + const { c2s, s2c } = splitStreams(segments); + return { c2s, s2c, segments, meta }; +} diff --git a/src/components/AstTree/AstTree.tsx b/src/components/AstTree/AstTree.tsx index 98ecf81..cd4f402 100644 --- a/src/components/AstTree/AstTree.tsx +++ b/src/components/AstTree/AstTree.tsx @@ -1,6 +1,7 @@ import { memo, useCallback, useEffect } from 'react'; import { useStore } from '../../store/store'; import { AstNode } from '../../core/types/ast'; +import { ClickHouseFormat } from '../../core/types/formats'; import '../../styles/ast-tree.css'; function formatNodeCopyText(node: AstNode, label?: string): string { @@ -206,8 +207,12 @@ export function AstTree() { ); } + const isProtocol = parsedData.format === ClickHouseFormat.NativeProtocol; const isBlockBased = !!parsedData.blocks; const itemCount = parsedData.rows?.length ?? parsedData.blocks?.length ?? 0; + const packetCount = isProtocol + ? (parsedData.trailingNodes ?? []).reduce((sum, s) => sum + (s.children?.length ?? 0), 0) + : 0; return (

@@ -219,8 +224,17 @@ export function AstTree() { Collapse All - {parsedData.totalBytes} bytes | {itemCount} {isBlockBased ? 'block(s)' : 'row(s)'} |{' '} - {parsedData.header.columnCount} column(s) + {isProtocol ? ( + <> + {parsedData.totalBytes} bytes | {packetCount} packet(s) | protocol v + {String(parsedData.metadata?.negotiatedVersion ?? '?')} + + ) : ( + <> + {parsedData.totalBytes} bytes | {itemCount} {isBlockBased ? 'block(s)' : 'row(s)'} |{' '} + {parsedData.header.columnCount} column(s) + + )}
diff --git a/src/core/clickhouse/client.ts b/src/core/clickhouse/client.ts index 01c67fe..fc65c42 100644 --- a/src/core/clickhouse/client.ts +++ b/src/core/clickhouse/client.ts @@ -1,12 +1,14 @@ import { ClickHouseFormat } from '../types/formats'; import { DEFAULT_NATIVE_PROTOCOL_VERSION } from '../types/native-protocol'; import { appendClickHouseRequestParams } from './request-params'; +import { parseChprotoDump } from '../decoder/protocol-dump'; /** * Electron IPC API exposed via preload script */ interface ElectronAPI { executeQuery(options: { query: string; format: string; nativeProtocolVersion?: number }): Promise; + captureNativeProtocol(options: { query: string }): Promise<{ c2s: Uint8Array; s2c: Uint8Array; meta?: Record }>; getConfig(): Promise<{ host: string }>; saveConfig(config: { host: string }): Promise; } @@ -33,11 +35,22 @@ export interface QueryResult { timing: number; } +export interface ProtocolCaptureResult { + /** Concatenated [c2s][s2c] buffer (rawData for the hex viewer). */ + combined: Uint8Array; + /** Split point: byte length of the client→server portion. */ + c2sLength: number; + timing: number; + meta?: Record; +} + export class ClickHouseClient { private baseUrl: string; + private captureUrl: string; - constructor(baseUrl = '/clickhouse') { + constructor(baseUrl = '/clickhouse', captureUrl = '/capture') { this.baseUrl = baseUrl; + this.captureUrl = captureUrl; } /** @@ -97,6 +110,52 @@ export class ClickHouseClient { clearTimeout(timeoutId); } } + + /** + * Capture a query over the native TCP protocol. Drives clickhouse-client + * through a capturing proxy and returns both per-direction streams + * concatenated for the protocol decoder. + * + * - Desktop (Electron): the proxy runs in the main process via IPC. + * - Web: POSTs the SQL to the `/capture` endpoint (Vite dev/preview server), + * which runs the proxy server-side and returns a `.chproto` dump. The + * browser cannot open raw TCP itself, so this requires the dev/preview + * server (or another host serving `/capture`). + */ + async captureProtocol(query: string): Promise { + const startTime = performance.now(); + + if (window.electronAPI?.captureNativeProtocol) { + const { c2s, s2c, meta } = await window.electronAPI.captureNativeProtocol({ query }); + return assembleCapture(new Uint8Array(c2s), new Uint8Array(s2c), meta, startTime); + } + + const response = await fetch(this.captureUrl, { + method: 'POST', + body: query, + headers: { 'Content-Type': 'text/plain' }, + }); + if (!response.ok) { + const text = await response.text(); + throw new Error(`Capture failed (${response.status}): ${text}`); + } + const dump = new Uint8Array(await response.arrayBuffer()); + const { c2s, s2c, meta } = parseChprotoDump(dump); + return assembleCapture(c2s, s2c, meta, startTime); + } +} + +/** Concatenate the two per-direction streams and record the split point. */ +function assembleCapture( + c2s: Uint8Array, + s2c: Uint8Array, + meta: Record | undefined, + startTime: number, +): ProtocolCaptureResult { + const combined = new Uint8Array(c2s.length + s2c.length); + combined.set(c2s, 0); + combined.set(s2c, c2s.length); + return { combined, c2sLength: c2s.length, timing: performance.now() - startTime, meta }; } // Default client instance diff --git a/src/core/decoder/fixtures/protocol/01-simple-select.chproto b/src/core/decoder/fixtures/protocol/01-simple-select.chproto new file mode 100644 index 0000000..36dee68 Binary files /dev/null and b/src/core/decoder/fixtures/protocol/01-simple-select.chproto differ diff --git a/src/core/decoder/fixtures/protocol/02-mixed-types.chproto b/src/core/decoder/fixtures/protocol/02-mixed-types.chproto new file mode 100644 index 0000000..4884133 Binary files /dev/null and b/src/core/decoder/fixtures/protocol/02-mixed-types.chproto differ diff --git a/src/core/decoder/fixtures/protocol/03-totals-extremes.chproto b/src/core/decoder/fixtures/protocol/03-totals-extremes.chproto new file mode 100644 index 0000000..bf8d26b Binary files /dev/null and b/src/core/decoder/fixtures/protocol/03-totals-extremes.chproto differ diff --git a/src/core/decoder/fixtures/protocol/04-exception.chproto b/src/core/decoder/fixtures/protocol/04-exception.chproto new file mode 100644 index 0000000..5db4d17 Binary files /dev/null and b/src/core/decoder/fixtures/protocol/04-exception.chproto differ diff --git a/src/core/decoder/fixtures/protocol/05-multiblock.chproto b/src/core/decoder/fixtures/protocol/05-multiblock.chproto new file mode 100644 index 0000000..88c57db Binary files /dev/null and b/src/core/decoder/fixtures/protocol/05-multiblock.chproto differ diff --git a/src/core/decoder/fixtures/protocol/06-logs.chproto b/src/core/decoder/fixtures/protocol/06-logs.chproto new file mode 100644 index 0000000..74b1fca Binary files /dev/null and b/src/core/decoder/fixtures/protocol/06-logs.chproto differ diff --git a/src/core/decoder/fixtures/protocol/07-insert.chproto b/src/core/decoder/fixtures/protocol/07-insert.chproto new file mode 100644 index 0000000..7bd35a0 Binary files /dev/null and b/src/core/decoder/fixtures/protocol/07-insert.chproto differ diff --git a/src/core/decoder/fixtures/protocol/08-parameters.chproto b/src/core/decoder/fixtures/protocol/08-parameters.chproto new file mode 100644 index 0000000..7b9bb54 Binary files /dev/null and b/src/core/decoder/fixtures/protocol/08-parameters.chproto differ diff --git a/src/core/decoder/format-utils.ts b/src/core/decoder/format-utils.ts new file mode 100644 index 0000000..99c4c5f --- /dev/null +++ b/src/core/decoder/format-utils.ts @@ -0,0 +1,36 @@ +/** + * Shared formatting helpers used by the RowBinary and Native decoders. + */ + +/** + * Format eight 16-bit groups as a canonical (RFC 5952) IPv6 string: lowercase + * hex, no leading zeros, and the longest run of consecutive zero groups (length + * >= 2, leftmost on ties) collapsed to "::". + */ +export function formatIPv6(groups: number[]): string { + let bestStart = -1; + let bestLen = 0; + let curStart = -1; + let curLen = 0; + for (let i = 0; i < groups.length; i++) { + if (groups[i] === 0) { + if (curStart === -1) curStart = i; + curLen++; + if (curLen > bestLen) { + bestLen = curLen; + bestStart = curStart; + } + } else { + curStart = -1; + curLen = 0; + } + } + + if (bestLen < 2) { + return groups.map((g) => g.toString(16)).join(':'); + } + + const head = groups.slice(0, bestStart).map((g) => g.toString(16)); + const tail = groups.slice(bestStart + bestLen).map((g) => g.toString(16)); + return `${head.join(':')}::${tail.join(':')}`; +} diff --git a/src/core/decoder/index.ts b/src/core/decoder/index.ts index 6ff765c..1127b06 100644 --- a/src/core/decoder/index.ts +++ b/src/core/decoder/index.ts @@ -1,30 +1,44 @@ import { ClickHouseFormat } from '../types/formats'; -import { FormatDecoder } from './format-decoder'; +import { ParsedData } from '../types/ast'; import { RowBinaryDecoder } from './rowbinary-decoder'; import { NativeDecoder } from './native-decoder'; +import { ProtocolDecoder } from './protocol-decoder'; import { DEFAULT_NATIVE_PROTOCOL_VERSION } from '../types/native-protocol'; // Re-export types and classes export { FormatDecoder } from './format-decoder'; export { RowBinaryDecoder } from './rowbinary-decoder'; export { NativeDecoder } from './native-decoder'; +export { ProtocolDecoder } from './protocol-decoder'; +export type { ProtocolCapture } from './protocol-decoder'; export { BinaryReader } from './reader'; export { decodeLEB128, decodeLEB128BigInt } from './leb128'; +/** Minimal shape shared by every format decoder. */ +export interface Decoder { + decode(): ParsedData; +} + /** - * Factory function to create the appropriate decoder for a format + * Factory function to create the appropriate decoder for a format. + * + * For NativeProtocol the `data` is the concatenated [c2s][s2c] capture buffer + * and `options.protocolC2SLength` is the split point (length of the + * client→server portion). */ export function createDecoder( data: Uint8Array, format: ClickHouseFormat, - options?: { nativeProtocolVersion?: number }, -): FormatDecoder { + options?: { nativeProtocolVersion?: number; protocolC2SLength?: number }, +): Decoder { switch (format) { case ClickHouseFormat.RowBinaryWithNamesAndTypes: return new RowBinaryDecoder(data); case ClickHouseFormat.Native: return new NativeDecoder(data, options?.nativeProtocolVersion ?? DEFAULT_NATIVE_PROTOCOL_VERSION); + case ClickHouseFormat.NativeProtocol: + return new ProtocolDecoder(data, options?.protocolC2SLength ?? 0); default: - throw new Error(`Unsupported format: ${format}`); + throw new Error(`Unsupported format: ${format as string}`); } } diff --git a/src/core/decoder/native-decoder.ts b/src/core/decoder/native-decoder.ts index 01a3dd0..7b93ba5 100644 --- a/src/core/decoder/native-decoder.ts +++ b/src/core/decoder/native-decoder.ts @@ -1,4 +1,6 @@ import { FormatDecoder } from './format-decoder'; +import { BinaryReader } from './reader'; +import { formatIPv6 } from './format-utils'; import { decodeLEB128, decodeLEB128BigInt } from './leb128'; import { parseType } from '../parser/type-parser'; import { ClickHouseType, typeToString } from '../types/clickhouse-types'; @@ -27,17 +29,17 @@ interface DecodedColumnData { /** * Native format decoder (column-oriented with blocks) - * - * Native format structure: - * - Multiple blocks, each with: - * - numColumns (LEB128) - * - numRows (LEB128) - * - For each column: - * - name (LEB128 length + bytes) - * - type (LEB128 length + bytes) - * - column data (all values for this column) - * - Empty block (0 columns, 0 rows) signals end - */ + * + * Native format structure: + * - Multiple blocks, each with: + * - numColumns (LEB128) + * - numRows (LEB128) + * - For each column: + * - name (LEB128 length + bytes) + * - type (LEB128 length + bytes) + * - column data (all values for this column) + * - Empty block (0 columns, 0 rows) signals end + */ export class NativeDecoder extends FormatDecoder { readonly format = ClickHouseFormat.Native; private readonly protocolVersion: number; @@ -46,7 +48,7 @@ export class NativeDecoder extends FormatDecoder { super(data); this.protocolVersion = protocolVersion; } - + decode(): ParsedData { const { blocks, trailingNodes } = this.decodeBlocks(); const header = this.buildHeaderFromBlocks(blocks); @@ -60,6 +62,26 @@ export class NativeDecoder extends FormatDecoder { }; } + /** + * Decode exactly one Native Block at the shared reader's current offset and + * return its BlockNode, advancing the reader past the block. Used by the + * protocol decoder to render the Block carried inside a Data / Totals / + * Extremes / Log / ProfileEvents packet. Block and column AST node ids embed + * `index`, so the caller must pass a value unique across the whole stream. + */ + decodeProtocolBlock(index: number): BlockNode { + return this.decodeBlock(index, { readColumnsWhenZeroRows: true }); + } + + /** + * The underlying byte reader. Exposed so a protocol decoder can interleave + * its own packet-framing reads (VarUInt, String, fixed-width ints) with + * block decoding against a single shared offset cursor. + */ + get sharedReader(): BinaryReader { + return this.reader; + } + private decodeBlocks(): { blocks: BlockNode[]; trailingNodes: AstNode[] } { const blocks: BlockNode[] = []; const trailingNodes: AstNode[] = []; @@ -88,8 +110,8 @@ export class NativeDecoder extends FormatDecoder { return { blocks, trailingNodes }; } - - private decodeBlock(index: number): BlockNode { + + private decodeBlock(index: number, opts?: { readColumnsWhenZeroRows?: boolean }): BlockNode { const startOffset = this.reader.offset; const blockInfoResult = this.protocolVersion > 0 ? this.decodeBlockInfo(index) : undefined; const blockInfo = blockInfoResult?.blockInfo; @@ -152,34 +174,40 @@ export class NativeDecoder extends FormatDecoder { children: headerChildren, }, }; - - // Empty block check - if (numColumns === 0 || numRows === 0) { - return { - index, - byteRange: { start: startOffset, end: this.reader.offset }, - header, - rowCount: 0, - columns: [], - }; - } - - // Decode each column - const columns: BlockColumnNode[] = []; - for (let i = 0; i < numColumns; i++) { - const column = this.decodeBlockColumn(index, i, numRows); - columns.push(column); - } - + + // Empty block check. A truly empty block (0 columns) is always terminal. + // A 0-row block with N>0 columns is a schema/header block: in the native + // *protocol* it still carries each column's name+type (and the custom + // serialization byte) on the wire, so it must be decoded rather than + // skipped. The HTTP Native path keeps the legacy behaviour of treating any + // 0-row block as terminal. + const skipColumns = numColumns === 0 || (numRows === 0 && !opts?.readColumnsWhenZeroRows); + if (skipColumns) { + return { + index, + byteRange: { start: startOffset, end: this.reader.offset }, + header, + rowCount: 0, + columns: [], + }; + } + + // Decode each column + const columns: BlockColumnNode[] = []; + for (let i = 0; i < numColumns; i++) { + const column = this.decodeBlockColumn(index, i, numRows); + columns.push(column); + } + return { index, - byteRange: { start: startOffset, end: this.reader.offset }, - header, - rowCount: numRows, - columns, - }; - } - + byteRange: { start: startOffset, end: this.reader.offset }, + header, + rowCount: numRows, + columns, + }; + } + private decodeBlockColumn(blockIndex: number, columnIndex: number, rowCount: number): BlockColumnNode { const columnId = `block-${blockIndex}-col-${columnIndex}`; // Read column name @@ -286,7 +314,7 @@ export class NativeDecoder extends FormatDecoder { case 'Tuple': return { values: this.decodeTupleColumn(type.elements, type.names, rowCount), prefixNodes: [] }; case 'Nested': - throw new Error(`Native format: ${typeToString(type)} not yet implemented`); + return { values: this.decodeNestedColumn(type.fields, rowCount), prefixNodes: [] }; // Geometry - Variant of geo types case 'Geometry': return { values: this.decodeGeometryColumn(rowCount), prefixNodes: [] }; @@ -306,9 +334,9 @@ export class NativeDecoder extends FormatDecoder { case 'AggregateFunction': return { values: this.decodeAggregateFunctionColumn(type.functionName, type.argTypes, rowCount), prefixNodes: [] }; } - - // Simple types: decode rowCount values sequentially - const values: AstNode[] = []; + + // Simple types: decode rowCount values sequentially + const values: AstNode[] = []; for (let i = 0; i < rowCount; i++) { const node = this.decodeValue(type); node.label = `[${i}]`; @@ -842,305 +870,309 @@ export class NativeDecoder extends FormatDecoder { } private decodeValue(type: ClickHouseType): AstNode { - switch (type.kind) { - // Unsigned integers - case 'UInt8': - return this.decodeUInt8(); - case 'UInt16': - return this.decodeUInt16(); - case 'UInt32': - return this.decodeUInt32(); - case 'UInt64': - return this.decodeUInt64(); - case 'UInt128': - return this.decodeUInt128(); - case 'UInt256': - return this.decodeUInt256(); - - // Signed integers - case 'Int8': - return this.decodeInt8(); - case 'Int16': - return this.decodeInt16(); - case 'Int32': - return this.decodeInt32(); - case 'Int64': - return this.decodeInt64(); - case 'Int128': - return this.decodeInt128(); - case 'Int256': - return this.decodeInt256(); - - // Floats - case 'Float32': - return this.decodeFloat32(); - case 'Float64': - return this.decodeFloat64(); - case 'BFloat16': - return this.decodeBFloat16(); - - // Strings - case 'String': - return this.decodeString(); - case 'FixedString': - return this.decodeFixedString(type.length); - - // Bool - case 'Bool': - return this.decodeBool(); - - // Date/Time - case 'Date': - return this.decodeDate(); - case 'Date32': - return this.decodeDate32(); - case 'DateTime': - return this.decodeDateTime(type.timezone); - case 'DateTime64': - return this.decodeDateTime64(type.precision, type.timezone); - case 'Time': - return this.decodeTime(); - case 'Time64': - return this.decodeTime64(type.precision); - - // Special - case 'UUID': - return this.decodeUUID(); - case 'IPv4': - return this.decodeIPv4(); - case 'IPv6': - return this.decodeIPv6(); - - // Decimal - case 'Decimal32': - return this.decodeDecimal32(type.scale); - case 'Decimal64': - return this.decodeDecimal64(type.scale); - case 'Decimal128': - return this.decodeDecimal128(type.scale); - case 'Decimal256': - return this.decodeDecimal256(type.scale); - - // Enum - case 'Enum8': - return this.decodeEnum8(type.values); - case 'Enum16': - return this.decodeEnum16(type.values); - - // Tuple (fixed-size, same encoding) - case 'Tuple': - return this.decodeTuple(type.elements, type.names); - - // Array (single value - used in SharedVariant) - case 'Array': - return this.decodeArrayValue(type.element); - - // Map (single value - used in SharedVariant) - case 'Map': - return this.decodeMapValue(type.key, type.value); - - // Nullable (single value - used in SharedVariant) - case 'Nullable': - return this.decodeNullableValue(type.inner); - - // LowCardinality - transparent wrapper, decode inner type - case 'LowCardinality': - return this.decodeValue(type.inner); - - // Geo types (same encoding as RowBinary) - case 'Point': - return this.decodePoint(); - - // QBit vector type - case 'QBit': - return this.decodeQBit(type.element, type.dimension); - - // AggregateFunction - case 'AggregateFunction': - return this.decodeAggregateFunction(type.functionName, type.argTypes); - - // Interval types (all stored as Int64) - case 'IntervalNanosecond': - return this.decodeInterval('IntervalNanosecond', 'nanoseconds'); - case 'IntervalMicrosecond': - return this.decodeInterval('IntervalMicrosecond', 'microseconds'); - case 'IntervalMillisecond': - return this.decodeInterval('IntervalMillisecond', 'milliseconds'); - case 'IntervalSecond': - return this.decodeInterval('IntervalSecond', 'seconds'); - case 'IntervalMinute': - return this.decodeInterval('IntervalMinute', 'minutes'); - case 'IntervalHour': - return this.decodeInterval('IntervalHour', 'hours'); - case 'IntervalDay': - return this.decodeInterval('IntervalDay', 'days'); - case 'IntervalWeek': - return this.decodeInterval('IntervalWeek', 'weeks'); - case 'IntervalMonth': - return this.decodeInterval('IntervalMonth', 'months'); - case 'IntervalQuarter': - return this.decodeInterval('IntervalQuarter', 'quarters'); - case 'IntervalYear': - return this.decodeInterval('IntervalYear', 'years'); - - default: - throw new Error(`Native format: ${typeToString(type)} not yet implemented`); - } - } - - /** - * Decode a single Array value (used in SharedVariant) - * Format: LEB128 size + elements - */ - private decodeArrayValue(elementType: ClickHouseType): AstNode { - const start = this.reader.offset; - const typeStr = `Array(${typeToString(elementType)})`; - - // Read array size - const { value: size } = decodeLEB128(this.reader); - - // Read elements - const children: AstNode[] = []; - children.push({ - id: this.generateId(), - type: 'VarUInt', - byteRange: { start, end: this.reader.offset }, - value: size, - displayValue: String(size), - label: 'size', - }); - - const values: unknown[] = []; - for (let i = 0; i < size; i++) { - const elem = this.decodeValue(elementType); - elem.label = `[${i}]`; - children.push(elem); - values.push(elem.value); - } - - return { - id: this.generateId(), - type: typeStr, - byteRange: { start, end: this.reader.offset }, - value: values, - displayValue: `[${values.map(v => typeof v === 'string' ? `"${v}"` : String(v)).join(', ')}]`, - children, - }; - } - - /** - * Decode a single Map value (used in SharedVariant) - * Format: LEB128 size + key-value pairs - */ - private decodeMapValue(keyType: ClickHouseType, valueType: ClickHouseType): AstNode { - const start = this.reader.offset; - const typeStr = `Map(${typeToString(keyType)}, ${typeToString(valueType)})`; - - // Read map size - const { value: size } = decodeLEB128(this.reader); - - // Read key-value pairs - const children: AstNode[] = []; - children.push({ - id: this.generateId(), - type: 'VarUInt', - byteRange: { start, end: this.reader.offset }, - value: size, - displayValue: String(size), - label: 'size', - }); - - const mapValue: Record = {}; - for (let i = 0; i < size; i++) { - const keyNode = this.decodeValue(keyType); - const valueNode = this.decodeValue(valueType); - - const pairNode: AstNode = { - id: this.generateId(), - type: `Tuple(${typeToString(keyType)}, ${typeToString(valueType)})`, - byteRange: { start: keyNode.byteRange.start, end: valueNode.byteRange.end }, - value: [keyNode.value, valueNode.value], - displayValue: `(${keyNode.displayValue}, ${valueNode.displayValue})`, - label: `[${i}]`, - children: [ - { ...keyNode, label: 'key' }, - { ...valueNode, label: 'value' }, - ], - }; - children.push(pairNode); - mapValue[String(keyNode.value)] = valueNode.value; - } - - return { - id: this.generateId(), - type: typeStr, - byteRange: { start, end: this.reader.offset }, - value: mapValue, - displayValue: `{${Object.entries(mapValue).map(([k, v]) => `${k}: ${typeof v === 'string' ? `"${v}"` : v}`).join(', ')}}`, - children, - }; - } - - /** - * Decode a single Nullable value (used in SharedVariant) - * Format: UInt8 null flag + value (if not null) - */ - private decodeNullableValue(innerType: ClickHouseType): AstNode { - const start = this.reader.offset; - const typeStr = `Nullable(${typeToString(innerType)})`; - - // Read null flag - const { value: isNull } = this.reader.readUInt8(); - - if (isNull) { - return { - id: this.generateId(), - type: typeStr, - byteRange: { start, end: this.reader.offset }, - value: null, - displayValue: 'NULL', - metadata: { isNull: true }, - }; - } - - const innerNode = this.decodeValue(innerType); - return { - id: this.generateId(), - type: typeStr, - byteRange: { start, end: innerNode.byteRange.end }, - value: innerNode.value, - displayValue: innerNode.displayValue, - children: [{ ...innerNode, label: 'value' }], - metadata: { isNull: false }, - }; - } - + switch (type.kind) { + // Unsigned integers + case 'UInt8': + return this.decodeUInt8(); + case 'UInt16': + return this.decodeUInt16(); + case 'UInt32': + return this.decodeUInt32(); + case 'UInt64': + return this.decodeUInt64(); + case 'UInt128': + return this.decodeUInt128(); + case 'UInt256': + return this.decodeUInt256(); + + // Signed integers + case 'Int8': + return this.decodeInt8(); + case 'Int16': + return this.decodeInt16(); + case 'Int32': + return this.decodeInt32(); + case 'Int64': + return this.decodeInt64(); + case 'Int128': + return this.decodeInt128(); + case 'Int256': + return this.decodeInt256(); + + // Floats + case 'Float32': + return this.decodeFloat32(); + case 'Float64': + return this.decodeFloat64(); + case 'BFloat16': + return this.decodeBFloat16(); + + // Strings + case 'String': + return this.decodeString(); + case 'FixedString': + return this.decodeFixedString(type.length); + + // Bool + case 'Bool': + return this.decodeBool(); + + // Nothing — one placeholder byte per row, content undefined + case 'Nothing': + return this.decodeNothing(); + + // Date/Time + case 'Date': + return this.decodeDate(); + case 'Date32': + return this.decodeDate32(); + case 'DateTime': + return this.decodeDateTime(type.timezone); + case 'DateTime64': + return this.decodeDateTime64(type.precision, type.timezone); + case 'Time': + return this.decodeTime(); + case 'Time64': + return this.decodeTime64(type.precision); + + // Special + case 'UUID': + return this.decodeUUID(); + case 'IPv4': + return this.decodeIPv4(); + case 'IPv6': + return this.decodeIPv6(); + + // Decimal + case 'Decimal32': + return this.decodeDecimal32(type.scale); + case 'Decimal64': + return this.decodeDecimal64(type.scale); + case 'Decimal128': + return this.decodeDecimal128(type.scale); + case 'Decimal256': + return this.decodeDecimal256(type.scale); + + // Enum + case 'Enum8': + return this.decodeEnum8(type.values); + case 'Enum16': + return this.decodeEnum16(type.values); + + // Tuple (fixed-size, same encoding) + case 'Tuple': + return this.decodeTuple(type.elements, type.names); + + // Array (single value - used in SharedVariant) + case 'Array': + return this.decodeArrayValue(type.element); + + // Map (single value - used in SharedVariant) + case 'Map': + return this.decodeMapValue(type.key, type.value); + + // Nullable (single value - used in SharedVariant) + case 'Nullable': + return this.decodeNullableValue(type.inner); + + // LowCardinality - transparent wrapper, decode inner type + case 'LowCardinality': + return this.decodeValue(type.inner); + + // Geo types (same encoding as RowBinary) + case 'Point': + return this.decodePoint(); + + // QBit vector type + case 'QBit': + return this.decodeQBit(type.element, type.dimension); + + // AggregateFunction + case 'AggregateFunction': + return this.decodeAggregateFunction(type.functionName, type.argTypes); + + // Interval types (all stored as Int64) + case 'IntervalNanosecond': + return this.decodeInterval('IntervalNanosecond', 'nanoseconds'); + case 'IntervalMicrosecond': + return this.decodeInterval('IntervalMicrosecond', 'microseconds'); + case 'IntervalMillisecond': + return this.decodeInterval('IntervalMillisecond', 'milliseconds'); + case 'IntervalSecond': + return this.decodeInterval('IntervalSecond', 'seconds'); + case 'IntervalMinute': + return this.decodeInterval('IntervalMinute', 'minutes'); + case 'IntervalHour': + return this.decodeInterval('IntervalHour', 'hours'); + case 'IntervalDay': + return this.decodeInterval('IntervalDay', 'days'); + case 'IntervalWeek': + return this.decodeInterval('IntervalWeek', 'weeks'); + case 'IntervalMonth': + return this.decodeInterval('IntervalMonth', 'months'); + case 'IntervalQuarter': + return this.decodeInterval('IntervalQuarter', 'quarters'); + case 'IntervalYear': + return this.decodeInterval('IntervalYear', 'years'); + + default: + throw new Error(`Native format: ${typeToString(type)} not yet implemented`); + } + } + + /** + * Decode a single Array value (used in SharedVariant) + * Format: LEB128 size + elements + */ + private decodeArrayValue(elementType: ClickHouseType): AstNode { + const start = this.reader.offset; + const typeStr = `Array(${typeToString(elementType)})`; + + // Read array size + const { value: size } = decodeLEB128(this.reader); + + // Read elements + const children: AstNode[] = []; + children.push({ + id: this.generateId(), + type: 'VarUInt', + byteRange: { start, end: this.reader.offset }, + value: size, + displayValue: String(size), + label: 'size', + }); + + const values: unknown[] = []; + for (let i = 0; i < size; i++) { + const elem = this.decodeValue(elementType); + elem.label = `[${i}]`; + children.push(elem); + values.push(elem.value); + } + + return { + id: this.generateId(), + type: typeStr, + byteRange: { start, end: this.reader.offset }, + value: values, + displayValue: `[${values.map(v => typeof v === 'string' ? `"${v}"` : String(v)).join(', ')}]`, + children, + }; + } + + /** + * Decode a single Map value (used in SharedVariant) + * Format: LEB128 size + key-value pairs + */ + private decodeMapValue(keyType: ClickHouseType, valueType: ClickHouseType): AstNode { + const start = this.reader.offset; + const typeStr = `Map(${typeToString(keyType)}, ${typeToString(valueType)})`; + + // Read map size + const { value: size } = decodeLEB128(this.reader); + + // Read key-value pairs + const children: AstNode[] = []; + children.push({ + id: this.generateId(), + type: 'VarUInt', + byteRange: { start, end: this.reader.offset }, + value: size, + displayValue: String(size), + label: 'size', + }); + + const mapValue: Record = {}; + for (let i = 0; i < size; i++) { + const keyNode = this.decodeValue(keyType); + const valueNode = this.decodeValue(valueType); + + const pairNode: AstNode = { + id: this.generateId(), + type: `Tuple(${typeToString(keyType)}, ${typeToString(valueType)})`, + byteRange: { start: keyNode.byteRange.start, end: valueNode.byteRange.end }, + value: [keyNode.value, valueNode.value], + displayValue: `(${keyNode.displayValue}, ${valueNode.displayValue})`, + label: `[${i}]`, + children: [ + { ...keyNode, label: 'key' }, + { ...valueNode, label: 'value' }, + ], + }; + children.push(pairNode); + mapValue[String(keyNode.value)] = valueNode.value; + } + + return { + id: this.generateId(), + type: typeStr, + byteRange: { start, end: this.reader.offset }, + value: mapValue, + displayValue: `{${Object.entries(mapValue).map(([k, v]) => `${k}: ${typeof v === 'string' ? `"${v}"` : v}`).join(', ')}}`, + children, + }; + } + + /** + * Decode a single Nullable value (used in SharedVariant) + * Format: UInt8 null flag + value (if not null) + */ + private decodeNullableValue(innerType: ClickHouseType): AstNode { + const start = this.reader.offset; + const typeStr = `Nullable(${typeToString(innerType)})`; + + // Read null flag + const { value: isNull } = this.reader.readUInt8(); + + if (isNull) { + return { + id: this.generateId(), + type: typeStr, + byteRange: { start, end: this.reader.offset }, + value: null, + displayValue: 'NULL', + metadata: { isNull: true }, + }; + } + + const innerNode = this.decodeValue(innerType); + return { + id: this.generateId(), + type: typeStr, + byteRange: { start, end: innerNode.byteRange.end }, + value: innerNode.value, + displayValue: innerNode.displayValue, + children: [{ ...innerNode, label: 'value' }], + metadata: { isNull: false }, + }; + } + private buildHeaderFromBlocks(blocks: BlockNode[]): HeaderNode { - if (blocks.length === 0) { - return { - byteRange: { start: 0, end: 0 }, - columnCount: 0, - columnCountRange: { start: 0, end: 0 }, - columns: [], - }; - } - - const firstBlock = blocks[0]; - const columns: ColumnDefinition[] = firstBlock.columns.map((col) => ({ - name: col.name, - nameByteRange: col.nameByteRange, - type: col.type, - typeString: col.typeString, - typeByteRange: col.typeByteRange, - })); - + if (blocks.length === 0) { + return { + byteRange: { start: 0, end: 0 }, + columnCount: 0, + columnCountRange: { start: 0, end: 0 }, + columns: [], + }; + } + + const firstBlock = blocks[0]; + const columns: ColumnDefinition[] = firstBlock.columns.map((col) => ({ + name: col.name, + nameByteRange: col.nameByteRange, + type: col.type, + typeString: col.typeString, + typeByteRange: col.typeByteRange, + })); + return { byteRange: { start: 0, end: firstBlock.columns[0]?.metadataByteRange.end ?? 0 }, columnCount: columns.length, // For Native format, column count is per-block, use first block's range columnCountRange: firstBlock.header.numColumnsRange, - columns, - }; + columns, + }; } private createDefaultNode(type: ClickHouseType, rowIndex: number): AstNode { @@ -1228,1946 +1260,2166 @@ export class NativeDecoder extends FormatDecoder { } // Integer decoders - private decodeUInt8(): AstNode { - const { value, range } = this.reader.readUInt8(); - return { - id: this.generateId(), - type: 'UInt8', - byteRange: range, - value, - displayValue: String(value), - }; - } - - private decodeUInt16(): AstNode { - const { value, range } = this.reader.readUInt16LE(); - return { - id: this.generateId(), - type: 'UInt16', - byteRange: range, - value, - displayValue: String(value), - }; - } - - private decodeUInt32(): AstNode { - const { value, range } = this.reader.readUInt32LE(); - return { - id: this.generateId(), - type: 'UInt32', - byteRange: range, - value, - displayValue: String(value), - }; - } - - private decodeUInt64(): AstNode { - const { value, range } = this.reader.readUInt64LE(); - return { - id: this.generateId(), - type: 'UInt64', - byteRange: range, - value, - displayValue: value.toString(), - }; - } - - private decodeUInt128(): AstNode { - const { value, range } = this.reader.readUInt128LE(); - return { - id: this.generateId(), - type: 'UInt128', - byteRange: range, - value, - displayValue: value.toString(), - }; - } - - private decodeUInt256(): AstNode { - const { value, range } = this.reader.readUInt256LE(); - return { - id: this.generateId(), - type: 'UInt256', - byteRange: range, - value, - displayValue: value.toString(), - }; - } - - private decodeInt8(): AstNode { - const { value, range } = this.reader.readInt8(); - return { - id: this.generateId(), - type: 'Int8', - byteRange: range, - value, - displayValue: String(value), - }; - } - - private decodeInt16(): AstNode { - const { value, range } = this.reader.readInt16LE(); - return { - id: this.generateId(), - type: 'Int16', - byteRange: range, - value, - displayValue: String(value), - }; - } - - private decodeInt32(): AstNode { - const { value, range } = this.reader.readInt32LE(); - return { - id: this.generateId(), - type: 'Int32', - byteRange: range, - value, - displayValue: String(value), - }; - } - - private decodeInt64(): AstNode { - const { value, range } = this.reader.readInt64LE(); - return { - id: this.generateId(), - type: 'Int64', - byteRange: range, - value, - displayValue: value.toString(), - }; - } - - private decodeInt128(): AstNode { - const { value, range } = this.reader.readInt128LE(); - return { - id: this.generateId(), - type: 'Int128', - byteRange: range, - value, - displayValue: value.toString(), - }; - } - - private decodeInt256(): AstNode { - const { value, range } = this.reader.readInt256LE(); - return { - id: this.generateId(), - type: 'Int256', - byteRange: range, - value, - displayValue: value.toString(), - }; - } - - // Float decoders - private decodeFloat32(): AstNode { - const { value, range } = this.reader.readFloat32LE(); - return { - id: this.generateId(), - type: 'Float32', - byteRange: range, - value, - displayValue: value.toString(), - }; - } - - private decodeFloat64(): AstNode { - const { value, range } = this.reader.readFloat64LE(); - return { - id: this.generateId(), - type: 'Float64', - byteRange: range, - value, - displayValue: value.toString(), - }; - } - - private decodeBFloat16(): AstNode { - const { value, range } = this.reader.readBFloat16LE(); - return { - id: this.generateId(), - type: 'BFloat16', - byteRange: range, - value, - displayValue: value.toString(), - }; - } - - // String decoders - private decodeString(): AstNode { - const startOffset = this.reader.offset; - const { value: length } = decodeLEB128(this.reader); - const { value: bytes } = this.reader.readBytes(length); - const str = new TextDecoder().decode(bytes); - - return { - id: this.generateId(), - type: 'String', - byteRange: { start: startOffset, end: this.reader.offset }, - value: str, - displayValue: `"${str}"`, - }; - } - - private decodeFixedString(length: number): AstNode { - const { value: bytes, range } = this.reader.readBytes(length); - // Find first null byte to get actual string length - let actualLength = length; - for (let i = 0; i < bytes.length; i++) { - if (bytes[i] === 0) { - actualLength = i; - break; - } - } - const str = new TextDecoder().decode(bytes.slice(0, actualLength)); - - return { - id: this.generateId(), - type: `FixedString(${length})`, - byteRange: range, - value: str, - displayValue: `"${str}"`, - metadata: { fixedLength: length, actualLength }, - }; - } - - // Bool decoder - private decodeBool(): AstNode { - const { value, range } = this.reader.readUInt8(); - return { - id: this.generateId(), - type: 'Bool', - byteRange: range, - value: value !== 0, - displayValue: value !== 0 ? 'true' : 'false', - }; - } - - // Date/Time decoders - private decodeDate(): AstNode { - const { value, range } = this.reader.readUInt16LE(); - const date = new Date(value * 24 * 60 * 60 * 1000); - return { - id: this.generateId(), - type: 'Date', - byteRange: range, - value: date, - displayValue: date.toISOString().split('T')[0], - metadata: { daysSinceEpoch: value }, - }; - } - - private decodeDate32(): AstNode { - const { value, range } = this.reader.readInt32LE(); - const date = new Date(value * 24 * 60 * 60 * 1000); - return { - id: this.generateId(), - type: 'Date32', - byteRange: range, - value: date, - displayValue: date.toISOString().split('T')[0], - metadata: { daysSinceEpoch: value }, - }; - } - - private decodeDateTime(timezone?: string): AstNode { - const { value, range } = this.reader.readUInt32LE(); - const date = new Date(value * 1000); - return { - id: this.generateId(), - type: timezone ? `DateTime('${timezone}')` : 'DateTime', - byteRange: range, - value: date, - displayValue: date.toISOString().replace('T', ' ').replace('Z', ''), - metadata: { secondsSinceEpoch: value, timezone }, - }; - } - - private decodeDateTime64(precision: number, timezone?: string): AstNode { - const { value, range } = this.reader.readInt64LE(); - const divisor = BigInt(Math.pow(10, precision)); - const seconds = Number(value / divisor); - const subseconds = Number(value % divisor); - const date = new Date(seconds * 1000 + subseconds / Math.pow(10, precision - 3)); - - return { - id: this.generateId(), - type: timezone ? `DateTime64(${precision}, '${timezone}')` : `DateTime64(${precision})`, - byteRange: range, - value: date, - displayValue: date.toISOString().replace('T', ' ').replace('Z', ''), - metadata: { ticksSinceEpoch: value.toString(), precision, timezone }, - }; - } - - private decodeTime(): AstNode { - const { value, range } = this.reader.readInt32LE(); - const sign = value < 0 ? '-' : ''; - const absValue = Math.abs(value); - const hours = Math.floor(absValue / 3600); - const minutes = Math.floor((absValue % 3600) / 60); - const seconds = absValue % 60; - - return { - id: this.generateId(), - type: 'Time', - byteRange: range, - value, - displayValue: `${sign}${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`, - metadata: { totalSeconds: value }, - }; - } - - private decodeTime64(precision: number): AstNode { - const { value, range } = this.reader.readInt64LE(); - const divisor = BigInt(Math.pow(10, precision)); - const totalSeconds = Number(value / divisor); - const subseconds = Number(value % divisor); - - const sign = totalSeconds < 0 ? '-' : ''; - const absSeconds = Math.abs(totalSeconds); - const hours = Math.floor(absSeconds / 3600); - const minutes = Math.floor((absSeconds % 3600) / 60); - const seconds = absSeconds % 60; - - return { - id: this.generateId(), - type: `Time64(${precision})`, - byteRange: range, - value: value.toString(), - displayValue: `${sign}${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}.${Math.abs(subseconds).toString().padStart(precision, '0')}`, - metadata: { precision, rawValue: value.toString() }, - }; - } - - // Special type decoders - private decodeUUID(): AstNode { - const { value: bytes, range } = this.reader.readBytes(16); - - // ClickHouse UUID has special byte ordering - const hex = (b: number) => b.toString(16).padStart(2, '0'); - - const uuid = [ - hex(bytes[7]), - hex(bytes[6]), - hex(bytes[5]), - hex(bytes[4]), - '-', - hex(bytes[3]), - hex(bytes[2]), - '-', - hex(bytes[1]), - hex(bytes[0]), - '-', - hex(bytes[15]), - hex(bytes[14]), - '-', - hex(bytes[13]), - hex(bytes[12]), - hex(bytes[11]), - hex(bytes[10]), - hex(bytes[9]), - hex(bytes[8]), - ].join(''); - - return { - id: this.generateId(), - type: 'UUID', - byteRange: range, - value: uuid, - displayValue: uuid, - }; - } - - private decodeIPv4(): AstNode { - const { value: bytes, range } = this.reader.readBytes(4); - // IPv4 stored as little-endian UInt32 - const ip = `${bytes[3]}.${bytes[2]}.${bytes[1]}.${bytes[0]}`; - - return { - id: this.generateId(), - type: 'IPv4', - byteRange: range, - value: ip, - displayValue: ip, - }; - } - - private decodeIPv6(): AstNode { - const { value: bytes, range } = this.reader.readBytes(16); - // Format as standard IPv6 - const groups: string[] = []; - for (let i = 0; i < 16; i += 2) { - groups.push(((bytes[i] << 8) | bytes[i + 1]).toString(16)); - } - const ip = groups.join(':'); - - return { - id: this.generateId(), - type: 'IPv6', - byteRange: range, - value: ip, - displayValue: ip, - }; - } - - // Decimal decoders - private decodeDecimal32(scale: number): AstNode { - const { value, range } = this.reader.readInt32LE(); - const scaleFactor = Math.pow(10, scale); - const decoded = value / scaleFactor; - return { - id: this.generateId(), - type: `Decimal32(${scale})`, - byteRange: range, - value: decoded, - displayValue: decoded.toFixed(scale), - metadata: { scale, rawValue: value }, - }; - } - - private decodeDecimal64(scale: number): AstNode { - const { value, range } = this.reader.readInt64LE(); - const scaleFactor = BigInt(Math.pow(10, scale)); - const wholePart = value / scaleFactor; - const fracPart = value % scaleFactor; - const decoded = Number(wholePart) + Number(fracPart) / Number(scaleFactor); - return { - id: this.generateId(), - type: `Decimal64(${scale})`, - byteRange: range, - value: decoded, - displayValue: decoded.toFixed(scale), - metadata: { scale, rawValue: value.toString() }, - }; - } - - private decodeDecimal128(scale: number): AstNode { - const { value, range } = this.reader.readInt128LE(); - const scaleFactor = 10n ** BigInt(scale); - const wholePart = value / scaleFactor; - const fracPart = value >= 0n ? value % scaleFactor : -((-value) % scaleFactor); - return { - id: this.generateId(), - type: `Decimal128(${scale})`, - byteRange: range, - value: value.toString(), - displayValue: `${wholePart}.${fracPart.toString().padStart(scale, '0')}`, - metadata: { scale, rawValue: value.toString() }, - }; - } - - private decodeDecimal256(scale: number): AstNode { - const { value, range } = this.reader.readInt256LE(); - const scaleFactor = 10n ** BigInt(scale); - const wholePart = value / scaleFactor; - const fracPart = value >= 0n ? value % scaleFactor : -((-value) % scaleFactor); - return { - id: this.generateId(), - type: `Decimal256(${scale})`, - byteRange: range, - value: value.toString(), - displayValue: `${wholePart}.${fracPart.toString().padStart(scale, '0')}`, - metadata: { scale, rawValue: value.toString() }, - }; - } - - // Enum decoders - private decodeEnum8(values: Map): AstNode { - const { value, range } = this.reader.readInt8(); - const name = values.get(value) ?? ``; - - return { - id: this.generateId(), - type: 'Enum8', - byteRange: range, - value, - displayValue: `'${name}'`, - metadata: { enumValue: value, enumName: name }, - }; - } - - private decodeEnum16(values: Map): AstNode { - const { value, range } = this.reader.readUInt16LE(); - const name = values.get(value) ?? ``; - - return { - id: this.generateId(), - type: 'Enum16', - byteRange: range, - value, - displayValue: `'${name}'`, - metadata: { enumValue: value, enumName: name }, - }; - } - - // Tuple decoder for single value (used in nested contexts) - private decodeTuple(elements: ClickHouseType[], names?: string[]): AstNode { - const startOffset = this.reader.offset; - const children: AstNode[] = []; - - for (let i = 0; i < elements.length; i++) { - const label = names?.[i] ?? `[${i}]`; - const child = this.decodeValue(elements[i]); - child.label = label; - children.push(child); - } - - const typeStr = names - ? `Tuple(${elements.map((e, i) => `${names[i]} ${typeToString(e)}`).join(', ')})` - : `Tuple(${elements.map(typeToString).join(', ')})`; - - return { - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: this.reader.offset }, - value: children.map((c) => c.value), - displayValue: `(${children.map((c) => c.displayValue).join(', ')})`, - children, - }; - } - - /** - * Tuple in Native format (columnar): - * Each element stream is written sequentially for all rows - * - For Tuple(A, B) with N rows: A0, A1, ..., A(N-1), B0, B1, ..., B(N-1) - */ - private decodeTupleColumn(elements: ClickHouseType[], names: string[] | undefined, rowCount: number): AstNode[] { - const typeStr = names - ? `Tuple(${elements.map((e, i) => `${names[i]} ${typeToString(e)}`).join(', ')})` - : `Tuple(${elements.map(typeToString).join(', ')})`; - - // Read all values for each element type - const elementColumns: AstNode[][] = []; - for (let i = 0; i < elements.length; i++) { - elementColumns.push(this.decodeColumnData(elements[i], rowCount).values); - } - - // Assemble tuples - const values: AstNode[] = []; - for (let row = 0; row < rowCount; row++) { - const children: AstNode[] = []; - - for (let el = 0; el < elements.length; el++) { - const label = names?.[el] ?? `[${el}]`; - const child = elementColumns[el][row]; - child.label = label; - children.push(child); - } - - const startOffset = children[0]?.byteRange.start ?? this.reader.offset; - const endOffset = children[children.length - 1]?.byteRange.end ?? this.reader.offset; - - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: endOffset }, - value: children.map((c) => c.value), - displayValue: `(${children.map((c) => c.displayValue).join(', ')})`, - label: `[${row}]`, - children, - }); - } - - return values; - } - - // ========================================= - // Complex type column decoders (Native-specific) - // ========================================= - - /** - * Nullable in Native format: - * 1. NullMap stream: N bytes (0x00 = not null, 0x01 = null) - * 2. Values stream: N values of inner type (including placeholders for NULLs) - */ - private decodeNullableColumn(innerType: ClickHouseType, rowCount: number): AstNode[] { - const typeStr = `Nullable(${typeToString(innerType)})`; - - // Read null map first - const nullMapStart = this.reader.offset; - const nullMap: boolean[] = []; - for (let i = 0; i < rowCount; i++) { - const { value } = this.reader.readUInt8(); - nullMap.push(value !== 0); - } - - // Read all values (even for NULLs) - const innerValues = this.decodeColumnData(innerType, rowCount).values; - - // Combine null map with values - const values: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const isNull = nullMap[i]; - const innerNode = innerValues[i]; - - if (isNull) { - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { - start: nullMapStart + i, - end: innerNode.byteRange.end, - }, - value: null, - displayValue: 'NULL', - label: `[${i}]`, - metadata: { isNull: true }, - }); - } else { - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { - start: nullMapStart + i, - end: innerNode.byteRange.end, - }, - value: innerNode.value, - displayValue: innerNode.displayValue, - label: `[${i}]`, - children: innerNode.children, - metadata: { isNull: false }, - }); - } - } - - return values; - } - - /** - * Array in Native format: - * 1. ArraySizes stream: N cumulative offsets as UInt64 - * 2. ArrayElements stream: flattened elements - * - * Note: For Array(JSON) inside variant types, the JSON structure prefix is read - * separately (via readColumnPrefix) and decoding uses decodeArrayColumnWithPrefix. - */ - private decodeArrayColumn(elementType: ClickHouseType, rowCount: number): AstNode[] { - const typeStr = `Array(${typeToString(elementType)})`; - - // Read cumulative offsets with AST nodes - const offsetNodes: AstNode[] = []; - const offsets: bigint[] = []; - for (let i = 0; i < rowCount; i++) { - const start = this.reader.offset; - const { value } = this.reader.readUInt64LE(); - offsets.push(value); - offsetNodes.push({ - id: this.generateId(), - type: 'UInt64', - byteRange: { start, end: this.reader.offset }, - value, - displayValue: String(value), - label: 'offset', - }); - } - - // Calculate total elements and individual array sizes - const totalElements = rowCount > 0 ? Number(offsets[rowCount - 1]) : 0; - const sizes: number[] = []; - let prevOffset = 0n; - for (let i = 0; i < rowCount; i++) { - sizes.push(Number(offsets[i] - prevOffset)); - prevOffset = offsets[i]; - } - - // Read all elements - const allElements = this.decodeColumnData(elementType, totalElements).values; - const elementsEnd = this.reader.offset; - - // Distribute elements to arrays - const values: AstNode[] = []; - let elementIndex = 0; - for (let i = 0; i < rowCount; i++) { - const size = sizes[i]; - const arrayElements = allElements.slice(elementIndex, elementIndex + size); - elementIndex += size; - - // Update labels for array elements - arrayElements.forEach((el, j) => { - el.label = `[${j}]`; - }); - - // Create length node showing the computed size from offset difference - const lengthNode: AstNode = { - id: this.generateId(), - type: 'UInt64', - byteRange: offsetNodes[i].byteRange, - value: BigInt(size), - displayValue: `${size} (cumulative: ${offsets[i]})`, - label: 'length', - }; - - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: offsetNodes[i].byteRange.start, end: elementsEnd }, - value: arrayElements.map(e => e.value), - displayValue: `[${arrayElements.map(e => e.displayValue).join(', ')}]`, - label: `[${i}]`, - children: [lengthNode, ...arrayElements], - metadata: { size }, - }); - } - - return values; - } - - /** - * Map in Native format (as Array(Tuple(K, V))): - * 1. ArraySizes stream: cumulative offsets - * 2. Keys stream: all keys - * 3. Values stream: all values - */ - private decodeMapColumn(keyType: ClickHouseType, valueType: ClickHouseType, rowCount: number): AstNode[] { - const typeStr = `Map(${typeToString(keyType)}, ${typeToString(valueType)})`; + private decodeUInt8(): AstNode { + const { value, range } = this.reader.readUInt8(); + return { + id: this.generateId(), + type: 'UInt8', + byteRange: range, + value, + displayValue: String(value), + }; + } - // Read cumulative offsets - const offsets: bigint[] = []; - const offsetNodes: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const { value, range } = this.reader.readUInt64LE(); - offsets.push(value); - offsetNodes.push({ - id: this.generateId(), - type: 'ArraySizes', - byteRange: range, - value, - displayValue: `${value} (cumulative)`, - label: `[${i}]`, - }); - } - - // Calculate sizes - const totalEntries = rowCount > 0 ? Number(offsets[rowCount - 1]) : 0; - const sizes: number[] = []; - let prevOffset = 0n; - for (let i = 0; i < rowCount; i++) { - sizes.push(Number(offsets[i] - prevOffset)); - prevOffset = offsets[i]; - } - - // Read all keys - const allKeys = this.decodeColumnData(keyType, totalEntries).values; - - // Read all values - const allValues = this.decodeColumnData(valueType, totalEntries).values; - - // Distribute to maps - const values: AstNode[] = []; - let entryIndex = 0; - for (let i = 0; i < rowCount; i++) { - const size = sizes[i]; - const entries: AstNode[] = []; - - for (let j = 0; j < size; j++) { - const key = allKeys[entryIndex + j]; - const value = allValues[entryIndex + j]; - - key.label = 'key'; - value.label = 'value'; - - entries.push({ - id: this.generateId(), - type: `Tuple(${typeToString(keyType)}, ${typeToString(valueType)})`, - byteRange: { start: key.byteRange.start, end: value.byteRange.end }, - value: [key.value, value.value], - displayValue: `${key.displayValue}: ${value.displayValue}`, - label: `[${j}]`, - children: [key, value], - }); - } + private decodeUInt16(): AstNode { + const { value, range } = this.reader.readUInt16LE(); + return { + id: this.generateId(), + type: 'UInt16', + byteRange: range, + value, + displayValue: String(value), + }; + } - entryIndex += size; - const entriesEnd = entries[entries.length - 1]?.byteRange.end ?? offsetNodes[i].byteRange.end; - const lengthNode: AstNode = { - id: this.generateId(), - type: 'UInt64', - byteRange: offsetNodes[i].byteRange, - value: BigInt(size), - displayValue: `${size} (cumulative: ${offsets[i]})`, - label: 'length', - }; + private decodeUInt32(): AstNode { + const { value, range } = this.reader.readUInt32LE(); + return { + id: this.generateId(), + type: 'UInt32', + byteRange: range, + value, + displayValue: String(value), + }; + } - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: offsetNodes[i].byteRange.start, end: entriesEnd }, - value: Object.fromEntries(entries.map(e => [e.children![0].value, e.children![1].value])), - displayValue: `{${entries.map(e => e.displayValue).join(', ')}}`, - label: `[${i}]`, - children: [lengthNode, ...entries], - metadata: { size }, - }); - } - - return values; - } - - /** - * LowCardinality in Native format: - * 1. DictionaryKeys stream: KeysVersion (UInt64) - * 2. DictionaryIndexes stream: type + dictionary + indexes - */ - private decodeLowCardinalityColumn(innerType: ClickHouseType, rowCount: number): AstNode[] { - const typeStr = `LowCardinality(${typeToString(innerType)})`; - const startOffset = this.reader.offset; - - // Read KeysVersion (should be 1) - this.reader.readUInt64LE(); // keysVersion - not used - - // Read IndexesSerializationType - const { value: serializationType } = this.reader.readUInt64LE(); - - // Extract flags from serialization type - const indexType = Number(serializationType & 0xFFn); - const hasAdditionalKeys = ((serializationType >> 9n) & 1n) === 1n; - // const needGlobalDictionary = ((serializationType >> 8n) & 1n) === 1n; - // const needUpdateDictionary = ((serializationType >> 10n) & 1n) === 1n; - - // Read additional keys (dictionary) - let dictionary: AstNode[] = []; - if (hasAdditionalKeys) { - const { value: numKeys } = this.reader.readUInt64LE(); - - // Determine the actual inner type for decoding (unwrap Nullable if present) - const dictType = innerType.kind === 'Nullable' ? innerType.inner : innerType; - dictionary = this.decodeColumnData(dictType, Number(numKeys)).values; - } - - // Read row count - const { value: numRows } = this.reader.readUInt64LE(); - - // Read indexes - const indexes: number[] = []; - for (let i = 0; i < Number(numRows); i++) { - let idx: number; - switch (indexType) { - case 0: // UInt8 - idx = this.reader.readUInt8().value; - break; - case 1: // UInt16 - idx = this.reader.readUInt16LE().value; - break; - case 2: // UInt32 - idx = this.reader.readUInt32LE().value; - break; - case 3: // UInt64 - idx = Number(this.reader.readUInt64LE().value); - break; - default: - throw new Error(`Unknown LowCardinality index type: ${indexType}`); - } - indexes.push(idx); - } - - // Handle Nullable inner type - index 0 is the null placeholder - const isNullable = innerType.kind === 'Nullable'; - - // Build result values from indexes - // Note: The dictionary includes a placeholder at index 0 (empty/default value for nullable) - // Actual data values start at index 1, so we use direct dictionary lookup - const values: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const idx = indexes[i]; - - if (isNullable && idx === 0) { - // NULL value (index 0 is the null placeholder in dictionary) - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: this.reader.offset }, - value: null, - displayValue: 'NULL', - label: `[${i}]`, - }); - } else { - // Non-null value - direct dictionary lookup (index maps directly to dictionary position) - const dictEntry = dictionary[idx]; - - if (dictEntry) { - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: this.reader.offset }, - value: dictEntry.value, - displayValue: dictEntry.displayValue, - label: `[${i}]`, - metadata: { dictionaryIndex: idx }, - }); - } else { - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: this.reader.offset }, - value: ``, - displayValue: ``, - label: `[${i}]`, - }); - } - } - } - - return values; - } - - /** - * Variant in Native format: - * 1. Discriminators prefix: mode (UInt64) - * 2. Discriminators: N bytes (0xFF = NULL) - * 3. Variant elements: sparse columns for each variant type - */ - private decodeVariantColumn(variants: ClickHouseType[], rowCount: number): AstNode[] { - const typeStr = `Variant(${variants.map(typeToString).join(', ')})`; - const startOffset = this.reader.offset; - - // Read mode (0 = BASIC) - const { value: _mode } = this.reader.readUInt64LE(); - - // Read discriminators - const discriminators: number[] = []; - for (let i = 0; i < rowCount; i++) { - const { value } = this.reader.readUInt8(); - discriminators.push(value); - } - - // Count values per variant type - const countPerVariant: number[] = new Array(variants.length).fill(0); - for (const disc of discriminators) { - if (disc !== 0xFF && disc < variants.length) { - countPerVariant[disc]++; - } - } - - // Read sparse data for each variant - const variantData: AstNode[][] = []; - for (let v = 0; v < variants.length; v++) { - const count = countPerVariant[v]; - if (count > 0) { - variantData[v] = this.decodeColumnData(variants[v], count).values; - } else { - variantData[v] = []; - } - } - - // Track current position in each variant's data - const variantPositions: number[] = new Array(variants.length).fill(0); - - // Build result values - const values: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const disc = discriminators[i]; - - if (disc === 0xFF) { - // NULL - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: this.reader.offset }, - value: null, - displayValue: 'NULL', - label: `[${i}]`, - metadata: { discriminator: disc }, - }); - } else if (disc < variants.length) { - const variantNode = variantData[disc][variantPositions[disc]++]; - values.push({ - id: this.generateId(), - type: typeToString(variants[disc]), - byteRange: variantNode.byteRange, - value: variantNode.value, - displayValue: variantNode.displayValue, - label: `[${i}]`, - children: variantNode.children, - metadata: { discriminator: disc, variantType: typeToString(variants[disc]) }, - }); - } else { - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: this.reader.offset }, - value: ``, - displayValue: ``, - label: `[${i}]`, - }); - } - } - - return values; - } - - /** - * Dynamic in Native format: - * 1. DynamicStructure stream: version + type list - * 2. DynamicData stream: internal Variant data (with extra SharedVariant) - * - * Discriminator encoding: - * - disc 0 to numTypes-1: declared types (0-indexed) - * - disc numTypes: SharedVariant (values of other types) - * - disc 0xFF: NULL - * - * V1 format (version=1): SharedVariant stores values as String representation - * V2 format (version=2): SharedVariant stores values as binary type index + binary value - */ - private decodeDynamicColumn(rowCount: number): AstNode[] { - const startOffset = this.reader.offset; - const headerChildren: AstNode[] = []; - - // Read version (1 = V1, 2 = V2) - const versionStart = this.reader.offset; - const { value: version } = this.reader.readUInt64LE(); - headerChildren.push({ - id: this.generateId(), - type: 'UInt64', - byteRange: { start: versionStart, end: this.reader.offset }, - value: version, - displayValue: version.toString(), - label: 'version', - }); - - // Read max_dynamic_types (V1 only - in V2 this field doesn't exist or has different meaning) - if (version === 1n) { - const maxTypesStart = this.reader.offset; - const { value: maxTypes } = decodeLEB128(this.reader); - headerChildren.push({ - id: this.generateId(), - type: 'VarUInt', - byteRange: { start: maxTypesStart, end: this.reader.offset }, - value: maxTypes, - displayValue: maxTypes.toString(), - label: 'max_dynamic_types', - }); - } - - // Read num_dynamic_types - const numTypesStart = this.reader.offset; - const { value: numTypes } = decodeLEB128(this.reader); - headerChildren.push({ - id: this.generateId(), - type: 'VarUInt', - byteRange: { start: numTypesStart, end: this.reader.offset }, - value: numTypes, - displayValue: numTypes.toString(), - label: 'num_dynamic_types', - }); - - // Read type names - const typeNames: string[] = []; - const typeNamesStart = this.reader.offset; - const typeNameNodes: AstNode[] = []; - for (let i = 0; i < numTypes; i++) { - const nameStart = this.reader.offset; - const { value: len } = decodeLEB128(this.reader); - const { value: bytes } = this.reader.readBytes(len); - const typeName = new TextDecoder().decode(bytes); - typeNames.push(typeName); - typeNameNodes.push({ - id: this.generateId(), - type: 'String', - byteRange: { start: nameStart, end: this.reader.offset }, - value: typeName, - displayValue: `"${typeName}"`, - label: `[${i}]`, - }); - } - if (numTypes > 0) { - headerChildren.push({ - id: this.generateId(), - type: 'Array(String)', - byteRange: { start: typeNamesStart, end: this.reader.offset }, - value: typeNames, - displayValue: `[${typeNames.map(t => `"${t}"`).join(', ')}]`, - label: 'type_names', - children: typeNameNodes, - }); - } - - // Parse types - const variants = typeNames.map(name => parseType(name)); - - // Build discriminator mapping based on alphabetical sort of [typeNames + "SharedVariant"] - // Discriminators are assigned based on alphabetically sorted list, NOT positional indexing - const sortedWithShared = [...typeNames, 'SharedVariant'].sort(); - // discToTypeIndex: maps discriminator -> original type index (-1 for SharedVariant) - const discToTypeIndex = new Map(); - // discToTypeName: maps discriminator -> type name for display - const discToTypeName = new Map(); - for (let idx = 0; idx < sortedWithShared.length; idx++) { - const name = sortedWithShared[idx]; - discToTypeName.set(idx, name); - if (name === 'SharedVariant') { - discToTypeIndex.set(idx, -1); // -1 indicates SharedVariant - } else { - const originalIndex = typeNames.indexOf(name); - discToTypeIndex.set(idx, originalIndex); - } - } - - // Read Variant discriminators prefix (mode) - const modeStart = this.reader.offset; - const { value: mode } = this.reader.readUInt64LE(); - headerChildren.push({ - id: this.generateId(), - type: 'UInt64', - byteRange: { start: modeStart, end: this.reader.offset }, - value: mode, - displayValue: mode.toString(), - label: 'variant_mode', - }); - - // Read discriminators - const discriminatorsStart = this.reader.offset; - const discriminators: number[] = []; - const discNodes: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const discStart = this.reader.offset; - const { value } = this.reader.readUInt8(); - discriminators.push(value); - - // Map discriminator to type name for display (using alphabetical sort mapping) - let discType: string; - if (value === 0xFF) discType = 'NULL'; - else discType = discToTypeName.get(value) ?? `unknown(${value})`; - - discNodes.push({ - id: this.generateId(), - type: 'UInt8', - byteRange: { start: discStart, end: this.reader.offset }, - value, - displayValue: `${value} (${discType})`, - label: `[${i}]`, - }); - } - if (rowCount > 0) { - headerChildren.push({ - id: this.generateId(), - type: 'Array(UInt8)', - byteRange: { start: discriminatorsStart, end: this.reader.offset }, - value: discriminators, - displayValue: `[${discriminators.join(', ')}]`, - label: 'discriminators', - children: discNodes, - }); - } - - const headerEndOffset = this.reader.offset; - - // Count values per variant type (using alphabetical discriminator mapping) - // discToTypeIndex maps: disc -> original type index (-1 for SharedVariant) - // We count per original type index, with SharedVariant at numTypes - const countPerVariant: number[] = new Array(numTypes + 1).fill(0); - for (const disc of discriminators) { - if (disc === 0xFF) continue; // NULL - const typeIdx = discToTypeIndex.get(disc); - if (typeIdx === -1) { - // SharedVariant (stored at index numTypes in countPerVariant) - countPerVariant[numTypes]++; - } else if (typeIdx !== undefined && typeIdx >= 0) { - // Declared type (original index) - countPerVariant[typeIdx]++; - } - } - - // Count values per DISCRIMINATOR (data is serialized in discriminator order, not original type order) - const countPerDiscriminator: number[] = new Array(sortedWithShared.length).fill(0); - for (const disc of discriminators) { - if (disc !== 0xFF && disc < sortedWithShared.length) { - countPerDiscriminator[disc]++; - } - } - - // Read sparse data in DISCRIMINATOR ORDER (alphabetically sorted) - // variantDataByDisc[disc] = array of AstNodes for that discriminator - const variantDataByDisc: AstNode[][] = []; - - for (let disc = 0; disc < sortedWithShared.length; disc++) { - const count = countPerDiscriminator[disc]; - const typeIdx = discToTypeIndex.get(disc); - - if (count === 0) { - variantDataByDisc[disc] = []; - continue; - } - - if (typeIdx === -1) { - // SharedVariant - const sharedVariantData: AstNode[] = []; - if (version === 1n) { - // V1: SharedVariant stores values as length-prefixed blob containing BinaryTypeIndex + binary_value - for (let i = 0; i < count; i++) { - const valueStart = this.reader.offset; - const { value: _len } = decodeLEB128(this.reader); - - // Read the BinaryTypeIndex (single byte) and decode the type - const { value: binTypeIdx } = this.reader.readUInt8(); - const innerType = this.decodeDynamicBinaryTypeExtended(binTypeIdx); - const innerValue = this.decodeValue(innerType); - - sharedVariantData.push({ - id: this.generateId(), - type: typeToString(innerType), - byteRange: { start: valueStart, end: this.reader.offset }, - value: innerValue.value, - displayValue: innerValue.displayValue, - children: innerValue.children, - metadata: { isSharedVariant: true, binaryTypeIndex: binTypeIdx, serializationVersion: 1 }, - }); - } - } else { - // V2: SharedVariant stores values as binary type index + binary value - for (let i = 0; i < count; i++) { - const valueStart = this.reader.offset; - const { value: binTypeIdx } = decodeLEB128(this.reader); - const innerType = this.decodeDynamicBinaryType(binTypeIdx); - const innerValue = this.decodeValue(innerType); - - sharedVariantData.push({ - id: this.generateId(), - type: typeToString(innerType), - byteRange: { start: valueStart, end: this.reader.offset }, - value: innerValue.value, - displayValue: innerValue.displayValue, - children: innerValue.children, - metadata: { isSharedVariant: true, binaryTypeIndex: binTypeIdx, serializationVersion: 2 }, - }); - } - } - variantDataByDisc[disc] = sharedVariantData; - } else if (typeIdx !== undefined) { - // Declared type - variantDataByDisc[disc] = this.decodeColumnData(variants[typeIdx], count).values; - } - } - - // Track current position in each discriminator's data (indexed by discriminator) - const variantPositions: number[] = new Array(sortedWithShared.length).fill(0); - - // Build result values (using discriminator to index directly into variantDataByDisc) - const values: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const disc = discriminators[i]; - - if (disc === 0xFF) { - // NULL - values.push({ - id: this.generateId(), - type: 'Dynamic(NULL)', - byteRange: { start: startOffset, end: this.reader.offset }, - value: null, - displayValue: 'NULL', - label: `[${i}]`, - metadata: { discriminator: disc, actualType: 'NULL' }, - }); - } else if (disc < sortedWithShared.length) { - const typeIdx = discToTypeIndex.get(disc); - const isSharedVariant = typeIdx === -1; - const variantNode = variantDataByDisc[disc][variantPositions[disc]++]; - const actualType = variantNode.type; - - variantNode.label = 'value'; - - values.push({ - id: this.generateId(), - type: `Dynamic(${actualType})`, - byteRange: variantNode.byteRange, - value: variantNode.value, - displayValue: variantNode.displayValue, - label: `[${i}]`, - children: [variantNode], - metadata: { - discriminator: disc, - actualType, - isSharedVariant, - serializationVersion: Number(version), - }, - }); - } else { - values.push({ - id: this.generateId(), - type: 'Dynamic', - byteRange: { start: startOffset, end: this.reader.offset }, - value: ``, - displayValue: ``, - label: `[${i}]`, - }); - } - } - - // Create header node that contains all the Dynamic column metadata - const headerNode: AstNode = { - id: this.generateId(), - type: 'Dynamic.Header', - byteRange: { start: startOffset, end: headerEndOffset }, - value: { version: Number(version), numTypes, typeNames }, - displayValue: `Dynamic(${numTypes} types, v${version})`, - label: 'header', - children: headerChildren, - }; - - // Return header followed by values - return [headerNode, ...values]; - } - - /** - * Decode binary type index used in Dynamic's SharedVariant (V2 format) - */ - private decodeDynamicBinaryType(typeIdx: number): ClickHouseType { - // Common binary type indexes from ClickHouse BinaryTypeIndex enum - const typeMap: Record = { - 0: { kind: 'String' } as ClickHouseType, // Nothing maps to String as fallback - 1: { kind: 'UInt8' }, - 2: { kind: 'UInt16' }, - 3: { kind: 'UInt32' }, - 4: { kind: 'UInt64' }, - 5: { kind: 'UInt128' }, - 6: { kind: 'UInt256' }, - 7: { kind: 'Int8' }, - 8: { kind: 'Int16' }, - 9: { kind: 'Int32' }, - 10: { kind: 'Int64' }, - 11: { kind: 'Int128' }, - 12: { kind: 'Int256' }, - 13: { kind: 'Float32' }, - 14: { kind: 'Float64' }, - 15: { kind: 'Date' }, - 16: { kind: 'Date32' }, - 17: { kind: 'DateTime' }, - // 18: DateTime64 with precision - handled specially - 19: { kind: 'String' }, - 20: { kind: 'UUID' }, - 21: { kind: 'IPv4' }, - 22: { kind: 'IPv6' }, - 23: { kind: 'Bool' }, - // 24+: Complex types that need additional parsing - }; - - const type = typeMap[typeIdx]; - if (type) { - return type; - } - - // For unknown types, return String as fallback - return { kind: 'String' }; - } - - /** - * Extended binary type index decoder that handles all ClickHouse types - * Used for SharedVariant decoding where complex types may appear - */ - private decodeDynamicBinaryTypeExtended(typeIdx: number): ClickHouseType { - switch (typeIdx) { - case 0x00: return { kind: 'String' }; // Nothing - fallback to String - case 0x01: return { kind: 'UInt8' }; - case 0x02: return { kind: 'UInt16' }; - case 0x03: return { kind: 'UInt32' }; - case 0x04: return { kind: 'UInt64' }; - case 0x05: return { kind: 'UInt128' }; - case 0x06: return { kind: 'UInt256' }; - case 0x07: return { kind: 'Int8' }; - case 0x08: return { kind: 'Int16' }; - case 0x09: return { kind: 'Int32' }; - case 0x0a: return { kind: 'Int64' }; - case 0x0b: return { kind: 'Int128' }; - case 0x0c: return { kind: 'Int256' }; - case 0x0d: return { kind: 'Float32' }; - case 0x0e: return { kind: 'Float64' }; - case 0x0f: return { kind: 'Date' }; - case 0x10: return { kind: 'Date32' }; - case 0x11: return { kind: 'DateTime' }; - case 0x12: { - // DateTime with timezone - const { value: tzLen } = decodeLEB128(this.reader); - const { value: tzBytes } = this.reader.readBytes(tzLen); - const timezone = new TextDecoder().decode(tzBytes); - return { kind: 'DateTime', timezone }; - } - case 0x13: { - // DateTime64 - const { value: precision } = this.reader.readUInt8(); - return { kind: 'DateTime64', precision }; - } - case 0x14: { - // DateTime64 with timezone - const { value: precision } = this.reader.readUInt8(); - const { value: tzLen } = decodeLEB128(this.reader); - const { value: tzBytes } = this.reader.readBytes(tzLen); - const timezone = new TextDecoder().decode(tzBytes); - return { kind: 'DateTime64', precision, timezone }; - } - case 0x15: return { kind: 'String' }; - case 0x16: { - // FixedString - const { value: length } = decodeLEB128(this.reader); - return { kind: 'FixedString', length }; - } - case 0x1d: return { kind: 'UUID' }; - case 0x1e: { - // Array - const { value: elemTypeIdx } = this.reader.readUInt8(); - const element = this.decodeDynamicBinaryTypeExtended(elemTypeIdx); - return { kind: 'Array', element }; - } - case 0x1f: { - // Tuple (unnamed) - const { value: count } = decodeLEB128(this.reader); - const elements: ClickHouseType[] = []; - for (let i = 0; i < count; i++) { - const { value: elemTypeIdx } = this.reader.readUInt8(); - const elem = this.decodeDynamicBinaryTypeExtended(elemTypeIdx); - elements.push(elem); - } - return { kind: 'Tuple', elements }; - } - case 0x20: { - // Named Tuple - const { value: count } = decodeLEB128(this.reader); - const elements: ClickHouseType[] = []; - const names: string[] = []; - for (let i = 0; i < count; i++) { - const { value: nameLen } = decodeLEB128(this.reader); - const { value: nameBytes } = this.reader.readBytes(nameLen); - names.push(new TextDecoder().decode(nameBytes)); - const { value: elemTypeIdx } = this.reader.readUInt8(); - const elem = this.decodeDynamicBinaryTypeExtended(elemTypeIdx); - elements.push(elem); - } - return { kind: 'Tuple', elements, names }; - } - case 0x23: { - // Nullable - const { value: innerTypeIdx } = this.reader.readUInt8(); - const inner = this.decodeDynamicBinaryTypeExtended(innerTypeIdx); - return { kind: 'Nullable', inner }; - } - case 0x27: { - // Map - const { value: keyTypeIdx } = this.reader.readUInt8(); - const key = this.decodeDynamicBinaryTypeExtended(keyTypeIdx); - const { value: valueTypeIdx } = this.reader.readUInt8(); - const value = this.decodeDynamicBinaryTypeExtended(valueTypeIdx); - return { kind: 'Map', key, value }; - } - case 0x28: return { kind: 'IPv4' }; - case 0x29: return { kind: 'IPv6' }; - case 0x2d: return { kind: 'Bool' }; - default: - // Fallback to String for unknown types - return { kind: 'String' }; - } - } - - /** - * JSON in Native format - * - * Actual format discovered through debugging: - * 1. max_dynamic_paths (UInt64) - typically 0 - * 2. typed_paths_count (LEB128) - number of dynamic paths in this column - * 3. columns_count (LEB128) - same as typed_paths_count - * 4. Path names (String for each) - * 5. Column info for each column: - * - offset (UInt64) - * - serialization_kind (UInt16) - * - type_name (String) - * - metadata (UInt64) - * 6. TYPED SUB-COLUMN VALUES (if JSON type has typed paths like JSON(a Int32)) - * - These are raw values without flag bytes - * 7. Dynamic column values (flag byte + value for each column/row) - * 8. Shared data offsets (UInt64 per row) - * - * The resulting AST includes all structural elements as children. - */ - private decodeJSONColumn(type: ClickHouseType, rowCount: number): AstNode[] { - const startOffset = this.reader.offset; - const values: AstNode[] = []; - - // Get JSON type info - const jsonType = type as { kind: 'JSON'; typedPaths?: Map; maxDynamicPaths?: number }; - const typedSubColumns = jsonType.typedPaths; - - // Always use the V1 format decoder - it handles all JSON types correctly - // The format is: version + max_dynamic_paths + num_dynamic_paths + path_names + - // dynamic_structures + typed_path_data + dynamic_data + shared_offsets - return this.decodeJSONColumnV1(type, rowCount, startOffset); - - // Legacy format: reads version as max_dynamic_paths (coincidentally works when version=0) - const maxDynPathsNode = this.decodeUInt64(); - maxDynPathsNode.label = 'max_dynamic_paths'; - - // Read typed_paths_count with AST node - const typedPathsCountStart = this.reader.offset; - const { value: typedPathsCount } = decodeLEB128(this.reader); - const typedPathsCountNode: AstNode = { - id: this.generateId(), - type: 'VarUInt', - byteRange: { start: typedPathsCountStart, end: this.reader.offset }, - value: typedPathsCount, - displayValue: String(typedPathsCount), - label: 'typed_paths_count', - }; - - // Handle JSON with no dynamic paths - if (typedPathsCount === 0) { - // If there are typed sub-columns, read them - const hasTypedPaths = typedSubColumns !== undefined && typedSubColumns!.size > 0; - if (hasTypedPaths) { - const typedPaths = typedSubColumns!; - // For fully-typed JSON: flag byte + typed values + shared offset - // Collect nodes per row: [flagNode, ...pathValueNodes] - const rowData: { flagNode: AstNode; pathNodes: Map }[] = []; - - for (let row = 0; row < rowCount; row++) { - // Read flag byte (0 = object present) - const flagNode = this.decodeUInt8(); - flagNode.label = 'object_present'; - - const pathNodes = new Map(); - for (const [pathName, pathType] of typedPaths) { - const node = this.decodeValue(pathType); - pathNodes.set(pathName, node); - } - - rowData.push({ flagNode, pathNodes }); - } - - // Read shared_data_offsets - const sharedOffsetNodes: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const node = this.decodeUInt64(); - node.label = `shared_data_offset[${i}]`; - sharedOffsetNodes.push(node); - } - - // Build result nodes with all structural AST children - for (let row = 0; row < rowCount; row++) { - const children: AstNode[] = []; - const jsonValue: Record = {}; - - // Add structural nodes - children.push(maxDynPathsNode); - children.push(typedPathsCountNode); - children.push(rowData[row].flagNode); - - // Add path nodes - for (const [pathName] of typedPaths) { - const valueNode = rowData[row].pathNodes.get(pathName)!; - jsonValue[pathName] = valueNode.value; - - children.push({ - id: this.generateId(), - type: 'JSON path', - byteRange: valueNode.byteRange, - value: { [pathName]: valueNode.value }, - displayValue: `${pathName}: ${valueNode.displayValue}`, - label: pathName, - children: [valueNode], - }); - } - - // Add shared offset node - children.push(sharedOffsetNodes[row]); - - values.push({ - id: this.generateId(), - type: 'JSON', - byteRange: { start: startOffset, end: this.reader.offset }, - value: jsonValue, - displayValue: `{${typedPaths.size} paths}`, - label: `[${row}]`, - children, - }); - } - - return values; - } - - // No typed sub-columns either - return empty JSON objects - // Read shared_data_offsets - const sharedOffsetNodes: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const node = this.decodeUInt64(); - node.label = `shared_data_offset[${i}]`; - sharedOffsetNodes.push(node); - } - - for (let i = 0; i < rowCount; i++) { - values.push({ - id: this.generateId(), - type: 'JSON', - byteRange: { start: startOffset, end: this.reader.offset }, - value: {}, - displayValue: '{0 paths}', - label: `[${i}]`, - children: [maxDynPathsNode, typedPathsCountNode, sharedOffsetNodes[i]], - }); - } - return values; - } - - // Read columns_count with AST node - const columnsCountStart = this.reader.offset; - const { value: columnsCount } = decodeLEB128(this.reader); - const columnsCountNode: AstNode = { - id: this.generateId(), - type: 'VarUInt', - byteRange: { start: columnsCountStart, end: this.reader.offset }, - value: columnsCount, - displayValue: String(columnsCount), - label: 'columns_count', - }; - - // Read path names with AST nodes - const pathNameNodes: AstNode[] = []; - const pathNames: string[] = []; - for (let i = 0; i < typedPathsCount; i++) { - const pathStart = this.reader.offset; - const { value: nameLen } = decodeLEB128(this.reader); - const { value: nameBytes } = this.reader.readBytes(nameLen); - const pathName = new TextDecoder().decode(nameBytes); - pathNames.push(pathName); - - pathNameNodes.push({ - id: this.generateId(), - type: 'String', - byteRange: { start: pathStart, end: this.reader.offset }, - value: pathName, - displayValue: `"${pathName}"`, - label: `path_name[${i}]`, - }); - } - - // Read column info with AST nodes - const columnInfoNodes: AstNode[] = []; - const columnInfos: { typeName: string }[] = []; - for (let i = 0; i < columnsCount; i++) { - const infoStart = this.reader.offset; - - const offsetNode = this.decodeUInt64(); - offsetNode.label = 'offset'; - - const kindNode = this.decodeUInt16(); - kindNode.label = 'serialization_kind'; - - const typeStart = this.reader.offset; - const { value: typeLen } = decodeLEB128(this.reader); - const { value: typeBytes } = this.reader.readBytes(typeLen); - const typeName = new TextDecoder().decode(typeBytes); - const typeNameNode: AstNode = { - id: this.generateId(), - type: 'String', - byteRange: { start: typeStart, end: this.reader.offset }, - value: typeName, - displayValue: `"${typeName}"`, - label: 'type', - }; - - const metadataNode = this.decodeUInt64(); - metadataNode.label = 'metadata'; - - columnInfos.push({ typeName }); - columnInfoNodes.push({ - id: this.generateId(), - type: 'column_info', - byteRange: { start: infoStart, end: this.reader.offset }, - value: { offset: offsetNode.value, kind: kindNode.value, type: typeName, metadata: metadataNode.value }, - displayValue: `${pathNames[i]}: ${typeName}`, - label: `column_info[${i}]`, - children: [offsetNode, kindNode, typeNameNode, metadataNode], - }); - } - - // Read typed sub-column values first (if any) - collect AstNodes per path - const typedPathNodes: Map = new Map(); - const hasTypedSubCols = typedSubColumns !== undefined && typedSubColumns!.size > 0; - if (hasTypedSubCols) { - for (const [pathName, pathType] of typedSubColumns!) { - const nodes: AstNode[] = []; - for (let row = 0; row < rowCount; row++) { - nodes.push(this.decodeValue(pathType)); - } - typedPathNodes.set(pathName, nodes); - } - } - - // Read dynamic column values (flag byte + value for each column/row) - collect AstNodes - const dynamicPathData: { flagNodes: AstNode[]; valueNodes: AstNode[] }[] = []; - for (let colIdx = 0; colIdx < columnsCount; colIdx++) { - const { typeName } = columnInfos[colIdx]; - const colType = parseType(typeName); - const flagNodes: AstNode[] = []; - const valueNodes: AstNode[] = []; - - for (let row = 0; row < rowCount; row++) { - const flagNode = this.decodeUInt8(); - flagNode.label = 'flag'; - flagNodes.push(flagNode); - valueNodes.push(this.decodeValue(colType)); - } - - dynamicPathData.push({ flagNodes, valueNodes }); - } - - // Read shared_data_offsets - const sharedOffsetNodes: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const node = this.decodeUInt64(); - node.label = `shared_data_offset[${i}]`; - sharedOffsetNodes.push(node); - } - - // Build result nodes with all structural AST children - for (let row = 0; row < rowCount; row++) { - const children: AstNode[] = []; - const jsonValue: Record = {}; - - // Add header structural nodes (shared across rows) - children.push(maxDynPathsNode); - children.push(typedPathsCountNode); - children.push(columnsCountNode); - - // Add path name nodes - for (const node of pathNameNodes) { - children.push(node); - } - - // Add column info nodes - for (const node of columnInfoNodes) { - children.push(node); - } - - // Add typed sub-column path nodes - for (const [pathName, nodes] of typedPathNodes) { - const valueNode = nodes[row]; - this.setNestedValue(jsonValue, pathName, valueNode.value); - - children.push({ - id: this.generateId(), - type: 'JSON path', - byteRange: valueNode.byteRange, - value: { [pathName]: valueNode.value }, - displayValue: `${pathName}: ${valueNode.displayValue}`, - label: pathName, - children: [valueNode], - }); - } - - // Add dynamic path nodes (with flag + value) - for (let colIdx = 0; colIdx < columnsCount; colIdx++) { - const pathName = pathNames[colIdx]; - const { flagNodes, valueNodes } = dynamicPathData[colIdx]; - const flagNode = flagNodes[row]; - const valueNode = valueNodes[row]; - - this.setNestedValue(jsonValue, pathName, valueNode.value); - - children.push({ - id: this.generateId(), - type: 'JSON path', - byteRange: { start: flagNode.byteRange.start, end: valueNode.byteRange.end }, - value: { [pathName]: valueNode.value }, - displayValue: `${pathName}: ${valueNode.displayValue}`, - label: pathName, - children: [flagNode, valueNode], - }); - } - - // Add shared offset node - children.push(sharedOffsetNodes[row]); - - const totalPaths = typedPathNodes.size + dynamicPathData.length; - values.push({ - id: this.generateId(), - type: 'JSON', - byteRange: { start: startOffset, end: this.reader.offset }, - value: jsonValue, - displayValue: `{${totalPaths} paths}`, - label: `[${row}]`, - children, - }); - } - - return values; - } - - /** - * Read the state prefix for a column type, if it needs one. - * In Native format, complex types like JSON write their structure as a prefix - * before the actual data. This must be read and stored for later use. - */ - private readColumnPrefix(type: ClickHouseType): unknown { - if (type.kind === 'JSON') { - const jsonType = type as { kind: 'JSON'; typedPaths?: Map }; - return this.readJSONColumnStructure(jsonType.typedPaths); - } - if (type.kind === 'Array') { - // For arrays, recursively check if element needs a prefix - return this.readColumnPrefix(type.element); - } - // Other types don't need prefixes - return null; - } - - /** - * Decode column data, using a pre-read prefix if available. - */ - private decodeColumnWithPrefix(type: ClickHouseType, rowCount: number, prefix: unknown): AstNode[] { - if (type.kind === 'JSON' && prefix) { - return this.readJSONColumnData( - prefix as ReturnType, - rowCount, - this.reader.offset - ); - } - if (type.kind === 'Array' && prefix) { - // For Array with prefix, the prefix is for the element type - return this.decodeArrayColumnWithPrefix(type.element, rowCount, prefix); - } - // No prefix - use normal decoding - return this.decodeColumnData(type, rowCount).values; - } - - /** - * Decode array column where the element type has a pre-read prefix. - */ - private decodeArrayColumnWithPrefix(elementType: ClickHouseType, rowCount: number, elementPrefix: unknown): AstNode[] { - const typeStr = `Array(${typeToString(elementType)})`; - const startOffset = this.reader.offset; - - // Read array offsets - const offsetNodes: AstNode[] = []; - const offsets: bigint[] = []; - for (let i = 0; i < rowCount; i++) { - const start = this.reader.offset; - const { value } = this.reader.readUInt64LE(); - offsets.push(value); - offsetNodes.push({ - id: this.generateId(), - type: 'UInt64', - byteRange: { start, end: this.reader.offset }, - value, - displayValue: String(value), - label: 'array_offset', - }); - } - - // Calculate total elements - const totalElements = rowCount > 0 ? Number(offsets[rowCount - 1]) : 0; - const sizes: number[] = []; - let prevOffset = 0n; - for (let i = 0; i < rowCount; i++) { - sizes.push(Number(offsets[i] - prevOffset)); - prevOffset = offsets[i]; - } - - // Decode elements using the prefix - const allElements = this.decodeColumnWithPrefix(elementType, totalElements, elementPrefix); - - // Distribute to arrays - const values: AstNode[] = []; - let elementIndex = 0; - for (let i = 0; i < rowCount; i++) { - const size = sizes[i]; - const arrayElements = allElements.slice(elementIndex, elementIndex + size); - elementIndex += size; - - arrayElements.forEach((el, j) => { - el.label = `[${j}]`; - }); - - const lengthNode: AstNode = { - id: this.generateId(), - type: 'UInt64', - byteRange: offsetNodes[i].byteRange, - value: BigInt(size), - displayValue: `${size} (cumulative: ${offsets[i]})`, - label: 'length', - }; - - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: this.reader.offset }, - value: arrayElements.map(e => e.value), - displayValue: `[${arrayElements.length} elements]`, - label: `[${i}]`, - children: [lengthNode, ...arrayElements], - metadata: { size }, - }); - } - - return values; - } - - /** - * Read JSON column structure (version, paths, dynamic structures). - * Used by decodeJSONColumnV1 and via readColumnPrefix for nested JSON in Arrays/Variants. - */ - private readJSONColumnStructure(typedSubColumns?: Map): { - serializationVersion: number; - structureChildren: AstNode[]; - dynamicPathNames: string[]; - dynamicStructures: Array<{ - serializationVersion: number; - typeNames: string[]; - variants: ClickHouseType[]; - numTypes: number; - discToTypeIndex: Map; - variantPrefixes: unknown[]; - }>; - typedSubColumns?: Map; - } { - const structureChildren: AstNode[] = []; - - // 1. Read version (UInt64) - const versionNode = this.decodeUInt64(); - versionNode.label = 'version'; - structureChildren.push(versionNode); - const objectSerializationVersion = Number(versionNode.value); + private decodeUInt64(): AstNode { + const { value, range } = this.reader.readUInt64LE(); + return { + id: this.generateId(), + type: 'UInt64', + byteRange: range, + value, + displayValue: value.toString(), + }; + } - if (objectSerializationVersion !== 0 && objectSerializationVersion !== 2) { - throw new Error( - `Unsupported JSON object serialization version ${objectSerializationVersion} in Native decoder`, - ); - } + private decodeUInt128(): AstNode { + const { value, range } = this.reader.readUInt128LE(); + return { + id: this.generateId(), + type: 'UInt128', + byteRange: range, + value, + displayValue: value.toString(), + }; + } - // 2. Read max_dynamic_paths (V1 only) - if (objectSerializationVersion === 0) { - const maxDynPathsStart = this.reader.offset; - const { value: maxDynamicPaths } = decodeLEB128(this.reader); - const maxDynPathsNode: AstNode = { - id: this.generateId(), - type: 'VarUInt', - byteRange: { start: maxDynPathsStart, end: this.reader.offset }, - value: maxDynamicPaths, - displayValue: String(maxDynamicPaths), - label: 'max_dynamic_paths', - }; - structureChildren.push(maxDynPathsNode); - } + private decodeUInt256(): AstNode { + const { value, range } = this.reader.readUInt256LE(); + return { + id: this.generateId(), + type: 'UInt256', + byteRange: range, + value, + displayValue: value.toString(), + }; + } - // 3. Read num_dynamic_paths (VarUInt) - const numDynPathsStart = this.reader.offset; - const { value: numDynamicPaths } = decodeLEB128(this.reader); - const numDynPathsNode: AstNode = { + private decodeInt8(): AstNode { + const { value, range } = this.reader.readInt8(); + return { id: this.generateId(), - type: 'VarUInt', - byteRange: { start: numDynPathsStart, end: this.reader.offset }, - value: numDynamicPaths, - displayValue: String(numDynamicPaths), - label: 'num_dynamic_paths', - }; - structureChildren.push(numDynPathsNode); - - // 4. Read sorted dynamic path names - const dynamicPathNames: string[] = []; - for (let i = 0; i < numDynamicPaths; i++) { - const pathNode = this.decodeString(); - pathNode.label = `dynamic_path[${i}]`; - structureChildren.push(pathNode); - dynamicPathNames.push(pathNode.value as string); - } - - // 5. Read ALL Dynamic structures (one per dynamic path) + type: 'Int8', + byteRange: range, + value, + displayValue: String(value), + }; + } + + private decodeInt16(): AstNode { + const { value, range } = this.reader.readInt16LE(); + return { + id: this.generateId(), + type: 'Int16', + byteRange: range, + value, + displayValue: String(value), + }; + } + + private decodeInt32(): AstNode { + const { value, range } = this.reader.readInt32LE(); + return { + id: this.generateId(), + type: 'Int32', + byteRange: range, + value, + displayValue: String(value), + }; + } + + private decodeInt64(): AstNode { + const { value, range } = this.reader.readInt64LE(); + return { + id: this.generateId(), + type: 'Int64', + byteRange: range, + value, + displayValue: value.toString(), + }; + } + + private decodeInt128(): AstNode { + const { value, range } = this.reader.readInt128LE(); + return { + id: this.generateId(), + type: 'Int128', + byteRange: range, + value, + displayValue: value.toString(), + }; + } + + private decodeInt256(): AstNode { + const { value, range } = this.reader.readInt256LE(); + return { + id: this.generateId(), + type: 'Int256', + byteRange: range, + value, + displayValue: value.toString(), + }; + } + + // Float decoders + private decodeFloat32(): AstNode { + const { value, range } = this.reader.readFloat32LE(); + return { + id: this.generateId(), + type: 'Float32', + byteRange: range, + value, + displayValue: value.toString(), + }; + } + + private decodeFloat64(): AstNode { + const { value, range } = this.reader.readFloat64LE(); + return { + id: this.generateId(), + type: 'Float64', + byteRange: range, + value, + displayValue: value.toString(), + }; + } + + private decodeBFloat16(): AstNode { + const { value, range } = this.reader.readBFloat16LE(); + return { + id: this.generateId(), + type: 'BFloat16', + byteRange: range, + value, + displayValue: value.toString(), + }; + } + + // String decoders + private decodeString(): AstNode { + const startOffset = this.reader.offset; + const { value: length } = decodeLEB128(this.reader); + const { value: bytes } = this.reader.readBytes(length); + const str = new TextDecoder().decode(bytes); + + return { + id: this.generateId(), + type: 'String', + byteRange: { start: startOffset, end: this.reader.offset }, + value: str, + displayValue: `"${str}"`, + }; + } + + private decodeFixedString(length: number): AstNode { + const { value: bytes, range } = this.reader.readBytes(length); + // Find first null byte to get actual string length + let actualLength = length; + for (let i = 0; i < bytes.length; i++) { + if (bytes[i] === 0) { + actualLength = i; + break; + } + } + const str = new TextDecoder().decode(bytes.slice(0, actualLength)); + + return { + id: this.generateId(), + type: `FixedString(${length})`, + byteRange: range, + value: str, + displayValue: `"${str}"`, + metadata: { fixedLength: length, actualLength }, + }; + } + + // Nothing decoder — consumes one placeholder byte; the value is always null + private decodeNothing(): AstNode { + const { range } = this.reader.readUInt8(); + return { + id: this.generateId(), + type: 'Nothing', + byteRange: range, + value: null, + displayValue: 'ø', + }; + } + + // Bool decoder + private decodeBool(): AstNode { + const { value, range } = this.reader.readUInt8(); + return { + id: this.generateId(), + type: 'Bool', + byteRange: range, + value: value !== 0, + displayValue: value !== 0 ? 'true' : 'false', + }; + } + + // Date/Time decoders + private decodeDate(): AstNode { + const { value, range } = this.reader.readUInt16LE(); + const date = new Date(value * 24 * 60 * 60 * 1000); + return { + id: this.generateId(), + type: 'Date', + byteRange: range, + value: date, + displayValue: date.toISOString().split('T')[0], + metadata: { daysSinceEpoch: value }, + }; + } + + private decodeDate32(): AstNode { + const { value, range } = this.reader.readInt32LE(); + const date = new Date(value * 24 * 60 * 60 * 1000); + return { + id: this.generateId(), + type: 'Date32', + byteRange: range, + value: date, + displayValue: date.toISOString().split('T')[0], + metadata: { daysSinceEpoch: value }, + }; + } + + private decodeDateTime(timezone?: string): AstNode { + const { value, range } = this.reader.readUInt32LE(); + const date = new Date(value * 1000); + return { + id: this.generateId(), + type: timezone ? `DateTime('${timezone}')` : 'DateTime', + byteRange: range, + value: date, + displayValue: date.toISOString().replace('T', ' ').replace('Z', ''), + metadata: { secondsSinceEpoch: value, timezone }, + }; + } + + private decodeDateTime64(precision: number, timezone?: string): AstNode { + const { value, range } = this.reader.readInt64LE(); + const divisor = BigInt(Math.pow(10, precision)); + const seconds = Number(value / divisor); + const subseconds = Number(value % divisor); + const date = new Date(seconds * 1000 + subseconds / Math.pow(10, precision - 3)); + + return { + id: this.generateId(), + type: timezone ? `DateTime64(${precision}, '${timezone}')` : `DateTime64(${precision})`, + byteRange: range, + value: date, + displayValue: date.toISOString().replace('T', ' ').replace('Z', ''), + metadata: { ticksSinceEpoch: value.toString(), precision, timezone }, + }; + } + + private decodeTime(): AstNode { + const { value, range } = this.reader.readInt32LE(); + const sign = value < 0 ? '-' : ''; + const absValue = Math.abs(value); + const hours = Math.floor(absValue / 3600); + const minutes = Math.floor((absValue % 3600) / 60); + const seconds = absValue % 60; + + return { + id: this.generateId(), + type: 'Time', + byteRange: range, + value, + displayValue: `${sign}${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`, + metadata: { totalSeconds: value }, + }; + } + + private decodeTime64(precision: number): AstNode { + const { value, range } = this.reader.readInt64LE(); + const divisor = BigInt(Math.pow(10, precision)); + const totalSeconds = Number(value / divisor); + const subseconds = Number(value % divisor); + + const sign = totalSeconds < 0 ? '-' : ''; + const absSeconds = Math.abs(totalSeconds); + const hours = Math.floor(absSeconds / 3600); + const minutes = Math.floor((absSeconds % 3600) / 60); + const seconds = absSeconds % 60; + + return { + id: this.generateId(), + type: `Time64(${precision})`, + byteRange: range, + value: value.toString(), + displayValue: `${sign}${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}.${Math.abs(subseconds).toString().padStart(precision, '0')}`, + metadata: { precision, rawValue: value.toString() }, + }; + } + + // Special type decoders + private decodeUUID(): AstNode { + const { value: bytes, range } = this.reader.readBytes(16); + + // ClickHouse UUID has special byte ordering + const hex = (b: number) => b.toString(16).padStart(2, '0'); + + const uuid = [ + hex(bytes[7]), + hex(bytes[6]), + hex(bytes[5]), + hex(bytes[4]), + '-', + hex(bytes[3]), + hex(bytes[2]), + '-', + hex(bytes[1]), + hex(bytes[0]), + '-', + hex(bytes[15]), + hex(bytes[14]), + '-', + hex(bytes[13]), + hex(bytes[12]), + hex(bytes[11]), + hex(bytes[10]), + hex(bytes[9]), + hex(bytes[8]), + ].join(''); + + return { + id: this.generateId(), + type: 'UUID', + byteRange: range, + value: uuid, + displayValue: uuid, + }; + } + + private decodeIPv4(): AstNode { + const { value: bytes, range } = this.reader.readBytes(4); + // IPv4 stored as little-endian UInt32 + const ip = `${bytes[3]}.${bytes[2]}.${bytes[1]}.${bytes[0]}`; + + return { + id: this.generateId(), + type: 'IPv4', + byteRange: range, + value: ip, + displayValue: ip, + }; + } + + private decodeIPv6(): AstNode { + const { value: bytes, range } = this.reader.readBytes(16); + const groups: number[] = []; + for (let i = 0; i < 16; i += 2) { + groups.push((bytes[i] << 8) | bytes[i + 1]); + } + const ip = formatIPv6(groups); + + return { + id: this.generateId(), + type: 'IPv6', + byteRange: range, + value: ip, + displayValue: ip, + }; + } + + // Decimal decoders + private decodeDecimal32(scale: number): AstNode { + const { value, range } = this.reader.readInt32LE(); + const scaleFactor = Math.pow(10, scale); + const decoded = value / scaleFactor; + return { + id: this.generateId(), + type: `Decimal32(${scale})`, + byteRange: range, + value: decoded, + displayValue: decoded.toFixed(scale), + metadata: { scale, rawValue: value }, + }; + } + + private decodeDecimal64(scale: number): AstNode { + const { value, range } = this.reader.readInt64LE(); + const scaleFactor = BigInt(Math.pow(10, scale)); + const wholePart = value / scaleFactor; + const fracPart = value % scaleFactor; + const decoded = Number(wholePart) + Number(fracPart) / Number(scaleFactor); + return { + id: this.generateId(), + type: `Decimal64(${scale})`, + byteRange: range, + value: decoded, + displayValue: decoded.toFixed(scale), + metadata: { scale, rawValue: value.toString() }, + }; + } + + private decodeDecimal128(scale: number): AstNode { + const { value, range } = this.reader.readInt128LE(); + const scaleFactor = 10n ** BigInt(scale); + const wholePart = value / scaleFactor; + const fracPart = value >= 0n ? value % scaleFactor : -((-value) % scaleFactor); + return { + id: this.generateId(), + type: `Decimal128(${scale})`, + byteRange: range, + value: value.toString(), + displayValue: `${wholePart}.${fracPart.toString().padStart(scale, '0')}`, + metadata: { scale, rawValue: value.toString() }, + }; + } + + private decodeDecimal256(scale: number): AstNode { + const { value, range } = this.reader.readInt256LE(); + const scaleFactor = 10n ** BigInt(scale); + const wholePart = value / scaleFactor; + const fracPart = value >= 0n ? value % scaleFactor : -((-value) % scaleFactor); + return { + id: this.generateId(), + type: `Decimal256(${scale})`, + byteRange: range, + value: value.toString(), + displayValue: `${wholePart}.${fracPart.toString().padStart(scale, '0')}`, + metadata: { scale, rawValue: value.toString() }, + }; + } + + // Enum decoders + private decodeEnum8(values: Map): AstNode { + const { value, range } = this.reader.readInt8(); + const name = values.get(value) ?? ``; + + return { + id: this.generateId(), + type: 'Enum8', + byteRange: range, + value, + displayValue: `'${name}'`, + metadata: { enumValue: value, enumName: name }, + }; + } + + private decodeEnum16(values: Map): AstNode { + const { value, range } = this.reader.readUInt16LE(); + const name = values.get(value) ?? ``; + + return { + id: this.generateId(), + type: 'Enum16', + byteRange: range, + value, + displayValue: `'${name}'`, + metadata: { enumValue: value, enumName: name }, + }; + } + + // Tuple decoder for single value (used in nested contexts) + private decodeTuple(elements: ClickHouseType[], names?: string[]): AstNode { + const startOffset = this.reader.offset; + const children: AstNode[] = []; + + for (let i = 0; i < elements.length; i++) { + const label = names?.[i] ?? `[${i}]`; + const child = this.decodeValue(elements[i]); + child.label = label; + children.push(child); + } + + const typeStr = names + ? `Tuple(${elements.map((e, i) => `${names[i]} ${typeToString(e)}`).join(', ')})` + : `Tuple(${elements.map(typeToString).join(', ')})`; + + return { + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: this.reader.offset }, + value: children.map((c) => c.value), + displayValue: `(${children.map((c) => c.displayValue).join(', ')})`, + children, + }; + } + + /** + * Tuple in Native format (columnar): + * Each element stream is written sequentially for all rows + * - For Tuple(A, B) with N rows: A0, A1, ..., A(N-1), B0, B1, ..., B(N-1) + */ + private decodeTupleColumn(elements: ClickHouseType[], names: string[] | undefined, rowCount: number): AstNode[] { + const typeStr = names + ? `Tuple(${elements.map((e, i) => `${names[i]} ${typeToString(e)}`).join(', ')})` + : `Tuple(${elements.map(typeToString).join(', ')})`; + + // Read all values for each element type + const elementColumns: AstNode[][] = []; + for (let i = 0; i < elements.length; i++) { + elementColumns.push(this.decodeColumnData(elements[i], rowCount).values); + } + + // Assemble tuples + const values: AstNode[] = []; + for (let row = 0; row < rowCount; row++) { + const children: AstNode[] = []; + + for (let el = 0; el < elements.length; el++) { + const label = names?.[el] ?? `[${el}]`; + const child = elementColumns[el][row]; + child.label = label; + children.push(child); + } + + const startOffset = children[0]?.byteRange.start ?? this.reader.offset; + const endOffset = children[children.length - 1]?.byteRange.end ?? this.reader.offset; + + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: endOffset }, + value: children.map((c) => c.value), + displayValue: `(${children.map((c) => c.displayValue).join(', ')})`, + label: `[${row}]`, + children, + }); + } + + return values; + } + + // ========================================= + // Complex type column decoders (Native-specific) + // ========================================= + + /** + * Nullable in Native format: + * 1. NullMap stream: N bytes (0x00 = not null, 0x01 = null) + * 2. Values stream: N values of inner type (including placeholders for NULLs) + */ + private decodeNullableColumn(innerType: ClickHouseType, rowCount: number): AstNode[] { + const typeStr = `Nullable(${typeToString(innerType)})`; + + // Read null map first + const nullMapStart = this.reader.offset; + const nullMap: boolean[] = []; + for (let i = 0; i < rowCount; i++) { + const { value } = this.reader.readUInt8(); + nullMap.push(value !== 0); + } + + // Read all values (even for NULLs) + const innerValues = this.decodeColumnData(innerType, rowCount).values; + + // Combine null map with values + const values: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const isNull = nullMap[i]; + const innerNode = innerValues[i]; + + if (isNull) { + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { + start: nullMapStart + i, + end: innerNode.byteRange.end, + }, + value: null, + displayValue: 'NULL', + label: `[${i}]`, + metadata: { isNull: true }, + }); + } else { + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { + start: nullMapStart + i, + end: innerNode.byteRange.end, + }, + value: innerNode.value, + displayValue: innerNode.displayValue, + label: `[${i}]`, + children: innerNode.children, + metadata: { isNull: false }, + }); + } + } + + return values; + } + + /** + * Array in Native format: + * 1. ArraySizes stream: N cumulative offsets as UInt64 + * 2. ArrayElements stream: flattened elements + * + * Note: For Array(JSON) inside variant types, the JSON structure prefix is read + * separately (via readColumnPrefix) and decoding uses decodeArrayColumnWithPrefix. + */ + private decodeArrayColumn(elementType: ClickHouseType, rowCount: number): AstNode[] { + const typeStr = `Array(${typeToString(elementType)})`; + + // Read cumulative offsets with AST nodes + const offsetNodes: AstNode[] = []; + const offsets: bigint[] = []; + for (let i = 0; i < rowCount; i++) { + const start = this.reader.offset; + const { value } = this.reader.readUInt64LE(); + offsets.push(value); + offsetNodes.push({ + id: this.generateId(), + type: 'UInt64', + byteRange: { start, end: this.reader.offset }, + value, + displayValue: String(value), + label: 'offset', + }); + } + + // Calculate total elements and individual array sizes + const totalElements = rowCount > 0 ? Number(offsets[rowCount - 1]) : 0; + const sizes: number[] = []; + let prevOffset = 0n; + for (let i = 0; i < rowCount; i++) { + sizes.push(Number(offsets[i] - prevOffset)); + prevOffset = offsets[i]; + } + + // Read all elements + const allElements = this.decodeColumnData(elementType, totalElements).values; + const elementsEnd = this.reader.offset; + + // Distribute elements to arrays + const values: AstNode[] = []; + let elementIndex = 0; + for (let i = 0; i < rowCount; i++) { + const size = sizes[i]; + const arrayElements = allElements.slice(elementIndex, elementIndex + size); + elementIndex += size; + + // Update labels for array elements + arrayElements.forEach((el, j) => { + el.label = `[${j}]`; + }); + + // Create length node showing the computed size from offset difference + const lengthNode: AstNode = { + id: this.generateId(), + type: 'UInt64', + byteRange: offsetNodes[i].byteRange, + value: BigInt(size), + displayValue: `${size} (cumulative: ${offsets[i]})`, + label: 'length', + }; + + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: offsetNodes[i].byteRange.start, end: elementsEnd }, + value: arrayElements.map(e => e.value), + displayValue: `[${arrayElements.map(e => e.displayValue).join(', ')}]`, + label: `[${i}]`, + children: [lengthNode, ...arrayElements], + metadata: { size }, + }); + } + + return values; + } + + /** + * Nested in Native format (flatten_nested = 0). + * Byte-identical to Array(Tuple(T1, ..., Tn)) with named fields: an offsets + * stream followed by one columnar stream per field. See docs/full_native_spec.md. + */ + private decodeNestedColumn( + fields: { name: string; type: ClickHouseType }[], + rowCount: number, + ): AstNode[] { + const elementType: ClickHouseType = { + kind: 'Tuple', + elements: fields.map((f) => f.type), + names: fields.map((f) => f.name), + }; + const typeStr = `Nested(${fields.map((f) => `${f.name} ${typeToString(f.type)}`).join(', ')})`; + const values = this.decodeArrayColumn(elementType, rowCount); + for (const node of values) { + node.type = typeStr; + } + return values; + } + + /** + * Map in Native format (as Array(Tuple(K, V))): + * 1. ArraySizes stream: cumulative offsets + * 2. Keys stream: all keys + * 3. Values stream: all values + */ + private decodeMapColumn(keyType: ClickHouseType, valueType: ClickHouseType, rowCount: number): AstNode[] { + const typeStr = `Map(${typeToString(keyType)}, ${typeToString(valueType)})`; + + // Read cumulative offsets + const offsets: bigint[] = []; + const offsetNodes: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const { value, range } = this.reader.readUInt64LE(); + offsets.push(value); + offsetNodes.push({ + id: this.generateId(), + type: 'ArraySizes', + byteRange: range, + value, + displayValue: `${value} (cumulative)`, + label: `[${i}]`, + }); + } + + // Calculate sizes + const totalEntries = rowCount > 0 ? Number(offsets[rowCount - 1]) : 0; + const sizes: number[] = []; + let prevOffset = 0n; + for (let i = 0; i < rowCount; i++) { + sizes.push(Number(offsets[i] - prevOffset)); + prevOffset = offsets[i]; + } + + // Read all keys + const allKeys = this.decodeColumnData(keyType, totalEntries).values; + + // Read all values + const allValues = this.decodeColumnData(valueType, totalEntries).values; + + // Distribute to maps + const values: AstNode[] = []; + let entryIndex = 0; + for (let i = 0; i < rowCount; i++) { + const size = sizes[i]; + const entries: AstNode[] = []; + + for (let j = 0; j < size; j++) { + const key = allKeys[entryIndex + j]; + const value = allValues[entryIndex + j]; + + key.label = 'key'; + value.label = 'value'; + + entries.push({ + id: this.generateId(), + type: `Tuple(${typeToString(keyType)}, ${typeToString(valueType)})`, + byteRange: { start: key.byteRange.start, end: value.byteRange.end }, + value: [key.value, value.value], + displayValue: `${key.displayValue}: ${value.displayValue}`, + label: `[${j}]`, + children: [key, value], + }); + } + + entryIndex += size; + const entriesEnd = entries[entries.length - 1]?.byteRange.end ?? offsetNodes[i].byteRange.end; + const lengthNode: AstNode = { + id: this.generateId(), + type: 'UInt64', + byteRange: offsetNodes[i].byteRange, + value: BigInt(size), + displayValue: `${size} (cumulative: ${offsets[i]})`, + label: 'length', + }; + + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: offsetNodes[i].byteRange.start, end: entriesEnd }, + value: Object.fromEntries(entries.map(e => [e.children![0].value, e.children![1].value])), + displayValue: `{${entries.map(e => e.displayValue).join(', ')}}`, + label: `[${i}]`, + children: [lengthNode, ...entries], + metadata: { size }, + }); + } + + return values; + } + + /** + * LowCardinality in Native format: + * 1. DictionaryKeys stream: KeysVersion (UInt64) + * 2. DictionaryIndexes stream: type + dictionary + indexes + */ + private decodeLowCardinalityColumn(innerType: ClickHouseType, rowCount: number): AstNode[] { + const typeStr = `LowCardinality(${typeToString(innerType)})`; + const startOffset = this.reader.offset; + + // Read KeysVersion (should be 1) + this.reader.readUInt64LE(); // keysVersion - not used + + // Read IndexesSerializationType + const { value: serializationType } = this.reader.readUInt64LE(); + + // Extract flags from serialization type + const indexType = Number(serializationType & 0xFFn); + const hasAdditionalKeys = ((serializationType >> 9n) & 1n) === 1n; + // const needGlobalDictionary = ((serializationType >> 8n) & 1n) === 1n; + // const needUpdateDictionary = ((serializationType >> 10n) & 1n) === 1n; + + // Read additional keys (dictionary) + let dictionary: AstNode[] = []; + if (hasAdditionalKeys) { + const { value: numKeys } = this.reader.readUInt64LE(); + + // Determine the actual inner type for decoding (unwrap Nullable if present) + const dictType = innerType.kind === 'Nullable' ? innerType.inner : innerType; + dictionary = this.decodeColumnData(dictType, Number(numKeys)).values; + } + + // Read row count + const { value: numRows } = this.reader.readUInt64LE(); + + // Read indexes + const indexes: number[] = []; + for (let i = 0; i < Number(numRows); i++) { + let idx: number; + switch (indexType) { + case 0: // UInt8 + idx = this.reader.readUInt8().value; + break; + case 1: // UInt16 + idx = this.reader.readUInt16LE().value; + break; + case 2: // UInt32 + idx = this.reader.readUInt32LE().value; + break; + case 3: // UInt64 + idx = Number(this.reader.readUInt64LE().value); + break; + default: + throw new Error(`Unknown LowCardinality index type: ${indexType}`); + } + indexes.push(idx); + } + + // Handle Nullable inner type - index 0 is the null placeholder + const isNullable = innerType.kind === 'Nullable'; + + // Build result values from indexes + // Note: The dictionary includes a placeholder at index 0 (empty/default value for nullable) + // Actual data values start at index 1, so we use direct dictionary lookup + const values: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const idx = indexes[i]; + + if (isNullable && idx === 0) { + // NULL value (index 0 is the null placeholder in dictionary) + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: this.reader.offset }, + value: null, + displayValue: 'NULL', + label: `[${i}]`, + }); + } else { + // Non-null value - direct dictionary lookup (index maps directly to dictionary position) + const dictEntry = dictionary[idx]; + + if (dictEntry) { + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: this.reader.offset }, + value: dictEntry.value, + displayValue: dictEntry.displayValue, + label: `[${i}]`, + metadata: { dictionaryIndex: idx }, + }); + } else { + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: this.reader.offset }, + value: ``, + displayValue: ``, + label: `[${i}]`, + }); + } + } + } + + return values; + } + + /** + * Variant in Native format: + * 1. Discriminators prefix: mode (UInt64) + * 2. Discriminators: N bytes (0xFF = NULL) + * 3. Variant elements: sparse columns for each variant type + */ + private decodeVariantColumn(variants: ClickHouseType[], rowCount: number): AstNode[] { + const typeStr = `Variant(${variants.map(typeToString).join(', ')})`; + const startOffset = this.reader.offset; + + // Read mode (0 = BASIC) + const { value: _mode } = this.reader.readUInt64LE(); + + // Read discriminators + const discriminators: number[] = []; + for (let i = 0; i < rowCount; i++) { + const { value } = this.reader.readUInt8(); + discriminators.push(value); + } + + // Count values per variant type + const countPerVariant: number[] = new Array(variants.length).fill(0); + for (const disc of discriminators) { + if (disc !== 0xFF && disc < variants.length) { + countPerVariant[disc]++; + } + } + + // Read sparse data for each variant + const variantData: AstNode[][] = []; + for (let v = 0; v < variants.length; v++) { + const count = countPerVariant[v]; + if (count > 0) { + variantData[v] = this.decodeColumnData(variants[v], count).values; + } else { + variantData[v] = []; + } + } + + // Track current position in each variant's data + const variantPositions: number[] = new Array(variants.length).fill(0); + + // Build result values + const values: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const disc = discriminators[i]; + + if (disc === 0xFF) { + // NULL + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: this.reader.offset }, + value: null, + displayValue: 'NULL', + label: `[${i}]`, + metadata: { discriminator: disc }, + }); + } else if (disc < variants.length) { + const variantNode = variantData[disc][variantPositions[disc]++]; + values.push({ + id: this.generateId(), + type: typeToString(variants[disc]), + byteRange: variantNode.byteRange, + value: variantNode.value, + displayValue: variantNode.displayValue, + label: `[${i}]`, + children: variantNode.children, + metadata: { discriminator: disc, variantType: typeToString(variants[disc]) }, + }); + } else { + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: this.reader.offset }, + value: ``, + displayValue: ``, + label: `[${i}]`, + }); + } + } + + return values; + } + + /** + * Dynamic in Native format: + * 1. DynamicStructure stream: version + type list + * 2. DynamicData stream: internal Variant data (with extra SharedVariant) + * + * Discriminator encoding: + * - disc 0 to numTypes-1: declared types (0-indexed) + * - disc numTypes: SharedVariant (values of other types) + * - disc 0xFF: NULL + * + * V1 format (version=1): SharedVariant stores values as String representation + * V2 format (version=2): SharedVariant stores values as binary type index + binary value + */ + private decodeDynamicColumn(rowCount: number): AstNode[] { + const startOffset = this.reader.offset; + const headerChildren: AstNode[] = []; + + // Read version (1 = V1, 2 = V2) + const versionStart = this.reader.offset; + const { value: version } = this.reader.readUInt64LE(); + headerChildren.push({ + id: this.generateId(), + type: 'UInt64', + byteRange: { start: versionStart, end: this.reader.offset }, + value: version, + displayValue: version.toString(), + label: 'version', + }); + + // FLATTENED (version 3): no max_dynamic_types, no internal-Variant mode word, + // discriminators in wire order with NULL = num_types. Selected by the query + // setting output_format_native_use_flattened_dynamic_and_json_serialization. + if (version === 3n) { + return this.decodeDynamicColumnFlattened(rowCount, startOffset, headerChildren); + } + + // Read max_dynamic_types (V1 only - in V2 this field doesn't exist or has different meaning) + if (version === 1n) { + const maxTypesStart = this.reader.offset; + const { value: maxTypes } = decodeLEB128(this.reader); + headerChildren.push({ + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: maxTypesStart, end: this.reader.offset }, + value: maxTypes, + displayValue: maxTypes.toString(), + label: 'max_dynamic_types', + }); + } + + // Read num_dynamic_types + const numTypesStart = this.reader.offset; + const { value: numTypes } = decodeLEB128(this.reader); + headerChildren.push({ + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: numTypesStart, end: this.reader.offset }, + value: numTypes, + displayValue: numTypes.toString(), + label: 'num_dynamic_types', + }); + + // Read type names + const typeNames: string[] = []; + const typeNamesStart = this.reader.offset; + const typeNameNodes: AstNode[] = []; + for (let i = 0; i < numTypes; i++) { + const nameStart = this.reader.offset; + const { value: len } = decodeLEB128(this.reader); + const { value: bytes } = this.reader.readBytes(len); + const typeName = new TextDecoder().decode(bytes); + typeNames.push(typeName); + typeNameNodes.push({ + id: this.generateId(), + type: 'String', + byteRange: { start: nameStart, end: this.reader.offset }, + value: typeName, + displayValue: `"${typeName}"`, + label: `[${i}]`, + }); + } + if (numTypes > 0) { + headerChildren.push({ + id: this.generateId(), + type: 'Array(String)', + byteRange: { start: typeNamesStart, end: this.reader.offset }, + value: typeNames, + displayValue: `[${typeNames.map(t => `"${t}"`).join(', ')}]`, + label: 'type_names', + children: typeNameNodes, + }); + } + + // Parse types + const variants = typeNames.map(name => parseType(name)); + + // Build discriminator mapping based on alphabetical sort of [typeNames + "SharedVariant"] + // Discriminators are assigned based on alphabetically sorted list, NOT positional indexing + const sortedWithShared = [...typeNames, 'SharedVariant'].sort(); + // discToTypeIndex: maps discriminator -> original type index (-1 for SharedVariant) + const discToTypeIndex = new Map(); + // discToTypeName: maps discriminator -> type name for display + const discToTypeName = new Map(); + for (let idx = 0; idx < sortedWithShared.length; idx++) { + const name = sortedWithShared[idx]; + discToTypeName.set(idx, name); + if (name === 'SharedVariant') { + discToTypeIndex.set(idx, -1); // -1 indicates SharedVariant + } else { + const originalIndex = typeNames.indexOf(name); + discToTypeIndex.set(idx, originalIndex); + } + } + + // Read Variant discriminators prefix (mode) + const modeStart = this.reader.offset; + const { value: mode } = this.reader.readUInt64LE(); + headerChildren.push({ + id: this.generateId(), + type: 'UInt64', + byteRange: { start: modeStart, end: this.reader.offset }, + value: mode, + displayValue: mode.toString(), + label: 'variant_mode', + }); + + // Read discriminators + const discriminatorsStart = this.reader.offset; + const discriminators: number[] = []; + const discNodes: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const discStart = this.reader.offset; + const { value } = this.reader.readUInt8(); + discriminators.push(value); + + // Map discriminator to type name for display (using alphabetical sort mapping) + let discType: string; + if (value === 0xFF) discType = 'NULL'; + else discType = discToTypeName.get(value) ?? `unknown(${value})`; + + discNodes.push({ + id: this.generateId(), + type: 'UInt8', + byteRange: { start: discStart, end: this.reader.offset }, + value, + displayValue: `${value} (${discType})`, + label: `[${i}]`, + }); + } + if (rowCount > 0) { + headerChildren.push({ + id: this.generateId(), + type: 'Array(UInt8)', + byteRange: { start: discriminatorsStart, end: this.reader.offset }, + value: discriminators, + displayValue: `[${discriminators.join(', ')}]`, + label: 'discriminators', + children: discNodes, + }); + } + + const headerEndOffset = this.reader.offset; + + // Count values per variant type (using alphabetical discriminator mapping) + // discToTypeIndex maps: disc -> original type index (-1 for SharedVariant) + // We count per original type index, with SharedVariant at numTypes + const countPerVariant: number[] = new Array(numTypes + 1).fill(0); + for (const disc of discriminators) { + if (disc === 0xFF) continue; // NULL + const typeIdx = discToTypeIndex.get(disc); + if (typeIdx === -1) { + // SharedVariant (stored at index numTypes in countPerVariant) + countPerVariant[numTypes]++; + } else if (typeIdx !== undefined && typeIdx >= 0) { + // Declared type (original index) + countPerVariant[typeIdx]++; + } + } + + // Count values per DISCRIMINATOR (data is serialized in discriminator order, not original type order) + const countPerDiscriminator: number[] = new Array(sortedWithShared.length).fill(0); + for (const disc of discriminators) { + if (disc !== 0xFF && disc < sortedWithShared.length) { + countPerDiscriminator[disc]++; + } + } + + // Read sparse data in DISCRIMINATOR ORDER (alphabetically sorted) + // variantDataByDisc[disc] = array of AstNodes for that discriminator + const variantDataByDisc: AstNode[][] = []; + + for (let disc = 0; disc < sortedWithShared.length; disc++) { + const count = countPerDiscriminator[disc]; + const typeIdx = discToTypeIndex.get(disc); + + if (count === 0) { + variantDataByDisc[disc] = []; + continue; + } + + if (typeIdx === -1) { + // SharedVariant + const sharedVariantData: AstNode[] = []; + if (version === 1n) { + // V1: SharedVariant stores values as length-prefixed blob containing BinaryTypeIndex + binary_value + for (let i = 0; i < count; i++) { + const valueStart = this.reader.offset; + const { value: _len } = decodeLEB128(this.reader); + + // Read the BinaryTypeIndex (single byte) and decode the type + const { value: binTypeIdx } = this.reader.readUInt8(); + const innerType = this.decodeDynamicBinaryTypeExtended(binTypeIdx); + const innerValue = this.decodeValue(innerType); + + sharedVariantData.push({ + id: this.generateId(), + type: typeToString(innerType), + byteRange: { start: valueStart, end: this.reader.offset }, + value: innerValue.value, + displayValue: innerValue.displayValue, + children: innerValue.children, + metadata: { isSharedVariant: true, binaryTypeIndex: binTypeIdx, serializationVersion: 1 }, + }); + } + } else { + // V2: SharedVariant stores values as binary type index + binary value + for (let i = 0; i < count; i++) { + const valueStart = this.reader.offset; + const { value: binTypeIdx } = decodeLEB128(this.reader); + const innerType = this.decodeDynamicBinaryType(binTypeIdx); + const innerValue = this.decodeValue(innerType); + + sharedVariantData.push({ + id: this.generateId(), + type: typeToString(innerType), + byteRange: { start: valueStart, end: this.reader.offset }, + value: innerValue.value, + displayValue: innerValue.displayValue, + children: innerValue.children, + metadata: { isSharedVariant: true, binaryTypeIndex: binTypeIdx, serializationVersion: 2 }, + }); + } + } + variantDataByDisc[disc] = sharedVariantData; + } else if (typeIdx !== undefined) { + // Declared type + variantDataByDisc[disc] = this.decodeColumnData(variants[typeIdx], count).values; + } + } + + // Track current position in each discriminator's data (indexed by discriminator) + const variantPositions: number[] = new Array(sortedWithShared.length).fill(0); + + // Build result values (using discriminator to index directly into variantDataByDisc) + const values: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const disc = discriminators[i]; + + if (disc === 0xFF) { + // NULL + values.push({ + id: this.generateId(), + type: 'Dynamic(NULL)', + byteRange: { start: startOffset, end: this.reader.offset }, + value: null, + displayValue: 'NULL', + label: `[${i}]`, + metadata: { discriminator: disc, actualType: 'NULL' }, + }); + } else if (disc < sortedWithShared.length) { + const typeIdx = discToTypeIndex.get(disc); + const isSharedVariant = typeIdx === -1; + const variantNode = variantDataByDisc[disc][variantPositions[disc]++]; + const actualType = variantNode.type; + + variantNode.label = 'value'; + + values.push({ + id: this.generateId(), + type: `Dynamic(${actualType})`, + byteRange: variantNode.byteRange, + value: variantNode.value, + displayValue: variantNode.displayValue, + label: `[${i}]`, + children: [variantNode], + metadata: { + discriminator: disc, + actualType, + isSharedVariant, + serializationVersion: Number(version), + }, + }); + } else { + values.push({ + id: this.generateId(), + type: 'Dynamic', + byteRange: { start: startOffset, end: this.reader.offset }, + value: ``, + displayValue: ``, + label: `[${i}]`, + }); + } + } + + // Create header node that contains all the Dynamic column metadata + const headerNode: AstNode = { + id: this.generateId(), + type: 'Dynamic.Header', + byteRange: { start: startOffset, end: headerEndOffset }, + value: { version: Number(version), numTypes, typeNames }, + displayValue: `Dynamic(${numTypes} types, v${version})`, + label: 'header', + children: headerChildren, + }; + + // Return header followed by values + return [headerNode, ...values]; + } + + /** + * Smallest unsigned integer width (in bytes) able to index `count` distinct + * values, matching ClickHouse's getSmallestIndexesType: UInt8/16/32/64. + */ + private smallestIndexWidth(count: number): number { + if (count <= 0xff) return 1; + if (count <= 0xffff) return 2; + if (count <= 0xffffffff) return 4; + return 8; + } + + private readDiscriminator(width: number): number { + switch (width) { + case 1: + return this.reader.readUInt8().value; + case 2: + return this.reader.readUInt16LE().value; + case 4: + return this.reader.readUInt32LE().value; + case 8: + return Number(this.reader.readUInt64LE().value); + default: + throw new Error(`Native format: unsupported discriminator width ${width}`); + } + } + + /** + * Decode a FLATTENED (version 3) Dynamic column. The version word has already + * been consumed. Layout (see docs/full_native_spec.md → Dynamic): + * [VarUInt num_types] + * [num_types × String] type names, in wire order + * [per type: its state prefix] empty for leaf types + * [per block with rows > 0]: + * [num_rows × discriminator] width = smallest index for num_types + 1; + * NULL discriminator = num_types + * [for each type i, in wire order: values for rows whose disc == i] + */ + private decodeDynamicColumnFlattened( + rowCount: number, + startOffset: number, + headerChildren: AstNode[], + ): AstNode[] { + // num_types + const numTypesStart = this.reader.offset; + const { value: numTypes } = decodeLEB128(this.reader); + headerChildren.push({ + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: numTypesStart, end: this.reader.offset }, + value: numTypes, + displayValue: numTypes.toString(), + label: 'num_dynamic_types', + }); + + // Type names (wire order) + const typeNames: string[] = []; + const typeNameNodes: AstNode[] = []; + const typeNamesStart = this.reader.offset; + for (let i = 0; i < numTypes; i++) { + const node = this.decodeString(); + node.label = `[${i}]`; + typeNames.push(node.value as string); + typeNameNodes.push(node); + } + if (numTypes > 0) { + headerChildren.push({ + id: this.generateId(), + type: 'Array(String)', + byteRange: { start: typeNamesStart, end: this.reader.offset }, + value: typeNames, + displayValue: `[${typeNames.map((t) => `"${t}"`).join(', ')}]`, + label: 'type_names', + children: typeNameNodes, + }); + } + + const variants = typeNames.map((name) => parseType(name)); + // Per-type state prefix (empty for leaf types; recurse for stateful inner types). + const variantPrefixes = variants.map((v) => this.readColumnPrefix(v)); + + // Discriminators: width by num_types + 1 (the extra slot is NULL = num_types). + const nullDiscriminator = numTypes; + const discWidth = this.smallestIndexWidth(numTypes + 1); + const discriminatorsStart = this.reader.offset; + const discriminators: number[] = []; + const discNodes: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const discStart = this.reader.offset; + const disc = this.readDiscriminator(discWidth); + discriminators.push(disc); + const label = disc === nullDiscriminator ? 'NULL' : (typeNames[disc] ?? `unknown(${disc})`); + discNodes.push({ + id: this.generateId(), + type: `UInt${discWidth * 8}`, + byteRange: { start: discStart, end: this.reader.offset }, + value: disc, + displayValue: `${disc} (${label})`, + label: `[${i}]`, + }); + } + if (rowCount > 0) { + headerChildren.push({ + id: this.generateId(), + type: `Array(UInt${discWidth * 8})`, + byteRange: { start: discriminatorsStart, end: this.reader.offset }, + value: discriminators, + displayValue: `[${discriminators.join(', ')}]`, + label: 'discriminators', + children: discNodes, + }); + } + const headerEndOffset = this.reader.offset; + + // Count rows per type and decode each type's dense run in wire order. + const countPerType = new Array(numTypes).fill(0); + for (const disc of discriminators) { + if (disc !== nullDiscriminator && disc < numTypes) countPerType[disc]++; + } + const runData: AstNode[][] = []; + for (let t = 0; t < numTypes; t++) { + runData[t] = countPerType[t] > 0 + ? this.decodeColumnWithPrefix(variants[t], countPerType[t], variantPrefixes[t]) + : []; + } + + // Reconstruct the dense column. + const positions = new Array(numTypes).fill(0); + const values: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const disc = discriminators[i]; + if (disc === nullDiscriminator || disc >= numTypes) { + values.push({ + id: this.generateId(), + type: 'Dynamic(NULL)', + byteRange: { start: startOffset, end: this.reader.offset }, + value: null, + displayValue: 'NULL', + label: `[${i}]`, + metadata: { discriminator: disc, actualType: 'NULL', serializationVersion: 3 }, + }); + continue; + } + const node = runData[disc][positions[disc]++]; + node.label = 'value'; + values.push({ + id: this.generateId(), + type: `Dynamic(${node.type})`, + byteRange: node.byteRange, + value: node.value, + displayValue: node.displayValue, + label: `[${i}]`, + children: [node], + metadata: { discriminator: disc, actualType: node.type, serializationVersion: 3 }, + }); + } + + const headerNode: AstNode = { + id: this.generateId(), + type: 'Dynamic.Header', + byteRange: { start: startOffset, end: headerEndOffset }, + value: { version: 3, numTypes, typeNames }, + displayValue: `Dynamic(${numTypes} types, v3 FLATTENED)`, + label: 'header', + children: headerChildren, + }; + return [headerNode, ...values]; + } + + /** + * Decode binary type index used in Dynamic's SharedVariant (V2 format) + */ + private decodeDynamicBinaryType(typeIdx: number): ClickHouseType { + // Common binary type indexes from ClickHouse BinaryTypeIndex enum + const typeMap: Record = { + 0: { kind: 'String' } as ClickHouseType, // Nothing maps to String as fallback + 1: { kind: 'UInt8' }, + 2: { kind: 'UInt16' }, + 3: { kind: 'UInt32' }, + 4: { kind: 'UInt64' }, + 5: { kind: 'UInt128' }, + 6: { kind: 'UInt256' }, + 7: { kind: 'Int8' }, + 8: { kind: 'Int16' }, + 9: { kind: 'Int32' }, + 10: { kind: 'Int64' }, + 11: { kind: 'Int128' }, + 12: { kind: 'Int256' }, + 13: { kind: 'Float32' }, + 14: { kind: 'Float64' }, + 15: { kind: 'Date' }, + 16: { kind: 'Date32' }, + 17: { kind: 'DateTime' }, + // 18: DateTime64 with precision - handled specially + 19: { kind: 'String' }, + 20: { kind: 'UUID' }, + 21: { kind: 'IPv4' }, + 22: { kind: 'IPv6' }, + 23: { kind: 'Bool' }, + // 24+: Complex types that need additional parsing + }; + + const type = typeMap[typeIdx]; + if (type) { + return type; + } + + // For unknown types, return String as fallback + return { kind: 'String' }; + } + + /** + * Extended binary type index decoder that handles all ClickHouse types + * Used for SharedVariant decoding where complex types may appear + */ + private decodeDynamicBinaryTypeExtended(typeIdx: number): ClickHouseType { + switch (typeIdx) { + case 0x00: return { kind: 'String' }; // Nothing - fallback to String + case 0x01: return { kind: 'UInt8' }; + case 0x02: return { kind: 'UInt16' }; + case 0x03: return { kind: 'UInt32' }; + case 0x04: return { kind: 'UInt64' }; + case 0x05: return { kind: 'UInt128' }; + case 0x06: return { kind: 'UInt256' }; + case 0x07: return { kind: 'Int8' }; + case 0x08: return { kind: 'Int16' }; + case 0x09: return { kind: 'Int32' }; + case 0x0a: return { kind: 'Int64' }; + case 0x0b: return { kind: 'Int128' }; + case 0x0c: return { kind: 'Int256' }; + case 0x0d: return { kind: 'Float32' }; + case 0x0e: return { kind: 'Float64' }; + case 0x0f: return { kind: 'Date' }; + case 0x10: return { kind: 'Date32' }; + case 0x11: return { kind: 'DateTime' }; + case 0x12: { + // DateTime with timezone + const { value: tzLen } = decodeLEB128(this.reader); + const { value: tzBytes } = this.reader.readBytes(tzLen); + const timezone = new TextDecoder().decode(tzBytes); + return { kind: 'DateTime', timezone }; + } + case 0x13: { + // DateTime64 + const { value: precision } = this.reader.readUInt8(); + return { kind: 'DateTime64', precision }; + } + case 0x14: { + // DateTime64 with timezone + const { value: precision } = this.reader.readUInt8(); + const { value: tzLen } = decodeLEB128(this.reader); + const { value: tzBytes } = this.reader.readBytes(tzLen); + const timezone = new TextDecoder().decode(tzBytes); + return { kind: 'DateTime64', precision, timezone }; + } + case 0x15: return { kind: 'String' }; + case 0x16: { + // FixedString + const { value: length } = decodeLEB128(this.reader); + return { kind: 'FixedString', length }; + } + case 0x1d: return { kind: 'UUID' }; + case 0x1e: { + // Array + const { value: elemTypeIdx } = this.reader.readUInt8(); + const element = this.decodeDynamicBinaryTypeExtended(elemTypeIdx); + return { kind: 'Array', element }; + } + case 0x1f: { + // Tuple (unnamed) + const { value: count } = decodeLEB128(this.reader); + const elements: ClickHouseType[] = []; + for (let i = 0; i < count; i++) { + const { value: elemTypeIdx } = this.reader.readUInt8(); + const elem = this.decodeDynamicBinaryTypeExtended(elemTypeIdx); + elements.push(elem); + } + return { kind: 'Tuple', elements }; + } + case 0x20: { + // Named Tuple + const { value: count } = decodeLEB128(this.reader); + const elements: ClickHouseType[] = []; + const names: string[] = []; + for (let i = 0; i < count; i++) { + const { value: nameLen } = decodeLEB128(this.reader); + const { value: nameBytes } = this.reader.readBytes(nameLen); + names.push(new TextDecoder().decode(nameBytes)); + const { value: elemTypeIdx } = this.reader.readUInt8(); + const elem = this.decodeDynamicBinaryTypeExtended(elemTypeIdx); + elements.push(elem); + } + return { kind: 'Tuple', elements, names }; + } + case 0x23: { + // Nullable + const { value: innerTypeIdx } = this.reader.readUInt8(); + const inner = this.decodeDynamicBinaryTypeExtended(innerTypeIdx); + return { kind: 'Nullable', inner }; + } + case 0x27: { + // Map + const { value: keyTypeIdx } = this.reader.readUInt8(); + const key = this.decodeDynamicBinaryTypeExtended(keyTypeIdx); + const { value: valueTypeIdx } = this.reader.readUInt8(); + const value = this.decodeDynamicBinaryTypeExtended(valueTypeIdx); + return { kind: 'Map', key, value }; + } + case 0x28: return { kind: 'IPv4' }; + case 0x29: return { kind: 'IPv6' }; + case 0x2d: return { kind: 'Bool' }; + default: + // Fallback to String for unknown types + return { kind: 'String' }; + } + } + + /** + * JSON in Native format + * + * Actual format discovered through debugging: + * 1. max_dynamic_paths (UInt64) - typically 0 + * 2. typed_paths_count (LEB128) - number of dynamic paths in this column + * 3. columns_count (LEB128) - same as typed_paths_count + * 4. Path names (String for each) + * 5. Column info for each column: + * - offset (UInt64) + * - serialization_kind (UInt16) + * - type_name (String) + * - metadata (UInt64) + * 6. TYPED SUB-COLUMN VALUES (if JSON type has typed paths like JSON(a Int32)) + * - These are raw values without flag bytes + * 7. Dynamic column values (flag byte + value for each column/row) + * 8. Shared data offsets (UInt64 per row) + * + * The resulting AST includes all structural elements as children. + */ + private decodeJSONColumn(type: ClickHouseType, rowCount: number): AstNode[] { + const startOffset = this.reader.offset; + const values: AstNode[] = []; + + // Get JSON type info + const jsonType = type as { kind: 'JSON'; typedPaths?: Map; maxDynamicPaths?: number }; + const typedSubColumns = jsonType.typedPaths; + + // Dispatch on the Object serialization version (first 8 bytes, UInt64 LE): + // 1 = Tier-1 String fallback (output_format_native_write_json_as_string) + // 3 = FLATTENED Object (output_format_native_use_flattened_dynamic_and_json_serialization) + // 0 / 2 = default V1 / V2 Object (max_dynamic_paths + dynamic structures + shared data) + const versionPeek = this.reader.peekBytes(8); + let peekedVersion = 0n; + for (let i = 7; i >= 0; i--) { + peekedVersion = (peekedVersion << 8n) | BigInt(versionPeek[i] ?? 0); + } + if (peekedVersion === 1n) { + return this.decodeJSONColumnStringFallback(rowCount, startOffset); + } + if (peekedVersion === 3n) { + return this.decodeJSONColumnFlattened(type, rowCount, startOffset); + } + return this.decodeJSONColumnV1(type, rowCount, startOffset); + + // Legacy format: reads version as max_dynamic_paths (coincidentally works when version=0) + const maxDynPathsNode = this.decodeUInt64(); + maxDynPathsNode.label = 'max_dynamic_paths'; + + // Read typed_paths_count with AST node + const typedPathsCountStart = this.reader.offset; + const { value: typedPathsCount } = decodeLEB128(this.reader); + const typedPathsCountNode: AstNode = { + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: typedPathsCountStart, end: this.reader.offset }, + value: typedPathsCount, + displayValue: String(typedPathsCount), + label: 'typed_paths_count', + }; + + // Handle JSON with no dynamic paths + if (typedPathsCount === 0) { + // If there are typed sub-columns, read them + const hasTypedPaths = typedSubColumns !== undefined && typedSubColumns!.size > 0; + if (hasTypedPaths) { + const typedPaths = typedSubColumns!; + // For fully-typed JSON: flag byte + typed values + shared offset + // Collect nodes per row: [flagNode, ...pathValueNodes] + const rowData: { flagNode: AstNode; pathNodes: Map }[] = []; + + for (let row = 0; row < rowCount; row++) { + // Read flag byte (0 = object present) + const flagNode = this.decodeUInt8(); + flagNode.label = 'object_present'; + + const pathNodes = new Map(); + for (const [pathName, pathType] of typedPaths) { + const node = this.decodeValue(pathType); + pathNodes.set(pathName, node); + } + + rowData.push({ flagNode, pathNodes }); + } + + // Read shared_data_offsets + const sharedOffsetNodes: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const node = this.decodeUInt64(); + node.label = `shared_data_offset[${i}]`; + sharedOffsetNodes.push(node); + } + + // Build result nodes with all structural AST children + for (let row = 0; row < rowCount; row++) { + const children: AstNode[] = []; + const jsonValue: Record = {}; + + // Add structural nodes + children.push(maxDynPathsNode); + children.push(typedPathsCountNode); + children.push(rowData[row].flagNode); + + // Add path nodes + for (const [pathName] of typedPaths) { + const valueNode = rowData[row].pathNodes.get(pathName)!; + jsonValue[pathName] = valueNode.value; + + children.push({ + id: this.generateId(), + type: 'JSON path', + byteRange: valueNode.byteRange, + value: { [pathName]: valueNode.value }, + displayValue: `${pathName}: ${valueNode.displayValue}`, + label: pathName, + children: [valueNode], + }); + } + + // Add shared offset node + children.push(sharedOffsetNodes[row]); + + values.push({ + id: this.generateId(), + type: 'JSON', + byteRange: { start: startOffset, end: this.reader.offset }, + value: jsonValue, + displayValue: `{${typedPaths.size} paths}`, + label: `[${row}]`, + children, + }); + } + + return values; + } + + // No typed sub-columns either - return empty JSON objects + // Read shared_data_offsets + const sharedOffsetNodes: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const node = this.decodeUInt64(); + node.label = `shared_data_offset[${i}]`; + sharedOffsetNodes.push(node); + } + + for (let i = 0; i < rowCount; i++) { + values.push({ + id: this.generateId(), + type: 'JSON', + byteRange: { start: startOffset, end: this.reader.offset }, + value: {}, + displayValue: '{0 paths}', + label: `[${i}]`, + children: [maxDynPathsNode, typedPathsCountNode, sharedOffsetNodes[i]], + }); + } + return values; + } + + // Read columns_count with AST node + const columnsCountStart = this.reader.offset; + const { value: columnsCount } = decodeLEB128(this.reader); + const columnsCountNode: AstNode = { + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: columnsCountStart, end: this.reader.offset }, + value: columnsCount, + displayValue: String(columnsCount), + label: 'columns_count', + }; + + // Read path names with AST nodes + const pathNameNodes: AstNode[] = []; + const pathNames: string[] = []; + for (let i = 0; i < typedPathsCount; i++) { + const pathStart = this.reader.offset; + const { value: nameLen } = decodeLEB128(this.reader); + const { value: nameBytes } = this.reader.readBytes(nameLen); + const pathName = new TextDecoder().decode(nameBytes); + pathNames.push(pathName); + + pathNameNodes.push({ + id: this.generateId(), + type: 'String', + byteRange: { start: pathStart, end: this.reader.offset }, + value: pathName, + displayValue: `"${pathName}"`, + label: `path_name[${i}]`, + }); + } + + // Read column info with AST nodes + const columnInfoNodes: AstNode[] = []; + const columnInfos: { typeName: string }[] = []; + for (let i = 0; i < columnsCount; i++) { + const infoStart = this.reader.offset; + + const offsetNode = this.decodeUInt64(); + offsetNode.label = 'offset'; + + const kindNode = this.decodeUInt16(); + kindNode.label = 'serialization_kind'; + + const typeStart = this.reader.offset; + const { value: typeLen } = decodeLEB128(this.reader); + const { value: typeBytes } = this.reader.readBytes(typeLen); + const typeName = new TextDecoder().decode(typeBytes); + const typeNameNode: AstNode = { + id: this.generateId(), + type: 'String', + byteRange: { start: typeStart, end: this.reader.offset }, + value: typeName, + displayValue: `"${typeName}"`, + label: 'type', + }; + + const metadataNode = this.decodeUInt64(); + metadataNode.label = 'metadata'; + + columnInfos.push({ typeName }); + columnInfoNodes.push({ + id: this.generateId(), + type: 'column_info', + byteRange: { start: infoStart, end: this.reader.offset }, + value: { offset: offsetNode.value, kind: kindNode.value, type: typeName, metadata: metadataNode.value }, + displayValue: `${pathNames[i]}: ${typeName}`, + label: `column_info[${i}]`, + children: [offsetNode, kindNode, typeNameNode, metadataNode], + }); + } + + // Read typed sub-column values first (if any) - collect AstNodes per path + const typedPathNodes: Map = new Map(); + const hasTypedSubCols = typedSubColumns !== undefined && typedSubColumns!.size > 0; + if (hasTypedSubCols) { + for (const [pathName, pathType] of typedSubColumns!) { + const nodes: AstNode[] = []; + for (let row = 0; row < rowCount; row++) { + nodes.push(this.decodeValue(pathType)); + } + typedPathNodes.set(pathName, nodes); + } + } + + // Read dynamic column values (flag byte + value for each column/row) - collect AstNodes + const dynamicPathData: { flagNodes: AstNode[]; valueNodes: AstNode[] }[] = []; + for (let colIdx = 0; colIdx < columnsCount; colIdx++) { + const { typeName } = columnInfos[colIdx]; + const colType = parseType(typeName); + const flagNodes: AstNode[] = []; + const valueNodes: AstNode[] = []; + + for (let row = 0; row < rowCount; row++) { + const flagNode = this.decodeUInt8(); + flagNode.label = 'flag'; + flagNodes.push(flagNode); + valueNodes.push(this.decodeValue(colType)); + } + + dynamicPathData.push({ flagNodes, valueNodes }); + } + + // Read shared_data_offsets + const sharedOffsetNodes: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const node = this.decodeUInt64(); + node.label = `shared_data_offset[${i}]`; + sharedOffsetNodes.push(node); + } + + // Build result nodes with all structural AST children + for (let row = 0; row < rowCount; row++) { + const children: AstNode[] = []; + const jsonValue: Record = {}; + + // Add header structural nodes (shared across rows) + children.push(maxDynPathsNode); + children.push(typedPathsCountNode); + children.push(columnsCountNode); + + // Add path name nodes + for (const node of pathNameNodes) { + children.push(node); + } + + // Add column info nodes + for (const node of columnInfoNodes) { + children.push(node); + } + + // Add typed sub-column path nodes + for (const [pathName, nodes] of typedPathNodes) { + const valueNode = nodes[row]; + this.setNestedValue(jsonValue, pathName, valueNode.value); + + children.push({ + id: this.generateId(), + type: 'JSON path', + byteRange: valueNode.byteRange, + value: { [pathName]: valueNode.value }, + displayValue: `${pathName}: ${valueNode.displayValue}`, + label: pathName, + children: [valueNode], + }); + } + + // Add dynamic path nodes (with flag + value) + for (let colIdx = 0; colIdx < columnsCount; colIdx++) { + const pathName = pathNames[colIdx]; + const { flagNodes, valueNodes } = dynamicPathData[colIdx]; + const flagNode = flagNodes[row]; + const valueNode = valueNodes[row]; + + this.setNestedValue(jsonValue, pathName, valueNode.value); + + children.push({ + id: this.generateId(), + type: 'JSON path', + byteRange: { start: flagNode.byteRange.start, end: valueNode.byteRange.end }, + value: { [pathName]: valueNode.value }, + displayValue: `${pathName}: ${valueNode.displayValue}`, + label: pathName, + children: [flagNode, valueNode], + }); + } + + // Add shared offset node + children.push(sharedOffsetNodes[row]); + + const totalPaths = typedPathNodes.size + dynamicPathData.length; + values.push({ + id: this.generateId(), + type: 'JSON', + byteRange: { start: startOffset, end: this.reader.offset }, + value: jsonValue, + displayValue: `{${totalPaths} paths}`, + label: `[${row}]`, + children, + }); + } + + return values; + } + + /** + * Read the state prefix for a column type, if it needs one. + * In Native format, complex types like JSON write their structure as a prefix + * before the actual data. This must be read and stored for later use. + */ + private readColumnPrefix(type: ClickHouseType): unknown { + if (type.kind === 'JSON') { + const jsonType = type as { kind: 'JSON'; typedPaths?: Map }; + return this.readJSONColumnStructure(jsonType.typedPaths); + } + if (type.kind === 'Array') { + // For arrays, recursively check if element needs a prefix + return this.readColumnPrefix(type.element); + } + // Other types don't need prefixes + return null; + } + + /** + * Decode column data, using a pre-read prefix if available. + */ + private decodeColumnWithPrefix(type: ClickHouseType, rowCount: number, prefix: unknown): AstNode[] { + if (type.kind === 'JSON' && prefix) { + return this.readJSONColumnData( + prefix as ReturnType, + rowCount, + this.reader.offset + ); + } + if (type.kind === 'Array' && prefix) { + // For Array with prefix, the prefix is for the element type + return this.decodeArrayColumnWithPrefix(type.element, rowCount, prefix); + } + // No prefix - use normal decoding + return this.decodeColumnData(type, rowCount).values; + } + + /** + * Decode array column where the element type has a pre-read prefix. + */ + private decodeArrayColumnWithPrefix(elementType: ClickHouseType, rowCount: number, elementPrefix: unknown): AstNode[] { + const typeStr = `Array(${typeToString(elementType)})`; + const startOffset = this.reader.offset; + + // Read array offsets + const offsetNodes: AstNode[] = []; + const offsets: bigint[] = []; + for (let i = 0; i < rowCount; i++) { + const start = this.reader.offset; + const { value } = this.reader.readUInt64LE(); + offsets.push(value); + offsetNodes.push({ + id: this.generateId(), + type: 'UInt64', + byteRange: { start, end: this.reader.offset }, + value, + displayValue: String(value), + label: 'array_offset', + }); + } + + // Calculate total elements + const totalElements = rowCount > 0 ? Number(offsets[rowCount - 1]) : 0; + const sizes: number[] = []; + let prevOffset = 0n; + for (let i = 0; i < rowCount; i++) { + sizes.push(Number(offsets[i] - prevOffset)); + prevOffset = offsets[i]; + } + + // Decode elements using the prefix + const allElements = this.decodeColumnWithPrefix(elementType, totalElements, elementPrefix); + + // Distribute to arrays + const values: AstNode[] = []; + let elementIndex = 0; + for (let i = 0; i < rowCount; i++) { + const size = sizes[i]; + const arrayElements = allElements.slice(elementIndex, elementIndex + size); + elementIndex += size; + + arrayElements.forEach((el, j) => { + el.label = `[${j}]`; + }); + + const lengthNode: AstNode = { + id: this.generateId(), + type: 'UInt64', + byteRange: offsetNodes[i].byteRange, + value: BigInt(size), + displayValue: `${size} (cumulative: ${offsets[i]})`, + label: 'length', + }; + + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: this.reader.offset }, + value: arrayElements.map(e => e.value), + displayValue: `[${arrayElements.length} elements]`, + label: `[${i}]`, + children: [lengthNode, ...arrayElements], + metadata: { size }, + }); + } + + return values; + } + + /** + * Read JSON column structure (version, paths, dynamic structures). + * Used by decodeJSONColumnV1 and via readColumnPrefix for nested JSON in Arrays/Variants. + */ + private readJSONColumnStructure(typedSubColumns?: Map): { + serializationVersion: number; + structureChildren: AstNode[]; + dynamicPathNames: string[]; + dynamicStructures: Array<{ + serializationVersion: number; + typeNames: string[]; + variants: ClickHouseType[]; + numTypes: number; + discToTypeIndex: Map; + variantPrefixes: unknown[]; + }>; + typedSubColumns?: Map; + } { + const structureChildren: AstNode[] = []; + + // 1. Read version (UInt64) + const versionNode = this.decodeUInt64(); + versionNode.label = 'version'; + structureChildren.push(versionNode); + const objectSerializationVersion = Number(versionNode.value); + + if (objectSerializationVersion !== 0 && objectSerializationVersion !== 2) { + throw new Error( + `Unsupported JSON object serialization version ${objectSerializationVersion} in Native decoder`, + ); + } + + // 2. Read max_dynamic_paths (V1 only) + if (objectSerializationVersion === 0) { + const maxDynPathsStart = this.reader.offset; + const { value: maxDynamicPaths } = decodeLEB128(this.reader); + const maxDynPathsNode: AstNode = { + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: maxDynPathsStart, end: this.reader.offset }, + value: maxDynamicPaths, + displayValue: String(maxDynamicPaths), + label: 'max_dynamic_paths', + }; + structureChildren.push(maxDynPathsNode); + } + + // 3. Read num_dynamic_paths (VarUInt) + const numDynPathsStart = this.reader.offset; + const { value: numDynamicPaths } = decodeLEB128(this.reader); + const numDynPathsNode: AstNode = { + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: numDynPathsStart, end: this.reader.offset }, + value: numDynamicPaths, + displayValue: String(numDynamicPaths), + label: 'num_dynamic_paths', + }; + structureChildren.push(numDynPathsNode); + + // 4. Read sorted dynamic path names + const dynamicPathNames: string[] = []; + for (let i = 0; i < numDynamicPaths; i++) { + const pathNode = this.decodeString(); + pathNode.label = `dynamic_path[${i}]`; + structureChildren.push(pathNode); + dynamicPathNames.push(pathNode.value as string); + } + + // 5. Read ALL Dynamic structures (one per dynamic path) const dynamicStructures: Array<{ serializationVersion: number; typeNames: string[]; @@ -3175,896 +3427,1154 @@ export class NativeDecoder extends FormatDecoder { numTypes: number; discToTypeIndex: Map; variantPrefixes: unknown[]; - }> = []; - - for (let i = 0; i < numDynamicPaths; i++) { - const dynamicStructureStart = this.reader.offset; - const dynamicStructureChildren: AstNode[] = []; - + }> = []; + + for (let i = 0; i < numDynamicPaths; i++) { + const dynamicStructureStart = this.reader.offset; + const dynamicStructureChildren: AstNode[] = []; + // Read Dynamic version const dynVersionNode = this.decodeUInt64(); dynVersionNode.label = 'dynamic_version'; dynamicStructureChildren.push(dynVersionNode); const dynamicSerializationVersion = Number(dynVersionNode.value); - if (dynamicSerializationVersion !== 1 && dynamicSerializationVersion !== 2) { - throw new Error( - `Unsupported Dynamic serialization version ${dynamicSerializationVersion} in JSON Native decoder`, - ); + if (dynamicSerializationVersion !== 1 && dynamicSerializationVersion !== 2) { + throw new Error( + `Unsupported Dynamic serialization version ${dynamicSerializationVersion} in JSON Native decoder`, + ); + } + + // Read max_dynamic_types (V1 only) + if (dynamicSerializationVersion === 1) { + const maxTypesStart = this.reader.offset; + const { value: maxTypes } = decodeLEB128(this.reader); + const maxTypesNode: AstNode = { + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: maxTypesStart, end: this.reader.offset }, + value: maxTypes, + displayValue: String(maxTypes), + label: 'max_dynamic_types', + }; + dynamicStructureChildren.push(maxTypesNode); + } + + // Read num_dynamic_types + const numTypesStart = this.reader.offset; + const { value: numTypes } = decodeLEB128(this.reader); + const numTypesNode: AstNode = { + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: numTypesStart, end: this.reader.offset }, + value: numTypes, + displayValue: String(numTypes), + label: 'num_dynamic_types', + }; + dynamicStructureChildren.push(numTypesNode); + + // Read type names + const typeNames: string[] = []; + for (let t = 0; t < numTypes; t++) { + const typeNameNode = this.decodeString(); + typeNameNode.label = `type_name[${t}]`; + dynamicStructureChildren.push(typeNameNode); + typeNames.push(typeNameNode.value as string); + } + const variants = typeNames.map(name => parseType(name)); + + // Read variant mode + const modeNode = this.decodeUInt64(); + modeNode.label = 'variant_mode'; + dynamicStructureChildren.push(modeNode); + + // Build discriminator mapping based on alphabetical sort of [typeNames + "SharedVariant"] + const sortedWithShared = [...typeNames, 'SharedVariant'].sort(); + const discToTypeIndex = new Map(); + for (let idx = 0; idx < sortedWithShared.length; idx++) { + const name = sortedWithShared[idx]; + if (name === 'SharedVariant') { + discToTypeIndex.set(idx, -1); + } else { + const originalIndex = typeNames.indexOf(name); + discToTypeIndex.set(idx, originalIndex); + } + } + + // Read variant state prefixes for types that need them + // In Native format with BASIC variant mode, the prefix for nested complex types + // is written BEFORE the discriminators. + const variantPrefixes = variants.map((v, idx) => { + const prefix = this.readColumnPrefix(v); + // If the prefix contains structure children (e.g., nested JSON), add them + if (prefix && typeof prefix === 'object' && 'structureChildren' in prefix) { + const jsonPrefix = prefix as { structureChildren: AstNode[] }; + for (const child of jsonPrefix.structureChildren) { + child.label = `${typeNames[idx]}.${child.label || ''}`; + dynamicStructureChildren.push(child); + } + } + return prefix; + }); + + // Create a container node for this dynamic path's structure + const dynamicPathName = dynamicPathNames[i]; + const dynamicStructureNode: AstNode = { + id: this.generateId(), + type: 'Dynamic.structure', + byteRange: { start: dynamicStructureStart, end: this.reader.offset }, + value: { typeNames, numTypes }, + displayValue: `Dynamic structure for "${dynamicPathName}" (${numTypes} types)`, + label: `${dynamicPathName}.structure`, + children: dynamicStructureChildren, + }; + structureChildren.push(dynamicStructureNode); + + dynamicStructures.push({ + serializationVersion: dynamicSerializationVersion, + typeNames, + variants, + numTypes, + discToTypeIndex, + variantPrefixes, + }); + } + + return { + serializationVersion: objectSerializationVersion, + structureChildren, + dynamicPathNames, + dynamicStructures, + typedSubColumns, + }; + } + + /** + * Read JSON column data using pre-read structure. + * This allows Array(JSON) to read structure first, then offsets, then data. + */ + private readJSONColumnData( + structure: ReturnType, + rowCount: number, + startOffset: number + ): AstNode[] { + const { structureChildren, dynamicPathNames, dynamicStructures, typedSubColumns } = structure; + const numDynamicPaths = dynamicPathNames.length; + const values: AstNode[] = []; + + // Read typed paths data (if any) + const typedPathData: Map = new Map(); + if (typedSubColumns && typedSubColumns.size > 0) { + const sortedTypedPaths = Array.from(typedSubColumns.entries()).sort((a, b) => + a[0].localeCompare(b[0]) + ); + for (const [pathName, pathType] of sortedTypedPaths) { + const nodes = this.decodeColumnData(pathType, rowCount).values; + typedPathData.set(pathName, nodes); + } + } + + // Read ALL Dynamic data (discriminator + values for each path, sequentially) + const dynamicPathData: AstNode[][] = []; + + for (let i = 0; i < numDynamicPaths; i++) { + const { typeNames, variants, numTypes, discToTypeIndex, variantPrefixes } = dynamicStructures[i]; + + // Build reverse mapping: discriminator -> type name for display + const discToTypeName = new Map(); + const sortedWithShared = [...typeNames, 'SharedVariant'].sort(); + for (let idx = 0; idx < sortedWithShared.length; idx++) { + discToTypeName.set(idx, sortedWithShared[idx]); + } + + // Read discriminators for all rows (with byte tracking) + const discriminators: { value: number; range: { start: number; end: number } }[] = []; + for (let r = 0; r < rowCount; r++) { + const start = this.reader.offset; + const { value: disc } = this.reader.readUInt8(); + discriminators.push({ value: disc, range: { start, end: this.reader.offset } }); + } + + // Count values per variant using the alphabetical discriminator mapping + const countPerVariant: number[] = new Array(numTypes).fill(0); + for (const { value: disc } of discriminators) { + if (disc === 255) continue; // NULL + const typeIdx = discToTypeIndex.get(disc); + if (typeIdx !== undefined && typeIdx >= 0) { + countPerVariant[typeIdx]++; + } + // typeIdx === -1 means SharedVariant, handled separately + } + + // Read data for each type (in original order, not sorted order) + const variantData: AstNode[][] = []; + for (let v = 0; v < numTypes; v++) { + const count = countPerVariant[v]; + if (count > 0) { + variantData[v] = this.decodeColumnWithPrefix(variants[v], count, variantPrefixes[v]); + } else { + variantData[v] = []; + } + } + + // Build values for this path + const variantPositions: number[] = new Array(numTypes).fill(0); + const pathValues: AstNode[] = []; + for (let r = 0; r < rowCount; r++) { + const { value: disc, range: discRange } = discriminators[r]; + const typeName = disc === 255 ? 'NULL' : (discToTypeName.get(disc) ?? `Unknown(${disc})`); + + // Create discriminator AST node + const discNode: AstNode = { + id: this.generateId(), + type: 'UInt8', + byteRange: discRange, + value: disc, + displayValue: `${disc} → ${typeName}`, + label: 'discriminator', + }; + + if (disc === 255) { + // NULL discriminator + pathValues.push({ + id: this.generateId(), + type: 'Dynamic(NULL)', + byteRange: discRange, + value: null, + displayValue: 'NULL', + children: [discNode], + }); + } else { + const typeIdx = discToTypeIndex.get(disc); + if (typeIdx !== undefined && typeIdx >= 0) { + // Valid type + const node = variantData[typeIdx][variantPositions[typeIdx]++]; + pathValues.push({ + id: this.generateId(), + type: `Dynamic(${node.type})`, + byteRange: { start: discRange.start, end: node.byteRange.end }, + value: node.value, + displayValue: node.displayValue, + children: [discNode, node], + }); + } else if (typeIdx === -1) { + // SharedVariant - value is in shared data section + pathValues.push({ + id: this.generateId(), + type: 'Dynamic(SharedVariant)', + byteRange: discRange, + value: null, + displayValue: 'SharedVariant', + children: [discNode], + }); + } else { + // Unknown discriminator + pathValues.push({ + id: this.generateId(), + type: 'Dynamic(Unknown)', + byteRange: discRange, + value: null, + displayValue: `Unknown(disc=${disc})`, + children: [discNode], + }); + } + } + } + dynamicPathData.push(pathValues); + } + + // 6. Read shared data offsets (cumulative entry counts, UInt64 per row) + const sharedOffsets: bigint[] = []; + const sharedOffsetNodes: AstNode[] = []; + for (let i = 0; i < rowCount; i++) { + const offsetNode = this.decodeUInt64(); + offsetNode.label = `shared_data_offset[${i}]`; + sharedOffsets.push(offsetNode.value as bigint); + sharedOffsetNodes.push(offsetNode); + } + const totalSharedEntries = rowCount > 0 ? Number(sharedOffsets[rowCount - 1]) : 0; + + // 7. Read shared data entries + const sharedDataEntries: Array<{ path: string; value: unknown }> = []; + for (let i = 0; i < totalSharedEntries; i++) { + // Read path name + const pathNode = this.decodeString(); + const path = pathNode.value as string; + + // Read binary-encoded value: size (VarUInt) + type_index (VarUInt) + value + const { value: _dataSize } = decodeLEB128(this.reader); + const { value: typeIndex } = decodeLEB128(this.reader); + const entryType = this.decodeDynamicBinaryType(typeIndex); + + let value: unknown = null; + if (entryType) { + const valueNodes = this.decodeColumnData(entryType, 1).values; + value = valueNodes[0]?.value; + } + + sharedDataEntries.push({ path, value }); + } + + // Map shared entries to rows + const sharedDataPerRow: Array> = []; + let prevOffset = 0n; + for (let i = 0; i < rowCount; i++) { + const currentOffset = sharedOffsets[i]; + const entriesForRow = sharedDataEntries.slice(Number(prevOffset), Number(currentOffset)); + sharedDataPerRow.push(entriesForRow); + prevOffset = currentOffset; + } + + // Build result nodes + for (let row = 0; row < rowCount; row++) { + const children: AstNode[] = [...structureChildren]; + const jsonValue: Record = {}; + + // Add typed path values + for (const [pathName, nodes] of typedPathData) { + const valueNode = nodes[row]; + this.setNestedValue(jsonValue, pathName, valueNode.value); + children.push({ + id: this.generateId(), + type: 'JSON.typed_path', + byteRange: valueNode.byteRange, + value: { [pathName]: valueNode.value }, + displayValue: `${pathName}: ${valueNode.displayValue}`, + label: pathName, + children: [valueNode], + }); + } + + // Add dynamic path values + for (let i = 0; i < numDynamicPaths; i++) { + const pathName = dynamicPathNames[i]; + const dynamicNode = dynamicPathData[i][row]; + this.setNestedValue(jsonValue, pathName, dynamicNode.value); + children.push({ + id: this.generateId(), + type: 'JSON.dynamic_path', + byteRange: dynamicNode.byteRange, + value: { [pathName]: dynamicNode.value }, + displayValue: `${pathName}: ${dynamicNode.displayValue}`, + label: pathName, + children: [dynamicNode], + }); + } + + // Add shared data offset node for this row + children.push(sharedOffsetNodes[row]); + + // Add shared data values for this row + const rowSharedEntries = sharedDataPerRow[row] || []; + for (const entry of rowSharedEntries) { + this.setNestedValue(jsonValue, entry.path, entry.value); + } + + const totalPaths = typedPathData.size + numDynamicPaths + rowSharedEntries.length; + values.push({ + id: this.generateId(), + type: 'JSON', + byteRange: { start: startOffset, end: this.reader.offset }, + value: jsonValue, + displayValue: `{${totalPaths} paths}`, + label: `[${row}]`, + children, + }); + } + + return values; + } + + /** + * Decode JSON column using V1 format (version=0) + * This calls readJSONColumnStructure then readJSONColumnData. + */ + private decodeJSONColumnV1(type: ClickHouseType, rowCount: number, startOffset: number): AstNode[] { + const jsonType = type as { kind: 'JSON'; typedPaths?: Map }; + const structure = this.readJSONColumnStructure(jsonType.typedPaths); + return this.readJSONColumnData(structure, rowCount, startOffset); + } + + /** + * Tier-1 JSON-as-String fallback (Object serialization version 1). The column + * is a UInt64 state prefix (= 1) followed by a standard String column whose + * values are the JSON text for each row. See docs/full_native_spec.md. + */ + private decodeJSONColumnStringFallback(rowCount: number, startOffset: number): AstNode[] { + const versionNode = this.decodeUInt64(); + versionNode.label = 'version'; + + const values: AstNode[] = []; + for (let row = 0; row < rowCount; row++) { + const strNode = this.decodeString(); + const text = strNode.value as string; + let parsed: unknown = text; + try { + parsed = JSON.parse(text); + } catch { + parsed = text; + } + values.push({ + id: this.generateId(), + type: 'JSON', + byteRange: { start: row === 0 ? startOffset : strNode.byteRange.start, end: strNode.byteRange.end }, + value: parsed, + displayValue: text, + label: `[${row}]`, + children: row === 0 ? [versionNode, strNode] : [strNode], + metadata: { serializationVersion: 1, jsonText: text }, + }); + } + return values; + } + + /** + * FLATTENED JSON (Object serialization version 3). One sub-column per path, + * no shared-data store. Two-phase layout: all path state prefixes first, then + * all path data. Typed paths are decoded in their declared type; dynamic paths + * are each a FLATTENED Dynamic column. See docs/full_native_spec.md. + */ + private decodeJSONColumnFlattened(type: ClickHouseType, rowCount: number, startOffset: number): AstNode[] { + const jsonType = type as { kind: 'JSON'; typedPaths?: Map }; + const structureChildren: AstNode[] = []; + + const versionNode = this.decodeUInt64(); + versionNode.label = 'version'; + structureChildren.push(versionNode); + + // num_dynamic_paths + dynamic path names (wire/sorted order) + const numDynStart = this.reader.offset; + const { value: numDynamicPaths } = decodeLEB128(this.reader); + structureChildren.push({ + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: numDynStart, end: this.reader.offset }, + value: numDynamicPaths, + displayValue: String(numDynamicPaths), + label: 'num_dynamic_paths', + }); + const dynamicPathNames: string[] = []; + for (let i = 0; i < numDynamicPaths; i++) { + const node = this.decodeString(); + node.label = `dynamic_path[${i}]`; + structureChildren.push(node); + dynamicPathNames.push(node.value as string); + } + + // Typed paths come from the type string, decoded in sorted-by-name order. + const typedPaths = jsonType.typedPaths + ? Array.from(jsonType.typedPaths.entries()).sort((a, b) => a[0].localeCompare(b[0])) + : []; + + // --- Prefix phase: typed path prefixes (empty for leaf), then dynamic Dynamic prefixes --- + const typedPrefixes = typedPaths.map(([, t]) => this.readColumnPrefix(t)); + const dynamicStructures = dynamicPathNames.map((name) => + this.readFlattenedDynamicStructure(structureChildren, name), + ); + + // --- Data phase: typed path data, then dynamic path data --- + const typedData = new Map(); + typedPaths.forEach(([name, t], idx) => { + typedData.set(name, this.decodeColumnWithPrefix(t, rowCount, typedPrefixes[idx])); + }); + const dynamicData = dynamicStructures.map((s) => this.readFlattenedDynamicData(s, rowCount)); + + // Assemble per-row objects. + const values: AstNode[] = []; + for (let row = 0; row < rowCount; row++) { + const children: AstNode[] = row === 0 ? [...structureChildren] : []; + const jsonValue: Record = {}; + + for (const [name] of typedPaths) { + const node = typedData.get(name)![row]; + this.setNestedValue(jsonValue, name, node.value); + children.push({ + id: this.generateId(), + type: 'JSON.typed_path', + byteRange: node.byteRange, + value: { [name]: node.value }, + displayValue: `${name}: ${node.displayValue}`, + label: name, + children: [node], + }); + } + dynamicPathNames.forEach((name, i) => { + const node = dynamicData[i][row]; + // A dynamic path that is NULL for this row contributes no key. + if (node.value !== null) { + this.setNestedValue(jsonValue, name, node.value); + } + children.push({ + id: this.generateId(), + type: 'JSON.dynamic_path', + byteRange: node.byteRange, + value: { [name]: node.value }, + displayValue: `${name}: ${node.displayValue}`, + label: name, + children: [node], + }); + }); + + const totalPaths = typedPaths.length + numDynamicPaths; + values.push({ + id: this.generateId(), + type: 'JSON', + byteRange: { start: startOffset, end: this.reader.offset }, + value: jsonValue, + displayValue: `{${totalPaths} paths}`, + label: `[${row}]`, + children, + metadata: { serializationVersion: 3 }, + }); + } + return values; + } + + /** + * Read a FLATTENED Dynamic state prefix (version 3): version UInt64 + num_types + * VarUInt + type-name Strings + per-type nested prefixes (empty for leaf types). + * The structure nodes are appended to `parentChildren` for the AST. + */ + private readFlattenedDynamicStructure( + parentChildren: AstNode[], + pathName: string, + ): { numTypes: number; typeNames: string[]; variants: ClickHouseType[]; variantPrefixes: unknown[] } { + const start = this.reader.offset; + const children: AstNode[] = []; + + const versionNode = this.decodeUInt64(); + versionNode.label = 'dynamic_version'; + children.push(versionNode); + if (versionNode.value !== 3n) { + throw new Error( + `Native format: FLATTENED JSON dynamic path expected Dynamic version 3, got ${versionNode.value}`, + ); + } + + const numTypesStart = this.reader.offset; + const { value: numTypes } = decodeLEB128(this.reader); + children.push({ + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: numTypesStart, end: this.reader.offset }, + value: numTypes, + displayValue: String(numTypes), + label: 'num_dynamic_types', + }); + + const typeNames: string[] = []; + for (let t = 0; t < numTypes; t++) { + const node = this.decodeString(); + node.label = `type_name[${t}]`; + children.push(node); + typeNames.push(node.value as string); + } + const variants = typeNames.map((name) => parseType(name)); + const variantPrefixes = variants.map((v) => this.readColumnPrefix(v)); + + parentChildren.push({ + id: this.generateId(), + type: 'Dynamic.structure', + byteRange: { start, end: this.reader.offset }, + value: { typeNames, numTypes }, + displayValue: `Dynamic structure for "${pathName}" (${numTypes} types, v3)`, + label: `${pathName}.structure`, + children, + }); + + return { numTypes, typeNames, variants, variantPrefixes }; + } + + /** + * Read the per-block data of a FLATTENED Dynamic sub-column given its already + * read structure: num_rows discriminators (width by num_types + 1, NULL = + * num_types) followed by each type's dense run in wire order. + */ + private readFlattenedDynamicData( + struct: { numTypes: number; typeNames: string[]; variants: ClickHouseType[]; variantPrefixes: unknown[] }, + rowCount: number, + ): AstNode[] { + const { numTypes, typeNames, variants, variantPrefixes } = struct; + const nullDiscriminator = numTypes; + const discWidth = this.smallestIndexWidth(numTypes + 1); + + const discriminators: number[] = []; + const discRanges: ByteRange[] = []; + for (let r = 0; r < rowCount; r++) { + const start = this.reader.offset; + discriminators.push(this.readDiscriminator(discWidth)); + discRanges.push({ start, end: this.reader.offset }); + } + + const countPerType = new Array(numTypes).fill(0); + for (const disc of discriminators) { + if (disc !== nullDiscriminator && disc < numTypes) countPerType[disc]++; + } + const runData: AstNode[][] = []; + for (let t = 0; t < numTypes; t++) { + runData[t] = countPerType[t] > 0 + ? this.decodeColumnWithPrefix(variants[t], countPerType[t], variantPrefixes[t]) + : []; + } + + const positions = new Array(numTypes).fill(0); + const values: AstNode[] = []; + for (let r = 0; r < rowCount; r++) { + const disc = discriminators[r]; + const discNode: AstNode = { + id: this.generateId(), + type: `UInt${discWidth * 8}`, + byteRange: discRanges[r], + value: disc, + displayValue: `${disc} → ${disc === nullDiscriminator ? 'NULL' : (typeNames[disc] ?? `unknown(${disc})`)}`, + label: 'discriminator', + }; + if (disc === nullDiscriminator || disc >= numTypes) { + values.push({ + id: this.generateId(), + type: 'Dynamic(NULL)', + byteRange: discRanges[r], + value: null, + displayValue: 'NULL', + children: [discNode], + }); + continue; } + const node = runData[disc][positions[disc]++]; + values.push({ + id: this.generateId(), + type: `Dynamic(${node.type})`, + byteRange: { start: discRanges[r].start, end: node.byteRange.end }, + value: node.value, + displayValue: node.displayValue, + children: [discNode, node], + }); + } + return values; + } - // Read max_dynamic_types (V1 only) - if (dynamicSerializationVersion === 1) { - const maxTypesStart = this.reader.offset; - const { value: maxTypes } = decodeLEB128(this.reader); - const maxTypesNode: AstNode = { + /** + * Helper to set a nested path value in an object + * e.g., setNestedValue(obj, "nested.x", 10) -> obj.nested.x = 10 + */ + private setNestedValue(obj: Record, path: string, value: unknown): void { + const parts = path.split('.'); + let current = obj; + for (let i = 0; i < parts.length - 1; i++) { + if (!(parts[i] in current)) { + current[parts[i]] = {}; + } + current = current[parts[i]] as Record; + } + current[parts[parts.length - 1]] = value; + } + + /** + * Helper to convert BigInt values to displayable format + */ + private convertBigIntForDisplay(obj: unknown): unknown { + if (obj === null || obj === undefined) return obj; + if (typeof obj === 'bigint') return obj.toString(); + if (Array.isArray(obj)) return obj.map(v => this.convertBigIntForDisplay(v)); + if (typeof obj === 'object') { + const result: Record = {}; + for (const [k, v] of Object.entries(obj)) { + result[k] = this.convertBigIntForDisplay(v); + } + return result; + } + return obj; + } + + // Point decoder (same as RowBinary) + private decodePoint(): AstNode { + const startOffset = this.reader.offset; + const x = this.decodeFloat64(); + const y = this.decodeFloat64(); + + x.label = 'x'; + y.label = 'y'; + + return { + id: this.generateId(), + type: 'Point', + byteRange: { start: startOffset, end: this.reader.offset }, + value: [x.value, y.value], + displayValue: `(${x.displayValue}, ${y.displayValue})`, + children: [x, y], + }; + } + + // ========================================= + // Geo type column decoders + // ========================================= + + /** + * Ring = Array(Point) in columnar format + */ + private decodeRingColumn(rowCount: number): AstNode[] { + // Read cumulative offsets + const offsets: bigint[] = []; + for (let i = 0; i < rowCount; i++) { + const { value } = this.reader.readUInt64LE(); + offsets.push(value); + } + + // Calculate sizes + const totalPoints = rowCount > 0 ? Number(offsets[rowCount - 1]) : 0; + const sizes: number[] = []; + let prevOffset = 0n; + for (let i = 0; i < rowCount; i++) { + sizes.push(Number(offsets[i] - prevOffset)); + prevOffset = offsets[i]; + } + + // Read all points (columnar - all X's first, then all Y's) + const allX: AstNode[] = []; + for (let i = 0; i < totalPoints; i++) { + allX.push(this.decodeFloat64()); + } + const allY: AstNode[] = []; + for (let i = 0; i < totalPoints; i++) { + allY.push(this.decodeFloat64()); + } + + // Assemble points + const allPoints: AstNode[] = []; + for (let i = 0; i < totalPoints; i++) { + allX[i].label = 'x'; + allY[i].label = 'y'; + allPoints.push({ + id: this.generateId(), + type: 'Point', + byteRange: { start: allX[i].byteRange.start, end: allY[i].byteRange.end }, + value: [allX[i].value, allY[i].value], + displayValue: `(${allX[i].displayValue}, ${allY[i].displayValue})`, + children: [allX[i], allY[i]], + }); + } + + // Distribute to rings + const values: AstNode[] = []; + let pointIndex = 0; + for (let i = 0; i < rowCount; i++) { + const size = sizes[i]; + const ringPoints = allPoints.slice(pointIndex, pointIndex + size); + pointIndex += size; + + ringPoints.forEach((p, j) => { + p.label = `[${j}]`; + }); + + values.push({ + id: this.generateId(), + type: 'Ring', + byteRange: { start: ringPoints[0]?.byteRange.start ?? this.reader.offset, end: this.reader.offset }, + value: ringPoints.map(p => p.value), + displayValue: `[${ringPoints.map(p => p.displayValue).join(', ')}]`, + label: `[${i}]`, + children: ringPoints, + metadata: { size }, + }); + } + + return values; + } + + /** + * Polygon = Array(Ring) in columnar format + */ + private decodePolygonColumn(rowCount: number): AstNode[] { + // First: Array of Ring offsets + const ringOffsets: bigint[] = []; + for (let i = 0; i < rowCount; i++) { + const { value } = this.reader.readUInt64LE(); + ringOffsets.push(value); + } + + const totalRings = rowCount > 0 ? Number(ringOffsets[rowCount - 1]) : 0; + const ringSizes: number[] = []; + let prevOffset = 0n; + for (let i = 0; i < rowCount; i++) { + ringSizes.push(Number(ringOffsets[i] - prevOffset)); + prevOffset = ringOffsets[i]; + } + + // Decode all rings + const allRings = this.decodeRingColumn(totalRings); + + // Distribute to polygons + const values: AstNode[] = []; + let ringIndex = 0; + for (let i = 0; i < rowCount; i++) { + const size = ringSizes[i]; + const polygonRings = allRings.slice(ringIndex, ringIndex + size); + ringIndex += size; + + polygonRings.forEach((r, j) => { + r.label = `[${j}]`; + }); + + values.push({ + id: this.generateId(), + type: 'Polygon', + byteRange: { start: polygonRings[0]?.byteRange.start ?? this.reader.offset, end: this.reader.offset }, + value: polygonRings.map(r => r.value), + displayValue: `[${polygonRings.map(r => r.displayValue).join(', ')}]`, + label: `[${i}]`, + children: polygonRings, + metadata: { size }, + }); + } + + return values; + } + + /** + * MultiPolygon = Array(Polygon) in columnar format + */ + private decodeMultiPolygonColumn(rowCount: number): AstNode[] { + // First: Array of Polygon offsets + const polyOffsets: bigint[] = []; + for (let i = 0; i < rowCount; i++) { + const { value } = this.reader.readUInt64LE(); + polyOffsets.push(value); + } + + const totalPolygons = rowCount > 0 ? Number(polyOffsets[rowCount - 1]) : 0; + const polySizes: number[] = []; + let prevOffset = 0n; + for (let i = 0; i < rowCount; i++) { + polySizes.push(Number(polyOffsets[i] - prevOffset)); + prevOffset = polyOffsets[i]; + } + + // Decode all polygons + const allPolygons = this.decodePolygonColumn(totalPolygons); + + // Distribute to multipolygons + const values: AstNode[] = []; + let polyIndex = 0; + for (let i = 0; i < rowCount; i++) { + const size = polySizes[i]; + const multiPolyPolygons = allPolygons.slice(polyIndex, polyIndex + size); + polyIndex += size; + + multiPolyPolygons.forEach((p, j) => { + p.label = `[${j}]`; + }); + + values.push({ + id: this.generateId(), + type: 'MultiPolygon', + byteRange: { start: multiPolyPolygons[0]?.byteRange.start ?? this.reader.offset, end: this.reader.offset }, + value: multiPolyPolygons.map(p => p.value), + displayValue: `[${multiPolyPolygons.map(p => p.displayValue).join(', ')}]`, + label: `[${i}]`, + children: multiPolyPolygons, + metadata: { size }, + }); + } + + return values; + } + + /** + * LineString = Array(Point) in columnar format (same as Ring) + */ + private decodeLineStringColumn(rowCount: number): AstNode[] { + const rings = this.decodeRingColumn(rowCount); + // Change type from Ring to LineString + for (const r of rings) { + r.type = 'LineString'; + } + return rings; + } + + /** + * MultiLineString = Array(LineString) in columnar format + */ + private decodeMultiLineStringColumn(rowCount: number): AstNode[] { + // First: Array of LineString offsets + const lineOffsets: bigint[] = []; + for (let i = 0; i < rowCount; i++) { + const { value } = this.reader.readUInt64LE(); + lineOffsets.push(value); + } + + const totalLines = rowCount > 0 ? Number(lineOffsets[rowCount - 1]) : 0; + const lineSizes: number[] = []; + let prevOffset = 0n; + for (let i = 0; i < rowCount; i++) { + lineSizes.push(Number(lineOffsets[i] - prevOffset)); + prevOffset = lineOffsets[i]; + } + + // Decode all linestrings + const allLines = this.decodeLineStringColumn(totalLines); + + // Distribute to multilinestrings + const values: AstNode[] = []; + let lineIndex = 0; + for (let i = 0; i < rowCount; i++) { + const size = lineSizes[i]; + const multiLineLines = allLines.slice(lineIndex, lineIndex + size); + lineIndex += size; + + multiLineLines.forEach((l, j) => { + l.label = `[${j}]`; + }); + + values.push({ + id: this.generateId(), + type: 'MultiLineString', + byteRange: { start: multiLineLines[0]?.byteRange.start ?? this.reader.offset, end: this.reader.offset }, + value: multiLineLines.map(l => l.value), + displayValue: `[${multiLineLines.map(l => l.displayValue).join(', ')}]`, + label: `[${i}]`, + children: multiLineLines, + metadata: { size }, + }); + } + + return values; + } + + /** + * Geometry is a Variant of geo types: + * ClickHouse sorts variant types alphabetically: + * 0=LineString, 1=MultiLineString, 2=MultiPolygon, 3=Point, 4=Polygon, 5=Ring + */ + private decodeGeometryColumn(rowCount: number): AstNode[] { + // Geometry is internally a Variant type with alphabetically sorted geo types + const geoVariants: ClickHouseType[] = [ + { kind: 'LineString' }, // 0 + { kind: 'MultiLineString' }, // 1 + { kind: 'MultiPolygon' }, // 2 + { kind: 'Point' }, // 3 + { kind: 'Polygon' }, // 4 + { kind: 'Ring' }, // 5 + ]; + + // Decode as Variant + const variantValues = this.decodeVariantColumn(geoVariants, rowCount); + + // Update type to Geometry + for (const v of variantValues) { + const originalType = v.type; + v.metadata = { ...v.metadata, geometryType: originalType }; + } + + return variantValues; + } + + /** + * QBit columnar decoder - bit-transposed vector format + * + * In Native format, QBit uses bit-transposed encoding: + * - Data is organized as: for each bit plane (MSB to LSB), for each row, 1 byte + * - Each byte contains packed bits from all vector elements + * - Element 0 → bit 0, element 1 → bit 1, etc. + * + * For Float32: 32 bit planes, so 32 bytes per row + * For Float64: 64 bit planes, so 64 bytes per row + * For BFloat16: 16 bit planes, so 16 bytes per row + */ + private decodeQBitColumn(elementType: ClickHouseType, dimension: number, rowCount: number): AstNode[] { + const startOffset = this.reader.offset; + + // Determine bits per element based on type + const bitsPerElement = elementType.kind === 'Float64' ? 64 : elementType.kind === 'BFloat16' ? 16 : 32; + const totalBytes = bitsPerElement * rowCount; + + // Read all bit-transposed data + const { value: data } = this.reader.readBytes(totalBytes); + + // Decode each row + const values: AstNode[] = []; + for (let row = 0; row < rowCount; row++) { + const children: AstNode[] = []; + + for (let elem = 0; elem < dimension; elem++) { + // Reconstruct the float value for this element + let bits = 0n; + + for (let bitPlane = 0; bitPlane < bitsPerElement; bitPlane++) { + // Data is organized as: [bp31_row0, bp31_row1, ..., bp30_row0, bp30_row1, ...] + const byteIndex = bitPlane * rowCount + row; + const byte = data[byteIndex]; + + // Extract this element's bit from the byte + const bit = (byte >> elem) & 1; + + // Place it in the correct position (MSB first) + const bitPosition = bitsPerElement - 1 - bitPlane; + if (bit) { + bits |= 1n << BigInt(bitPosition); + } + } + + // Convert bits to float + const floatValue = this.bitsToFloat(bits, elementType.kind as 'Float32' | 'Float64' | 'BFloat16'); + + children.push({ id: this.generateId(), - type: 'VarUInt', - byteRange: { start: maxTypesStart, end: this.reader.offset }, - value: maxTypes, - displayValue: String(maxTypes), - label: 'max_dynamic_types', - }; - dynamicStructureChildren.push(maxTypesNode); + type: typeToString(elementType), + byteRange: { start: startOffset, end: this.reader.offset }, + value: floatValue, + displayValue: floatValue.toString(), + label: `[${elem}]`, + }); } - // Read num_dynamic_types - const numTypesStart = this.reader.offset; - const { value: numTypes } = decodeLEB128(this.reader); - const numTypesNode: AstNode = { - id: this.generateId(), - type: 'VarUInt', - byteRange: { start: numTypesStart, end: this.reader.offset }, - value: numTypes, - displayValue: String(numTypes), - label: 'num_dynamic_types', - }; - dynamicStructureChildren.push(numTypesNode); - - // Read type names - const typeNames: string[] = []; - for (let t = 0; t < numTypes; t++) { - const typeNameNode = this.decodeString(); - typeNameNode.label = `type_name[${t}]`; - dynamicStructureChildren.push(typeNameNode); - typeNames.push(typeNameNode.value as string); - } - const variants = typeNames.map(name => parseType(name)); - - // Read variant mode - const modeNode = this.decodeUInt64(); - modeNode.label = 'variant_mode'; - dynamicStructureChildren.push(modeNode); - - // Build discriminator mapping based on alphabetical sort of [typeNames + "SharedVariant"] - const sortedWithShared = [...typeNames, 'SharedVariant'].sort(); - const discToTypeIndex = new Map(); - for (let idx = 0; idx < sortedWithShared.length; idx++) { - const name = sortedWithShared[idx]; - if (name === 'SharedVariant') { - discToTypeIndex.set(idx, -1); - } else { - const originalIndex = typeNames.indexOf(name); - discToTypeIndex.set(idx, originalIndex); - } - } - - // Read variant state prefixes for types that need them - // In Native format with BASIC variant mode, the prefix for nested complex types - // is written BEFORE the discriminators. - const variantPrefixes = variants.map((v, idx) => { - const prefix = this.readColumnPrefix(v); - // If the prefix contains structure children (e.g., nested JSON), add them - if (prefix && typeof prefix === 'object' && 'structureChildren' in prefix) { - const jsonPrefix = prefix as { structureChildren: AstNode[] }; - for (const child of jsonPrefix.structureChildren) { - child.label = `${typeNames[idx]}.${child.label || ''}`; - dynamicStructureChildren.push(child); - } - } - return prefix; - }); - - // Create a container node for this dynamic path's structure - const dynamicPathName = dynamicPathNames[i]; - const dynamicStructureNode: AstNode = { - id: this.generateId(), - type: 'Dynamic.structure', - byteRange: { start: dynamicStructureStart, end: this.reader.offset }, - value: { typeNames, numTypes }, - displayValue: `Dynamic structure for "${dynamicPathName}" (${numTypes} types)`, - label: `${dynamicPathName}.structure`, - children: dynamicStructureChildren, + const typeStr = `QBit(${typeToString(elementType)}, ${dimension})`; + values.push({ + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: this.reader.offset }, + value: children.map(c => c.value), + displayValue: `[${children.map(c => c.displayValue).join(', ')}]`, + label: `[${row}]`, + children, + metadata: { dimension, elementType: typeToString(elementType) }, + }); + } + + return values; + } + + /** + * Convert bit pattern to float value + */ + private bitsToFloat(bits: bigint, type: 'Float32' | 'Float64' | 'BFloat16'): number { + const buffer = new ArrayBuffer(8); + const view = new DataView(buffer); + + if (type === 'Float32') { + view.setUint32(0, Number(bits), true); + return view.getFloat32(0, true); + } else if (type === 'Float64') { + view.setBigUint64(0, bits, true); + return view.getFloat64(0, true); + } else { + // BFloat16: 16-bit format, convert to Float32 by left-shifting + const float32Bits = Number(bits) << 16; + view.setUint32(0, float32Bits, false); + return view.getFloat32(0, false); + } + } + + // Single QBit value decoder (for nested contexts - delegates to column decoder) + private decodeQBit(elementType: ClickHouseType, dimension: number): AstNode { + const values = this.decodeQBitColumn(elementType, dimension, 1); + return values[0]; + } + + // AggregateFunction column decoder - stored as length-prefixed binary states + private decodeAggregateFunctionColumn(functionName: string, argTypes: ClickHouseType[], rowCount: number): AstNode[] { + const values: AstNode[] = []; + + for (let i = 0; i < rowCount; i++) { + const node = this.decodeAggregateFunction(functionName, argTypes); + node.label = `[${i}]`; + values.push(node); + } + + return values; + } + + // Single AggregateFunction value decoder - format is function-specific, NO length prefix + private decodeAggregateFunction(functionName: string, argTypes: ClickHouseType[]): AstNode { + const startOffset = this.reader.offset; + const children: AstNode[] = []; + const funcLower = functionName.toLowerCase(); + + const argTypesStr = argTypes.map(typeToString).join(', '); + const typeStr = argTypesStr + ? `AggregateFunction(${functionName}, ${argTypesStr})` + : `AggregateFunction(${functionName})`; + + let displayValue: string; + let value: unknown; + + if (funcLower === 'avg') { + // avg: numerator (type depends on arg) + VarUInt denominator + const numNode = argTypes.length > 0 + ? this.decodeValue(argTypes[0]) + : this.decodeUInt64(); + numNode.label = 'numerator (sum)'; + children.push(numNode); + + const denomStart = this.reader.offset; + const { value: denominator } = decodeLEB128(this.reader); + const denomNode: AstNode = { + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: denomStart, end: this.reader.offset }, + value: denominator, + displayValue: String(denominator), + label: 'denominator (count)', }; - structureChildren.push(dynamicStructureNode); + children.push(denomNode); - dynamicStructures.push({ - serializationVersion: dynamicSerializationVersion, - typeNames, - variants, - numTypes, - discToTypeIndex, - variantPrefixes, - }); + const sum = numNode.value; + const avg = denominator > 0 ? Number(sum) / denominator : 0; + displayValue = `avg=${avg.toFixed(2)} (sum=${sum}, count=${denominator})`; + value = { sum, count: denominator, avg }; + } else if (funcLower === 'sum') { + // sum: fixed-size value based on argument type + const sumNode = argTypes.length > 0 + ? this.decodeValue(argTypes[0]) + : this.decodeUInt64(); + sumNode.label = 'sum'; + children.push(sumNode); + displayValue = `sum=${sumNode.displayValue}`; + value = sumNode.value; + } else if (funcLower === 'count') { + // count: VarUInt + const countStart = this.reader.offset; + const { value: count } = decodeLEB128(this.reader); + const countNode: AstNode = { + id: this.generateId(), + type: 'VarUInt', + byteRange: { start: countStart, end: this.reader.offset }, + value: count, + displayValue: String(count), + label: 'count', + }; + children.push(countNode); + displayValue = `count=${count}`; + value = count; + } else { + // Unknown aggregate function - we can't decode without knowing the format + throw new Error( + `AggregateFunction(${functionName}) has no length prefix and format is unknown. ` + + `Supported: avg, sum, count` + ); } return { - serializationVersion: objectSerializationVersion, - structureChildren, - dynamicPathNames, - dynamicStructures, - typedSubColumns, + id: this.generateId(), + type: typeStr, + byteRange: { start: startOffset, end: this.reader.offset }, + value, + displayValue, + children, + metadata: { functionName, argTypes: argTypesStr }, }; } - - /** - * Read JSON column data using pre-read structure. - * This allows Array(JSON) to read structure first, then offsets, then data. - */ - private readJSONColumnData( - structure: ReturnType, - rowCount: number, - startOffset: number - ): AstNode[] { - const { structureChildren, dynamicPathNames, dynamicStructures, typedSubColumns } = structure; - const numDynamicPaths = dynamicPathNames.length; - const values: AstNode[] = []; - - // Read typed paths data (if any) - const typedPathData: Map = new Map(); - if (typedSubColumns && typedSubColumns.size > 0) { - const sortedTypedPaths = Array.from(typedSubColumns.entries()).sort((a, b) => - a[0].localeCompare(b[0]) - ); - for (const [pathName, pathType] of sortedTypedPaths) { - const nodes = this.decodeColumnData(pathType, rowCount).values; - typedPathData.set(pathName, nodes); - } - } - - // Read ALL Dynamic data (discriminator + values for each path, sequentially) - const dynamicPathData: AstNode[][] = []; - - for (let i = 0; i < numDynamicPaths; i++) { - const { typeNames, variants, numTypes, discToTypeIndex, variantPrefixes } = dynamicStructures[i]; - - // Build reverse mapping: discriminator -> type name for display - const discToTypeName = new Map(); - const sortedWithShared = [...typeNames, 'SharedVariant'].sort(); - for (let idx = 0; idx < sortedWithShared.length; idx++) { - discToTypeName.set(idx, sortedWithShared[idx]); - } - - // Read discriminators for all rows (with byte tracking) - const discriminators: { value: number; range: { start: number; end: number } }[] = []; - for (let r = 0; r < rowCount; r++) { - const start = this.reader.offset; - const { value: disc } = this.reader.readUInt8(); - discriminators.push({ value: disc, range: { start, end: this.reader.offset } }); - } - - // Count values per variant using the alphabetical discriminator mapping - const countPerVariant: number[] = new Array(numTypes).fill(0); - for (const { value: disc } of discriminators) { - if (disc === 255) continue; // NULL - const typeIdx = discToTypeIndex.get(disc); - if (typeIdx !== undefined && typeIdx >= 0) { - countPerVariant[typeIdx]++; - } - // typeIdx === -1 means SharedVariant, handled separately - } - - // Read data for each type (in original order, not sorted order) - const variantData: AstNode[][] = []; - for (let v = 0; v < numTypes; v++) { - const count = countPerVariant[v]; - if (count > 0) { - variantData[v] = this.decodeColumnWithPrefix(variants[v], count, variantPrefixes[v]); - } else { - variantData[v] = []; - } - } - - // Build values for this path - const variantPositions: number[] = new Array(numTypes).fill(0); - const pathValues: AstNode[] = []; - for (let r = 0; r < rowCount; r++) { - const { value: disc, range: discRange } = discriminators[r]; - const typeName = disc === 255 ? 'NULL' : (discToTypeName.get(disc) ?? `Unknown(${disc})`); - - // Create discriminator AST node - const discNode: AstNode = { - id: this.generateId(), - type: 'UInt8', - byteRange: discRange, - value: disc, - displayValue: `${disc} → ${typeName}`, - label: 'discriminator', - }; - - if (disc === 255) { - // NULL discriminator - pathValues.push({ - id: this.generateId(), - type: 'Dynamic(NULL)', - byteRange: discRange, - value: null, - displayValue: 'NULL', - children: [discNode], - }); - } else { - const typeIdx = discToTypeIndex.get(disc); - if (typeIdx !== undefined && typeIdx >= 0) { - // Valid type - const node = variantData[typeIdx][variantPositions[typeIdx]++]; - pathValues.push({ - id: this.generateId(), - type: `Dynamic(${node.type})`, - byteRange: { start: discRange.start, end: node.byteRange.end }, - value: node.value, - displayValue: node.displayValue, - children: [discNode, node], - }); - } else if (typeIdx === -1) { - // SharedVariant - value is in shared data section - pathValues.push({ - id: this.generateId(), - type: 'Dynamic(SharedVariant)', - byteRange: discRange, - value: null, - displayValue: 'SharedVariant', - children: [discNode], - }); - } else { - // Unknown discriminator - pathValues.push({ - id: this.generateId(), - type: 'Dynamic(Unknown)', - byteRange: discRange, - value: null, - displayValue: `Unknown(disc=${disc})`, - children: [discNode], - }); - } - } - } - dynamicPathData.push(pathValues); - } - - // 6. Read shared data offsets (cumulative entry counts, UInt64 per row) - const sharedOffsets: bigint[] = []; - const sharedOffsetNodes: AstNode[] = []; - for (let i = 0; i < rowCount; i++) { - const offsetNode = this.decodeUInt64(); - offsetNode.label = `shared_data_offset[${i}]`; - sharedOffsets.push(offsetNode.value as bigint); - sharedOffsetNodes.push(offsetNode); - } - const totalSharedEntries = rowCount > 0 ? Number(sharedOffsets[rowCount - 1]) : 0; - - // 7. Read shared data entries - const sharedDataEntries: Array<{ path: string; value: unknown }> = []; - for (let i = 0; i < totalSharedEntries; i++) { - // Read path name - const pathNode = this.decodeString(); - const path = pathNode.value as string; - - // Read binary-encoded value: size (VarUInt) + type_index (VarUInt) + value - const { value: _dataSize } = decodeLEB128(this.reader); - const { value: typeIndex } = decodeLEB128(this.reader); - const entryType = this.decodeDynamicBinaryType(typeIndex); - - let value: unknown = null; - if (entryType) { - const valueNodes = this.decodeColumnData(entryType, 1).values; - value = valueNodes[0]?.value; - } - - sharedDataEntries.push({ path, value }); - } - - // Map shared entries to rows - const sharedDataPerRow: Array> = []; - let prevOffset = 0n; - for (let i = 0; i < rowCount; i++) { - const currentOffset = sharedOffsets[i]; - const entriesForRow = sharedDataEntries.slice(Number(prevOffset), Number(currentOffset)); - sharedDataPerRow.push(entriesForRow); - prevOffset = currentOffset; - } - - // Build result nodes - for (let row = 0; row < rowCount; row++) { - const children: AstNode[] = [...structureChildren]; - const jsonValue: Record = {}; - - // Add typed path values - for (const [pathName, nodes] of typedPathData) { - const valueNode = nodes[row]; - this.setNestedValue(jsonValue, pathName, valueNode.value); - children.push({ - id: this.generateId(), - type: 'JSON.typed_path', - byteRange: valueNode.byteRange, - value: { [pathName]: valueNode.value }, - displayValue: `${pathName}: ${valueNode.displayValue}`, - label: pathName, - children: [valueNode], - }); - } - - // Add dynamic path values - for (let i = 0; i < numDynamicPaths; i++) { - const pathName = dynamicPathNames[i]; - const dynamicNode = dynamicPathData[i][row]; - this.setNestedValue(jsonValue, pathName, dynamicNode.value); - children.push({ - id: this.generateId(), - type: 'JSON.dynamic_path', - byteRange: dynamicNode.byteRange, - value: { [pathName]: dynamicNode.value }, - displayValue: `${pathName}: ${dynamicNode.displayValue}`, - label: pathName, - children: [dynamicNode], - }); - } - - // Add shared data offset node for this row - children.push(sharedOffsetNodes[row]); - - // Add shared data values for this row - const rowSharedEntries = sharedDataPerRow[row] || []; - for (const entry of rowSharedEntries) { - this.setNestedValue(jsonValue, entry.path, entry.value); - } - - const totalPaths = typedPathData.size + numDynamicPaths + rowSharedEntries.length; - values.push({ - id: this.generateId(), - type: 'JSON', - byteRange: { start: startOffset, end: this.reader.offset }, - value: jsonValue, - displayValue: `{${totalPaths} paths}`, - label: `[${row}]`, - children, - }); - } - - return values; - } - - /** - * Decode JSON column using V1 format (version=0) - * This calls readJSONColumnStructure then readJSONColumnData. - */ - private decodeJSONColumnV1(type: ClickHouseType, rowCount: number, startOffset: number): AstNode[] { - const jsonType = type as { kind: 'JSON'; typedPaths?: Map }; - const structure = this.readJSONColumnStructure(jsonType.typedPaths); - return this.readJSONColumnData(structure, rowCount, startOffset); - } - - /** - * Helper to set a nested path value in an object - * e.g., setNestedValue(obj, "nested.x", 10) -> obj.nested.x = 10 - */ - private setNestedValue(obj: Record, path: string, value: unknown): void { - const parts = path.split('.'); - let current = obj; - for (let i = 0; i < parts.length - 1; i++) { - if (!(parts[i] in current)) { - current[parts[i]] = {}; - } - current = current[parts[i]] as Record; - } - current[parts[parts.length - 1]] = value; - } - - /** - * Helper to convert BigInt values to displayable format - */ - private convertBigIntForDisplay(obj: unknown): unknown { - if (obj === null || obj === undefined) return obj; - if (typeof obj === 'bigint') return obj.toString(); - if (Array.isArray(obj)) return obj.map(v => this.convertBigIntForDisplay(v)); - if (typeof obj === 'object') { - const result: Record = {}; - for (const [k, v] of Object.entries(obj)) { - result[k] = this.convertBigIntForDisplay(v); - } - return result; - } - return obj; - } - - // Point decoder (same as RowBinary) - private decodePoint(): AstNode { - const startOffset = this.reader.offset; - const x = this.decodeFloat64(); - const y = this.decodeFloat64(); - - x.label = 'x'; - y.label = 'y'; - - return { - id: this.generateId(), - type: 'Point', - byteRange: { start: startOffset, end: this.reader.offset }, - value: [x.value, y.value], - displayValue: `(${x.displayValue}, ${y.displayValue})`, - children: [x, y], - }; - } - - // ========================================= - // Geo type column decoders - // ========================================= - - /** - * Ring = Array(Point) in columnar format - */ - private decodeRingColumn(rowCount: number): AstNode[] { - // Read cumulative offsets - const offsets: bigint[] = []; - for (let i = 0; i < rowCount; i++) { - const { value } = this.reader.readUInt64LE(); - offsets.push(value); - } - - // Calculate sizes - const totalPoints = rowCount > 0 ? Number(offsets[rowCount - 1]) : 0; - const sizes: number[] = []; - let prevOffset = 0n; - for (let i = 0; i < rowCount; i++) { - sizes.push(Number(offsets[i] - prevOffset)); - prevOffset = offsets[i]; - } - - // Read all points (columnar - all X's first, then all Y's) - const allX: AstNode[] = []; - for (let i = 0; i < totalPoints; i++) { - allX.push(this.decodeFloat64()); - } - const allY: AstNode[] = []; - for (let i = 0; i < totalPoints; i++) { - allY.push(this.decodeFloat64()); - } - - // Assemble points - const allPoints: AstNode[] = []; - for (let i = 0; i < totalPoints; i++) { - allX[i].label = 'x'; - allY[i].label = 'y'; - allPoints.push({ - id: this.generateId(), - type: 'Point', - byteRange: { start: allX[i].byteRange.start, end: allY[i].byteRange.end }, - value: [allX[i].value, allY[i].value], - displayValue: `(${allX[i].displayValue}, ${allY[i].displayValue})`, - children: [allX[i], allY[i]], - }); - } - - // Distribute to rings - const values: AstNode[] = []; - let pointIndex = 0; - for (let i = 0; i < rowCount; i++) { - const size = sizes[i]; - const ringPoints = allPoints.slice(pointIndex, pointIndex + size); - pointIndex += size; - - ringPoints.forEach((p, j) => { - p.label = `[${j}]`; - }); - - values.push({ - id: this.generateId(), - type: 'Ring', - byteRange: { start: ringPoints[0]?.byteRange.start ?? this.reader.offset, end: this.reader.offset }, - value: ringPoints.map(p => p.value), - displayValue: `[${ringPoints.map(p => p.displayValue).join(', ')}]`, - label: `[${i}]`, - children: ringPoints, - metadata: { size }, - }); - } - - return values; - } - - /** - * Polygon = Array(Ring) in columnar format - */ - private decodePolygonColumn(rowCount: number): AstNode[] { - // First: Array of Ring offsets - const ringOffsets: bigint[] = []; - for (let i = 0; i < rowCount; i++) { - const { value } = this.reader.readUInt64LE(); - ringOffsets.push(value); - } - - const totalRings = rowCount > 0 ? Number(ringOffsets[rowCount - 1]) : 0; - const ringSizes: number[] = []; - let prevOffset = 0n; - for (let i = 0; i < rowCount; i++) { - ringSizes.push(Number(ringOffsets[i] - prevOffset)); - prevOffset = ringOffsets[i]; - } - - // Decode all rings - const allRings = this.decodeRingColumn(totalRings); - - // Distribute to polygons - const values: AstNode[] = []; - let ringIndex = 0; - for (let i = 0; i < rowCount; i++) { - const size = ringSizes[i]; - const polygonRings = allRings.slice(ringIndex, ringIndex + size); - ringIndex += size; - - polygonRings.forEach((r, j) => { - r.label = `[${j}]`; - }); - - values.push({ - id: this.generateId(), - type: 'Polygon', - byteRange: { start: polygonRings[0]?.byteRange.start ?? this.reader.offset, end: this.reader.offset }, - value: polygonRings.map(r => r.value), - displayValue: `[${polygonRings.map(r => r.displayValue).join(', ')}]`, - label: `[${i}]`, - children: polygonRings, - metadata: { size }, - }); - } - - return values; - } - - /** - * MultiPolygon = Array(Polygon) in columnar format - */ - private decodeMultiPolygonColumn(rowCount: number): AstNode[] { - // First: Array of Polygon offsets - const polyOffsets: bigint[] = []; - for (let i = 0; i < rowCount; i++) { - const { value } = this.reader.readUInt64LE(); - polyOffsets.push(value); - } - - const totalPolygons = rowCount > 0 ? Number(polyOffsets[rowCount - 1]) : 0; - const polySizes: number[] = []; - let prevOffset = 0n; - for (let i = 0; i < rowCount; i++) { - polySizes.push(Number(polyOffsets[i] - prevOffset)); - prevOffset = polyOffsets[i]; - } - - // Decode all polygons - const allPolygons = this.decodePolygonColumn(totalPolygons); - - // Distribute to multipolygons - const values: AstNode[] = []; - let polyIndex = 0; - for (let i = 0; i < rowCount; i++) { - const size = polySizes[i]; - const multiPolyPolygons = allPolygons.slice(polyIndex, polyIndex + size); - polyIndex += size; - - multiPolyPolygons.forEach((p, j) => { - p.label = `[${j}]`; - }); - - values.push({ - id: this.generateId(), - type: 'MultiPolygon', - byteRange: { start: multiPolyPolygons[0]?.byteRange.start ?? this.reader.offset, end: this.reader.offset }, - value: multiPolyPolygons.map(p => p.value), - displayValue: `[${multiPolyPolygons.map(p => p.displayValue).join(', ')}]`, - label: `[${i}]`, - children: multiPolyPolygons, - metadata: { size }, - }); - } - - return values; - } - - /** - * LineString = Array(Point) in columnar format (same as Ring) - */ - private decodeLineStringColumn(rowCount: number): AstNode[] { - const rings = this.decodeRingColumn(rowCount); - // Change type from Ring to LineString - for (const r of rings) { - r.type = 'LineString'; - } - return rings; - } - - /** - * MultiLineString = Array(LineString) in columnar format - */ - private decodeMultiLineStringColumn(rowCount: number): AstNode[] { - // First: Array of LineString offsets - const lineOffsets: bigint[] = []; - for (let i = 0; i < rowCount; i++) { - const { value } = this.reader.readUInt64LE(); - lineOffsets.push(value); - } - - const totalLines = rowCount > 0 ? Number(lineOffsets[rowCount - 1]) : 0; - const lineSizes: number[] = []; - let prevOffset = 0n; - for (let i = 0; i < rowCount; i++) { - lineSizes.push(Number(lineOffsets[i] - prevOffset)); - prevOffset = lineOffsets[i]; - } - - // Decode all linestrings - const allLines = this.decodeLineStringColumn(totalLines); - - // Distribute to multilinestrings - const values: AstNode[] = []; - let lineIndex = 0; - for (let i = 0; i < rowCount; i++) { - const size = lineSizes[i]; - const multiLineLines = allLines.slice(lineIndex, lineIndex + size); - lineIndex += size; - - multiLineLines.forEach((l, j) => { - l.label = `[${j}]`; - }); - - values.push({ - id: this.generateId(), - type: 'MultiLineString', - byteRange: { start: multiLineLines[0]?.byteRange.start ?? this.reader.offset, end: this.reader.offset }, - value: multiLineLines.map(l => l.value), - displayValue: `[${multiLineLines.map(l => l.displayValue).join(', ')}]`, - label: `[${i}]`, - children: multiLineLines, - metadata: { size }, - }); - } - - return values; - } - - /** - * Geometry is a Variant of geo types: - * ClickHouse sorts variant types alphabetically: - * 0=LineString, 1=MultiLineString, 2=MultiPolygon, 3=Point, 4=Polygon, 5=Ring - */ - private decodeGeometryColumn(rowCount: number): AstNode[] { - // Geometry is internally a Variant type with alphabetically sorted geo types - const geoVariants: ClickHouseType[] = [ - { kind: 'LineString' }, // 0 - { kind: 'MultiLineString' }, // 1 - { kind: 'MultiPolygon' }, // 2 - { kind: 'Point' }, // 3 - { kind: 'Polygon' }, // 4 - { kind: 'Ring' }, // 5 - ]; - - // Decode as Variant - const variantValues = this.decodeVariantColumn(geoVariants, rowCount); - - // Update type to Geometry - for (const v of variantValues) { - const originalType = v.type; - v.metadata = { ...v.metadata, geometryType: originalType }; - } - - return variantValues; - } - - /** - * QBit columnar decoder - bit-transposed vector format - * - * In Native format, QBit uses bit-transposed encoding: - * - Data is organized as: for each bit plane (MSB to LSB), for each row, 1 byte - * - Each byte contains packed bits from all vector elements - * - Element 0 → bit 0, element 1 → bit 1, etc. - * - * For Float32: 32 bit planes, so 32 bytes per row - * For Float64: 64 bit planes, so 64 bytes per row - * For BFloat16: 16 bit planes, so 16 bytes per row - */ - private decodeQBitColumn(elementType: ClickHouseType, dimension: number, rowCount: number): AstNode[] { - const startOffset = this.reader.offset; - - // Determine bits per element based on type - const bitsPerElement = elementType.kind === 'Float64' ? 64 : elementType.kind === 'BFloat16' ? 16 : 32; - const totalBytes = bitsPerElement * rowCount; - - // Read all bit-transposed data - const { value: data } = this.reader.readBytes(totalBytes); - - // Decode each row - const values: AstNode[] = []; - for (let row = 0; row < rowCount; row++) { - const children: AstNode[] = []; - - for (let elem = 0; elem < dimension; elem++) { - // Reconstruct the float value for this element - let bits = 0n; - - for (let bitPlane = 0; bitPlane < bitsPerElement; bitPlane++) { - // Data is organized as: [bp31_row0, bp31_row1, ..., bp30_row0, bp30_row1, ...] - const byteIndex = bitPlane * rowCount + row; - const byte = data[byteIndex]; - - // Extract this element's bit from the byte - const bit = (byte >> elem) & 1; - - // Place it in the correct position (MSB first) - const bitPosition = bitsPerElement - 1 - bitPlane; - if (bit) { - bits |= 1n << BigInt(bitPosition); - } - } - - // Convert bits to float - const floatValue = this.bitsToFloat(bits, elementType.kind as 'Float32' | 'Float64' | 'BFloat16'); - - children.push({ - id: this.generateId(), - type: typeToString(elementType), - byteRange: { start: startOffset, end: this.reader.offset }, - value: floatValue, - displayValue: floatValue.toString(), - label: `[${elem}]`, - }); - } - - const typeStr = `QBit(${typeToString(elementType)}, ${dimension})`; - values.push({ - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: this.reader.offset }, - value: children.map(c => c.value), - displayValue: `[${children.map(c => c.displayValue).join(', ')}]`, - label: `[${row}]`, - children, - metadata: { dimension, elementType: typeToString(elementType) }, - }); - } - - return values; - } - - /** - * Convert bit pattern to float value - */ - private bitsToFloat(bits: bigint, type: 'Float32' | 'Float64' | 'BFloat16'): number { - const buffer = new ArrayBuffer(8); - const view = new DataView(buffer); - - if (type === 'Float32') { - view.setUint32(0, Number(bits), true); - return view.getFloat32(0, true); - } else if (type === 'Float64') { - view.setBigUint64(0, bits, true); - return view.getFloat64(0, true); - } else { - // BFloat16: 16-bit format, convert to Float32 by left-shifting - const float32Bits = Number(bits) << 16; - view.setUint32(0, float32Bits, false); - return view.getFloat32(0, false); - } - } - - // Single QBit value decoder (for nested contexts - delegates to column decoder) - private decodeQBit(elementType: ClickHouseType, dimension: number): AstNode { - const values = this.decodeQBitColumn(elementType, dimension, 1); - return values[0]; - } - - // AggregateFunction column decoder - stored as length-prefixed binary states - private decodeAggregateFunctionColumn(functionName: string, argTypes: ClickHouseType[], rowCount: number): AstNode[] { - const values: AstNode[] = []; - - for (let i = 0; i < rowCount; i++) { - const node = this.decodeAggregateFunction(functionName, argTypes); - node.label = `[${i}]`; - values.push(node); - } - - return values; - } - - // Single AggregateFunction value decoder - format is function-specific, NO length prefix - private decodeAggregateFunction(functionName: string, argTypes: ClickHouseType[]): AstNode { - const startOffset = this.reader.offset; - const children: AstNode[] = []; - const funcLower = functionName.toLowerCase(); - - const argTypesStr = argTypes.map(typeToString).join(', '); - const typeStr = argTypesStr - ? `AggregateFunction(${functionName}, ${argTypesStr})` - : `AggregateFunction(${functionName})`; - - let displayValue: string; - let value: unknown; - - if (funcLower === 'avg') { - // avg: numerator (type depends on arg) + VarUInt denominator - const numNode = argTypes.length > 0 - ? this.decodeValue(argTypes[0]) - : this.decodeUInt64(); - numNode.label = 'numerator (sum)'; - children.push(numNode); - - const denomStart = this.reader.offset; - const { value: denominator } = decodeLEB128(this.reader); - const denomNode: AstNode = { - id: this.generateId(), - type: 'VarUInt', - byteRange: { start: denomStart, end: this.reader.offset }, - value: denominator, - displayValue: String(denominator), - label: 'denominator (count)', - }; - children.push(denomNode); - - const sum = numNode.value; - const avg = denominator > 0 ? Number(sum) / denominator : 0; - displayValue = `avg=${avg.toFixed(2)} (sum=${sum}, count=${denominator})`; - value = { sum, count: denominator, avg }; - } else if (funcLower === 'sum') { - // sum: fixed-size value based on argument type - const sumNode = argTypes.length > 0 - ? this.decodeValue(argTypes[0]) - : this.decodeUInt64(); - sumNode.label = 'sum'; - children.push(sumNode); - displayValue = `sum=${sumNode.displayValue}`; - value = sumNode.value; - } else if (funcLower === 'count') { - // count: VarUInt - const countStart = this.reader.offset; - const { value: count } = decodeLEB128(this.reader); - const countNode: AstNode = { - id: this.generateId(), - type: 'VarUInt', - byteRange: { start: countStart, end: this.reader.offset }, - value: count, - displayValue: String(count), - label: 'count', - }; - children.push(countNode); - displayValue = `count=${count}`; - value = count; - } else { - // Unknown aggregate function - we can't decode without knowing the format - throw new Error( - `AggregateFunction(${functionName}) has no length prefix and format is unknown. ` + - `Supported: avg, sum, count` - ); - } - - return { - id: this.generateId(), - type: typeStr, - byteRange: { start: startOffset, end: this.reader.offset }, - value, - displayValue, - children, - metadata: { functionName, argTypes: argTypesStr }, - }; - } - - /** - * Decode an Interval type (stored as Int64) - */ - private decodeInterval(typeName: string, unit: string): AstNode { - const { value, range } = this.reader.readInt64LE(); - return { - id: this.generateId(), - type: typeName, - byteRange: range, - value, - displayValue: `${value} ${unit}`, - }; - } -} + + /** + * Decode an Interval type (stored as Int64) + */ + private decodeInterval(typeName: string, unit: string): AstNode { + const { value, range } = this.reader.readInt64LE(); + return { + id: this.generateId(), + type: typeName, + byteRange: range, + value, + displayValue: `${value} ${unit}`, + }; + } +} diff --git a/src/core/decoder/native-spec.test.ts b/src/core/decoder/native-spec.test.ts new file mode 100644 index 0000000..df641d2 --- /dev/null +++ b/src/core/decoder/native-spec.test.ts @@ -0,0 +1,432 @@ +/** + * Regression tests that verify the Native decoder against the byte-level + * examples in docs/full_native_spec.md. Each test encodes a single-column, + * protocol-0 block (no BlockInfo, no has_custom_serialization byte) so the + * column `data` bytes are exactly the spec's example bytes. + * + * These cover the fixed-width / variable-length / composite families, where a + * faithful byte example fully exercises the encoding. The versioned/stateful + * types (LowCardinality, Variant, Dynamic, JSON) get dedicated structural + * tests in native-spec-versioned.test.ts after a semantic spec comparison. + */ +import { describe, expect, it } from 'vitest'; +import { NativeDecoder } from './native-decoder'; +import { AstNode } from '../types/ast'; +import { analyzeByteRange } from './test-helpers'; + +function encodeLeb128(value: number | bigint): number[] { + let current = BigInt(value); + const bytes: number[] = []; + do { + let byte = Number(current & 0x7fn); + current >>= 7n; + if (current !== 0n) byte |= 0x80; + bytes.push(byte); + } while (current !== 0n); + return bytes; +} + +function encodeString(value: string): number[] { + const bytes = Array.from(new TextEncoder().encode(value)); + return [...encodeLeb128(bytes.length), ...bytes]; +} + +function u32(v: number): number[] { + return [v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >> 24) & 0xff]; +} +function u64(v: number | bigint): number[] { + const out: number[] = []; + let cur = BigInt(v); + for (let i = 0; i < 8; i++) { + out.push(Number(cur & 0xffn)); + cur >>= 8n; + } + return out; +} + +/** Build a protocol-0 single-column block and return the decoded column values. */ +function decodeColumn(typeString: string, data: number[], numRows: number): AstNode[] { + const bytes = new Uint8Array([ + ...encodeLeb128(1), // numColumns + ...encodeLeb128(numRows), // numRows + ...encodeString('c'), + ...encodeString(typeString), + ...data, + ]); + const parsed = new NativeDecoder(bytes, 0).decode(); + const column = parsed.blocks?.[0]?.columns[0]; + if (!column) throw new Error('no column decoded'); + // Coverage sanity: every byte should be claimed by some leaf node. + const coverage = analyzeByteRange(parsed, bytes.length); + if (!coverage.isComplete) { + throw new Error(`incomplete byte coverage for ${typeString}: ${JSON.stringify(coverage.uncoveredRanges)}`); + } + return column.values; +} + +function values(typeString: string, data: number[], numRows: number): unknown[] { + return decodeColumn(typeString, data, numRows).map((n) => n.value); +} + +describe('Native spec — fixed-width types', () => { + it('UInt32 [1, 256, 65536]', () => { + expect(values('UInt32', [0x01, 0, 0, 0, 0x00, 0x01, 0, 0, 0x00, 0x00, 0x01, 0], 3)).toEqual([1, 256, 65536]); + }); + + it('Int32 [-1, 42]', () => { + expect(values('Int32', [0xff, 0xff, 0xff, 0xff, 0x2a, 0, 0, 0], 2)).toEqual([-1, 42]); + }); + + it('UInt64 / Int64 round-trip', () => { + expect(values('UInt64', [0x01, 0, 0, 0, 0, 0, 0, 0], 1)).toEqual([1n]); + expect(values('Int64', [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], 1)).toEqual([-1n]); + }); + + it('Float32 1.5', () => { + expect(values('Float32', [0x00, 0x00, 0xc0, 0x3f], 1)).toEqual([1.5]); + }); + + it('Float64 1.5', () => { + expect(values('Float64', [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x3f], 1)).toEqual([1.5]); + }); + + it('Bool [true, false, true]', () => { + expect(values('Bool', [0x01, 0x00, 0x01], 3)).toEqual([true, false, true]); + }); + + it('Date 1970-01-02 (1 day)', () => { + const node = decodeColumn('Date', [0x01, 0x00], 1)[0]; + expect(node.metadata?.daysSinceEpoch).toBe(1); + expect(node.displayValue).toBe('1970-01-02'); + }); + + it('Date32 1900-01-01 (-25567 days)', () => { + const node = decodeColumn('Date32', [0x21, 0x9c, 0xff, 0xff], 1)[0]; + expect(node.metadata?.daysSinceEpoch).toBe(-25567); + expect(node.displayValue).toBe('1900-01-01'); + }); + + it('DateTime — UInt32 LE seconds since epoch', () => { + // NOTE: docs/full_native_spec.md's DateTime example is internally + // inconsistent — the bytes A8 84 F4 65 decode to 1710523560, not the + // 1710513000 the prose claims (which would be 68 5B F4 65). The decoder + // faithfully decodes the bytes; we assert the byte-accurate value. + const node = decodeColumn("DateTime('UTC')", [0xa8, 0x84, 0xf4, 0x65], 1)[0]; + expect(node.metadata?.secondsSinceEpoch).toBe(1710523560); + }); + + it('DateTime64(3, UTC) 1705321845123 ms', () => { + const node = decodeColumn("DateTime64(3, 'UTC')", [0x83, 0x51, 0x1a, 0x0d, 0x8d, 0x01, 0x00, 0x00], 1)[0]; + expect(node.metadata?.ticksSinceEpoch).toBe('1705321845123'); + }); + + it('DateTime64(0) 1705321845 s', () => { + const node = decodeColumn('DateTime64(0)', [0x75, 0x25, 0xa5, 0x65, 0x00, 0x00, 0x00, 0x00], 1)[0]; + expect(node.metadata?.ticksSinceEpoch).toBe('1705321845'); + }); + + it('UUID 550e8400-e29b-41d4-a716-446655440000', () => { + const wire = [0xd4, 0x41, 0x9b, 0xe2, 0x00, 0x84, 0x0e, 0x55, 0x00, 0x00, 0x44, 0x55, 0x66, 0x44, 0x16, 0xa7]; + expect(values('UUID', wire, 1)).toEqual(['550e8400-e29b-41d4-a716-446655440000']); + }); + + it('IPv4 192.168.1.10', () => { + expect(values('IPv4', [0x0a, 0x01, 0xa8, 0xc0], 1)).toEqual(['192.168.1.10']); + }); + + it('IPv6 2001:db8::1', () => { + const wire = [0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01]; + const node = decodeColumn('IPv6', wire, 1)[0]; + // Spec canonical form is 2001:db8::1 (with :: zero compression). + expect(node.displayValue).toBe('2001:db8::1'); + }); + + it('Enum8 [active, inactive, active]', () => { + const nodes = decodeColumn("Enum8('active' = 1, 'inactive' = 2)", [0x01, 0x02, 0x01], 3); + expect(nodes.map((n) => n.value)).toEqual([1, 2, 1]); + expect(nodes.map((n) => n.metadata?.enumName)).toEqual(['active', 'inactive', 'active']); + }); + + it('Enum16 30000', () => { + const node = decodeColumn("Enum16('a' = 1, 'b' = 30000)", [0x30, 0x75], 1)[0]; + expect(node.value).toBe(30000); + expect(node.metadata?.enumName).toBe('b'); + }); + + it('Decimal(9, 4) 123.4567 -> 1234567', () => { + const node = decodeColumn('Decimal(9, 4)', [0x87, 0xd6, 0x12, 0x00], 1)[0]; + expect(node.metadata?.rawValue).toBe(1234567); + expect(node.displayValue).toBe('123.4567'); + }); + + it('Decimal(18, 1) -1.5 -> -15', () => { + const node = decodeColumn('Decimal(18, 1)', [0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], 1)[0]; + expect(node.metadata?.rawValue).toBe('-15'); + expect(node.displayValue).toBe('-1.5'); + }); + + it('Decimal(38, 4) 123.4567 (16 bytes)', () => { + const wire = [0x87, 0xd6, 0x12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const node = decodeColumn('Decimal(38, 4)', wire, 1)[0]; + expect(node.metadata?.rawValue).toBe('1234567'); + expect(node.displayValue).toBe('123.4567'); + }); +}); + +describe('Native spec — variable-length types', () => { + it('String ["ab", "", "c"]', () => { + expect(values('String', [0x02, 0x61, 0x62, 0x00, 0x01, 0x63], 3)).toEqual(['ab', '', 'c']); + }); + + it('FixedString(3) ["abc", "de\\0"]', () => { + const nodes = decodeColumn('FixedString(3)', [0x61, 0x62, 0x63, 0x64, 0x65, 0x00], 2); + expect(nodes.map((n) => n.value)).toEqual(['abc', 'de']); + }); +}); + +describe('Native spec — composite types', () => { + it('Nullable(UInt8) [5, NULL, 9]', () => { + expect(values('Nullable(UInt8)', [0x00, 0x01, 0x00, 0x05, 0x00, 0x09], 3)).toEqual([5, null, 9]); + }); + + it('Nullable(String) ["hello", NULL, "world"]', () => { + const data = [ + 0x00, 0x01, 0x00, + 0x05, 0x68, 0x65, 0x6c, 0x6c, 0x6f, + 0x00, + 0x05, 0x77, 0x6f, 0x72, 0x6c, 0x64, + ]; + expect(values('Nullable(String)', data, 3)).toEqual(['hello', null, 'world']); + }); + + it('Array(UInt32) [[10,20,30], [], [40,50]]', () => { + const data = [ + ...u64(3), ...u64(3), ...u64(5), + ...u32(10), ...u32(20), ...u32(30), ...u32(40), ...u32(50), + ]; + const nodes = decodeColumn('Array(UInt32)', data, 3); + expect(nodes.map((n) => n.value)).toEqual([[10, 20, 30], [], [40, 50]]); + }); + + it('Array(String) [["a","bb"], []]', () => { + const data = [...u64(2), ...u64(2), 0x01, 0x61, 0x02, 0x62, 0x62]; + const nodes = decodeColumn('Array(String)', data, 2); + expect(nodes.map((n) => n.value)).toEqual([['a', 'bb'], []]); + }); + + it('Tuple(UInt8, UInt8) columnar layout', () => { + const nodes = decodeColumn('Tuple(UInt8, UInt8)', [0x01, 0x02, 0x03, 0x04, 0x05, 0x06], 3); + expect(nodes.map((n) => n.value)).toEqual([[1, 4], [2, 5], [3, 6]]); + }); + + it('Tuple(UInt32, String)', () => { + const data = [...u32(10), ...u32(20), 0x01, 0x61, 0x02, 0x62, 0x62]; + const nodes = decodeColumn('Tuple(UInt32, String)', data, 2); + expect(nodes.map((n) => n.value)).toEqual([[10, 'a'], [20, 'bb']]); + }); + + it('Map(UInt8, UInt8) {1:10,2:20}, {3:30}', () => { + const data = [...u64(2), ...u64(3), 0x01, 0x02, 0x03, 0x0a, 0x14, 0x1e]; + const nodes = decodeColumn('Map(UInt8, UInt8)', data, 2); + expect(nodes.map((n) => n.value)).toEqual([{ '1': 10, '2': 20 }, { '3': 30 }]); + }); + + it('Map(String, UInt32) {a:1, b:2}', () => { + const data = [...u64(2), 0x01, 0x61, 0x01, 0x62, ...u32(1), ...u32(2)]; + const nodes = decodeColumn('Map(String, UInt32)', data, 1); + expect(nodes.map((n) => n.value)).toEqual([{ a: 1, b: 2 }]); + }); + + it('Nested(a UInt8, b String) [[(10,x),(20,y)], [(30,z)]]', () => { + const data = [ + ...u64(2), ...u64(3), + 0x0a, 0x14, 0x1e, + 0x01, 0x78, 0x01, 0x79, 0x01, 0x7a, + ]; + const nodes = decodeColumn('Nested(a UInt8, b String)', data, 2); + // Nested is byte-identical to Array(Tuple(a UInt8, b String)). + expect(nodes.map((n) => n.value)).toEqual([ + [[10, 'x'], [20, 'y']], + [[30, 'z']], + ]); + }); + + it('Nullable(Nothing) — SELECT NULL, 3 rows all NULL', () => { + const data = [0x01, 0x01, 0x01, 0x30, 0x30, 0x30]; + expect(values('Nullable(Nothing)', data, 3)).toEqual([null, null, null]); + }); +}); + +/** + * Versioned/stateful types. The byte fixtures below were captured from + * `clickhouse-local ... FORMAT Native` (protocol-0 output, no has_custom byte) + * so they are ground truth, not hand-derived. + */ +describe('Native spec — versioned types', () => { + it('LowCardinality(String) [a, b, a, c, b]', () => { + const data = [ + ...u64(1), // state prefix + ...u64(0x600), // metadata (HasAdditionalKeys + NeedUpdateDictionary) + ...u64(4), // dict_size + 0x00, // dict[0] = "" placeholder + 0x01, 0x61, // dict[1] = "a" + 0x01, 0x62, // dict[2] = "b" + 0x01, 0x63, // dict[3] = "c" + ...u64(5), // keys_count + 0x01, 0x02, 0x01, 0x03, 0x02, + ]; + expect(values('LowCardinality(String)', data, 5)).toEqual(['a', 'b', 'a', 'c', 'b']); + }); + + it('LowCardinality(Nullable(String)) [a, NULL, b] — NULL is dict index 0', () => { + // Real clickhouse-local bytes. The spec claims dict[1] is the null marker, + // but ClickHouse actually reserves index 0 for NULL and index 1 for the + // default/empty placeholder (real values start at index 2). dict = ["","","a","b"], + // keys = [2, 0, 3]. + const data = [ + ...u64(1), // state prefix + ...u64(0x600), // metadata + ...u64(4), // dict_size + 0x00, // dict[0] = "" (NULL slot) + 0x00, // dict[1] = "" (default placeholder) + 0x01, 0x61, // dict[2] = "a" + 0x01, 0x62, // dict[3] = "b" + ...u64(3), // keys_count + 0x02, 0x00, 0x03, // keys -> a, NULL, b + ]; + expect(values('LowCardinality(Nullable(String))', data, 3)).toEqual(['a', null, 'b']); + }); + + it('LowCardinality(Nullable(String)) [a, NULL, "", b] — NULL (idx 0) vs empty string (idx 1)', () => { + // Real clickhouse-local bytes. NULL and a genuine empty string both serialize + // as "" in the dictionary, but reference different reserved slots: NULL -> key 0, + // empty-string default -> key 1. dict = ["","","a","b"], keys = [2, 0, 1, 3]. + const data = [ + ...u64(1), // state prefix + ...u64(0x600), // metadata + ...u64(4), // dict_size + 0x00, // dict[0] = "" (NULL slot) + 0x00, // dict[1] = "" (default/empty placeholder) + 0x01, 0x61, // dict[2] = "a" + 0x01, 0x62, // dict[3] = "b" + ...u64(4), // keys_count + 0x02, 0x00, 0x01, 0x03, // keys -> a, NULL, "", b + ]; + const nodes = decodeColumn('LowCardinality(Nullable(String))', data, 4); + expect(nodes.map((n) => n.value)).toEqual(['a', null, '', 'b']); + // The empty-string row must be a real value, not NULL. + expect(nodes[1].value).toBeNull(); + expect(nodes[2].value).toBe(''); + }); + + it('Variant(String, UInt64) [42, "hi", NULL] (BASIC mode)', () => { + const data = [ + ...u64(0), // BASIC mode + 0x01, 0x00, 0xff, // discriminators: UInt64, String, NULL + 0x02, 0x68, 0x69, // String run "hi" + ...u64(42), // UInt64 run + ]; + expect(values('Variant(String, UInt64)', data, 3)).toEqual([42n, 'hi', null]); + }); + + it('Dynamic default (internal-variant V1) [42::UInt64, "hi", NULL]', () => { + const data = [ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // version = 1 + 0x02, // max_dynamic_types + 0x02, // num_dynamic_types + 0x06, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, // "String" + 0x06, 0x55, 0x49, 0x6e, 0x74, 0x36, 0x34, // "UInt64" + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // variant mode = 0 + 0x02, 0x01, 0xff, // discriminators (sorted: SharedVariant,String,UInt64): UInt64, String, NULL + 0x02, 0x68, 0x69, // String run "hi" + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // UInt64 run 42 + ]; + // The Dynamic header node is values[0]; row values follow. + const nodes = decodeColumn('Dynamic', data, 3); + expect(nodes.slice(1).map((n) => n.value)).toEqual([42n, 'hi', null]); + }); + + it('Dynamic FLATTENED (v3) [42::UInt64, "hi", NULL]', () => { + const data = [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // version = 3 + 0x02, // num_types + 0x06, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, // "String" + 0x06, 0x55, 0x49, 0x6e, 0x74, 0x36, 0x34, // "UInt64" + 0x01, 0x00, 0x02, // discriminators (wire order): UInt64(1), String(0), NULL(2) + 0x02, 0x68, 0x69, // String run "hi" + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // UInt64 run 42 + ]; + const nodes = decodeColumn('Dynamic', data, 3); + expect(nodes.slice(1).map((n) => n.value)).toEqual([42n, 'hi', null]); + }); +}); + +describe('Native spec — JSON', () => { + it('JSON default (Object v0) {"a":1,"b":"hi"}', () => { + const data = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // version 0 + 0x02, // max_dynamic_paths + 0x02, // num_dynamic_paths + 0x01, 0x61, 0x01, 0x62, // "a", "b" + // "a" Dynamic structure (v1) + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x05, 0x49, 0x6e, 0x74, 0x36, 0x34, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // "b" Dynamic structure (v1) + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x06, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // "a" data: disc 0, Int64 = 1 + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // "b" data: disc 0, String "hi" + 0x01, 0x02, 0x68, 0x69, + // shared data offset + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + const nodes = decodeColumn('JSON', data, 1); + expect(nodes[0].value).toEqual({ a: 1n, b: 'hi' }); + }); + + it('JSON Tier-1 String fallback (version 1)', () => { + const data = [ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // version 1 + 0x07, 0x7b, 0x22, 0x61, 0x22, 0x3a, 0x31, 0x7d, // {"a":1} + 0x0a, 0x7b, 0x22, 0x78, 0x22, 0x3a, 0x74, 0x72, 0x75, 0x65, 0x7d, // {"x":true} + ]; + const nodes = decodeColumn('JSON', data, 2); + expect(nodes.map((n) => n.value)).toEqual([{ a: 1 }, { x: true }]); + expect(nodes.map((n) => n.metadata?.jsonText)).toEqual(['{"a":1}', '{"x":true}']); + }); + + it('JSON FLATTENED (v3) {"a":1,"b":"hi"} — both dynamic', () => { + const data = [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // version 3 + 0x02, // num_dynamic_paths + 0x01, 0x61, 0x01, 0x62, // "a", "b" + // "a" Dynamic prefix (v3, [Int64]) + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x05, 0x49, 0x6e, 0x74, 0x36, 0x34, + // "b" Dynamic prefix (v3, [String]) + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x06, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, + // "a" data: disc 0, Int64 = 1 + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // "b" data: disc 0, String "hi" + 0x00, 0x02, 0x68, 0x69, + ]; + const nodes = decodeColumn('JSON', data, 1); + expect(nodes[0].value).toEqual({ a: 1n, b: 'hi' }); + }); + + it('JSON FLATTENED (v3) with typed path JSON(a UInt32)', () => { + const data = [ + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // version 3 + 0x01, // num_dynamic_paths + 0x01, 0x62, // dynamic path "b" + // "b" Dynamic prefix (v3, [String]) + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x06, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, + // typed path "a" data: UInt32 = 7 + 0x07, 0x00, 0x00, 0x00, + // "b" data: disc 0, String "hi" + 0x00, 0x02, 0x68, 0x69, + ]; + const nodes = decodeColumn('JSON(a UInt32)', data, 1); + expect(nodes[0].value).toEqual({ a: 7, b: 'hi' }); + }); +}); diff --git a/src/core/decoder/protocol-decoder.test.ts b/src/core/decoder/protocol-decoder.test.ts new file mode 100644 index 0000000..1da49fa --- /dev/null +++ b/src/core/decoder/protocol-decoder.test.ts @@ -0,0 +1,192 @@ +/** + * Regression tests for the native TCP protocol decoder, run against real + * packet captures (scripts/native-proxy.mjs driving clickhouse-client through a + * proxy). The captures live in fixtures/protocol/*.chproto and are decoded with + * no live ClickHouse needed. + * + * The core guarantee is 100% byte coverage: every byte of both the + * client→server and server→client streams must be attributed to a labeled AST + * node, and no Protocol.DecodeError node may appear. That is the forcing + * function that proves the positional decode stayed aligned across every + * packet and version-gated field. + */ +import { readFileSync, readdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ProtocolDecoder } from './protocol-decoder'; +import { parseChprotoDump } from './protocol-dump'; +import { analyzeByteRange, formatUncoveredRanges } from './test-helpers'; +import { AstNode, ParsedData } from '../types/ast'; + +const FIXTURE_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), 'fixtures', 'protocol'); + +function loadCapture(name: string) { + return parseChprotoDump(readFileSync(path.join(FIXTURE_DIR, name))); +} + +function decodeFixture(name: string): { parsed: ParsedData; combinedLength: number } { + const cap = loadCapture(name); + const decoder = ProtocolDecoder.fromCapture(cap); + const parsed = decoder.decode(); + return { parsed, combinedLength: cap.c2s.length + cap.s2c.length }; +} + +function walk(node: AstNode, visit: (n: AstNode) => void): void { + visit(node); + node.children?.forEach((c) => walk(c, visit)); +} + +function allNodes(parsed: ParsedData): AstNode[] { + const out: AstNode[] = []; + parsed.trailingNodes?.forEach((n) => walk(n, (x) => out.push(x))); + return out; +} + +function packetTypes(section: AstNode): string[] { + return (section.children ?? []).map((p) => p.type.replace('Protocol.', '')); +} + +const FIXTURES = readdirSync(FIXTURE_DIR).filter((f) => f.endsWith('.chproto')).sort(); + +describe('ProtocolDecoder — fixtures', () => { + it('discovers fixtures', () => { + expect(FIXTURES.length).toBeGreaterThan(0); + }); + + describe.each(FIXTURES)('%s', (name) => { + it('decodes with no error nodes', () => { + const { parsed } = decodeFixture(name); + const errors = allNodes(parsed).filter((n) => n.type === 'Protocol.DecodeError'); + expect(errors.map((e) => e.displayValue)).toEqual([]); + }); + + it('covers 100% of the captured bytes', () => { + const { parsed, combinedLength } = decodeFixture(name); + const coverage = analyzeByteRange(parsed, combinedLength); + if (!coverage.isComplete) { + const cap = loadCapture(name); + const combined = new Uint8Array(combinedLength); + combined.set(cap.c2s, 0); + combined.set(cap.s2c, cap.c2s.length); + throw new Error(`${name}\n${formatUncoveredRanges(coverage, combined)}`); + } + expect(coverage.isComplete).toBe(true); + expect(coverage.coveragePercent).toBe(100); + }); + + it('has exactly two direction sections', () => { + const { parsed } = decodeFixture(name); + expect(parsed.trailingNodes).toHaveLength(2); + expect(parsed.trailingNodes![0].type).toBe('Protocol.ClientStream'); + expect(parsed.trailingNodes![1].type).toBe('Protocol.ServerStream'); + }); + + it('starts each direction with a Hello', () => { + const { parsed } = decodeFixture(name); + const [client, server] = parsed.trailingNodes!; + expect(packetTypes(client)[0]).toBe('ClientHello'); + // Server may answer with ServerHello (or Exception only on auth failure). + expect(['ServerHello', 'Exception']).toContain(packetTypes(server)[0]); + }); + }); +}); + +describe('ProtocolDecoder — structural expectations', () => { + it('simple select: ClientHello, Addendum, Query, empty Data marker; server Data + EndOfStream', () => { + const { parsed } = decodeFixture('01-simple-select.chproto'); + const [client, server] = parsed.trailingNodes!; + const c = packetTypes(client); + expect(c).toContain('ClientHello'); + expect(c).toContain('Addendum'); + expect(c).toContain('Query'); + expect(c).toContain('Data'); // empty end-of-client-data marker + const s = packetTypes(server); + expect(s).toContain('ServerHello'); + expect(s).toContain('Data'); + expect(s[s.length - 1]).toBe('EndOfStream'); + }); + + it('negotiated version is recorded and below the server version', () => { + const { parsed } = decodeFixture('01-simple-select.chproto'); + const neg = parsed.metadata?.negotiatedVersion as number; + expect(neg).toBeGreaterThanOrEqual(54479); // cluster-function feature present + }); + + it('Query packet carries query_id, ClientInfo, settings, stage, compression, query_body', () => { + const { parsed } = decodeFixture('01-simple-select.chproto'); + const client = parsed.trailingNodes![0]; + const query = client.children!.find((p) => p.type === 'Protocol.Query')!; + const labels = query.children!.map((c) => c.label); + expect(labels).toEqual( + expect.arrayContaining(['query_id', 'settings', 'stage', 'compression', 'query_body']), + ); + const clientInfo = query.children!.find((c) => c.type === 'Protocol.ClientInfo'); + expect(clientInfo).toBeDefined(); + const qb = query.children!.find((c) => c.label === 'query_body')!; + expect(String(qb.value)).toContain('SELECT'); + }); + + it('exception fixture surfaces a server Exception with code and message', () => { + const { parsed } = decodeFixture('04-exception.chproto'); + const server = parsed.trailingNodes![1]; + const exc = server.children!.find((p) => p.type === 'Protocol.Exception'); + expect(exc).toBeDefined(); + const exBody = exc!.children!.find((c) => c.label === 'exception')!; + const code = exBody.children!.find((c) => c.label === 'code')!; + expect(Number(code.value)).toBeGreaterThan(0); + const msg = exBody.children!.find((c) => c.label === 'message')!; + expect(String(msg.value).length).toBeGreaterThan(0); + }); + + it('totals/extremes fixture yields Totals and Extremes packets', () => { + const { parsed } = decodeFixture('03-totals-extremes.chproto'); + const s = packetTypes(parsed.trailingNodes![1]); + expect(s).toContain('Totals'); + expect(s).toContain('Extremes'); + }); + + it('multiblock fixture yields several server Data packets', () => { + const { parsed } = decodeFixture('05-multiblock.chproto'); + const dataCount = packetTypes(parsed.trailingNodes![1]).filter((t) => t === 'Data').length; + expect(dataCount).toBeGreaterThan(2); + }); + + it('logs fixture yields Log and ProfileEvents packets', () => { + const { parsed } = decodeFixture('06-logs.chproto'); + const s = packetTypes(parsed.trailingNodes![1]); + expect(s).toContain('Log'); + expect(s).toContain('ProfileEvents'); + }); + + it('insert fixture: client sends a Data block with rows; server sends a 0-row schema block', () => { + const { parsed } = decodeFixture('07-insert.chproto'); + const [client, server] = parsed.trailingNodes!; + // A client Data packet carrying the inserted rows. + const clientBlocks = (client.children ?? []) + .filter((p) => p.type === 'Protocol.Data') + .map((p) => p.children!.find((c) => c.type === 'Native.Block')!) + .filter(Boolean); + const withRows = clientBlocks.filter((b) => (b.value as { rows: number }).rows > 0); + expect(withRows.length).toBeGreaterThan(0); + // The server's schema header block: columns present, 0 rows. + const serverBlocks = (server.children ?? []) + .filter((p) => p.type === 'Protocol.Data') + .map((p) => p.children!.find((c) => c.type === 'Native.Block')!) + .filter(Boolean); + const schema = serverBlocks.find( + (b) => (b.value as { rows: number; columns: number }).rows === 0 + && (b.value as { columns: number }).columns > 0, + ); + expect(schema).toBeDefined(); + }); + + it('parameters fixture: Query carries a parameters list with entries', () => { + const { parsed } = decodeFixture('08-parameters.chproto'); + const query = parsed.trailingNodes![0].children!.find((p) => p.type === 'Protocol.Query')!; + const params = query.children!.find((c) => c.label === 'parameters'); + expect(params).toBeDefined(); + const entries = (params!.children ?? []).filter((c) => c.label !== 'terminator'); + expect(entries.length).toBeGreaterThanOrEqual(2); + }); +}); diff --git a/src/core/decoder/protocol-decoder.ts b/src/core/decoder/protocol-decoder.ts new file mode 100644 index 0000000..83848d5 --- /dev/null +++ b/src/core/decoder/protocol-decoder.ts @@ -0,0 +1,780 @@ +import { BinaryReader } from './reader'; +import { decodeLEB128 } from './leb128'; +import { NativeDecoder } from './native-decoder'; +import { AstNode, BlockNode, ByteRange, HeaderNode, ParsedData } from '../types/ast'; +import { ClickHouseFormat } from '../types/formats'; + +const TEXT_DECODER = new TextDecoder(); + +/** + * Protocol feature gates, keyed by the protocol version that introduced them. + * A feature is active when the negotiated version is >= its value. See + * docs/full_native_protocol_spec.md (the feature table). + */ +const F = { + BLOCK_INFO: 51903, + TIMEZONE: 54058, + QUOTA_KEY_IN_CLIENT_INFO: 54060, + DISPLAY_NAME: 54372, + VERSION_PATCH: 54401, + WRITE_CLIENT_INFO: 54420, + SETTINGS_AS_STRINGS: 54429, + INTERSERVER_SECRET: 54441, + OPEN_TELEMETRY: 54442, + DISTRIBUTED_DEPTH: 54448, + INITIAL_QUERY_START_TIME: 54449, + PARALLEL_REPLICAS: 54453, + ADDENDUM: 54458, + PARAMETERS: 54459, + SERVER_QUERY_TIME_IN_PROGRESS: 54460, + PASSWORD_COMPLEXITY_RULES: 54461, + INTERSERVER_SECRET_V2: 54462, + TOTAL_BYTES_IN_PROGRESS: 54463, + TIMEZONE_UPDATES: 54464, + ROWS_BEFORE_AGGREGATION: 54469, + CHUNKED_PROTOCOL: 54470, + VERSIONED_PARALLEL_REPLICAS: 54471, + INTERSERVER_EXTERNALLY_GRANTED_ROLES: 54472, + SERVER_SETTINGS: 54474, + QUERY_AND_LINE_NUMBERS: 54475, + JWT_IN_INTERSERVER: 54476, + QUERY_PLAN_SERIALIZATION: 54477, + VERSIONED_CLUSTER_FUNCTION: 54479, +} as const; + +/** Client → Server packet type codes. */ +const ClientPacket = { + Hello: 0, + Query: 1, + Data: 2, + Cancel: 3, + Ping: 4, + SSHChallengeRequest: 11, + SSHChallengeResponse: 12, +} as const; + +/** Server → Client packet type codes. */ +const ServerPacket = { + Hello: 0, + Data: 1, + Exception: 2, + Progress: 3, + Pong: 4, + EndOfStream: 5, + ProfileInfo: 6, + Totals: 7, + Extremes: 8, + Log: 10, + TableColumns: 11, + ProfileEvents: 14, + TimezoneUpdate: 17, + SSHChallenge: 18, +} as const; + +/** + * Server packet codes that can legitimately appear as the *first* packet after + * ServerHello. Used to absorb undocumented trailing version VarUInts that some + * server builds append to ServerHello (see decodeServerHello): a trailing + * VarUInt whose value is not in this set cannot be a packet start, so it is a + * hello field. + */ +const VALID_FIRST_SERVER_PACKET = new Set([ + ServerPacket.Data, + ServerPacket.Exception, + ServerPacket.Progress, + ServerPacket.EndOfStream, + ServerPacket.ProfileInfo, + ServerPacket.Totals, + ServerPacket.Extremes, + ServerPacket.Log, + ServerPacket.TableColumns, + ServerPacket.ProfileEvents, + ServerPacket.TimezoneUpdate, +]); + +export interface ProtocolCapture { + /** Concatenated client → server byte stream. */ + c2s: Uint8Array; + /** Concatenated server → client byte stream. */ + s2c: Uint8Array; + meta?: Record; +} + +/** + * Decoder for the ClickHouse native TCP protocol. It consumes a capture of one + * connection's two per-direction byte streams (as produced by the proxy + * harness), concatenates them into a single buffer for the hex viewer, and + * produces an AstNode tree: two top-level "stream" nodes (client→server, + * server→client) each containing one node per packet, with packet fields and — + * for Data-family packets — the full Native Block subtree (reused from + * NativeDecoder) nested underneath. + * + * Compression and TLS are out of scope: captures are expected to be plaintext, + * uncompressed (localhost clickhouse-client disables compression by default). + */ +export class ProtocolDecoder { + readonly format = ClickHouseFormat.NativeProtocol; + private readonly combined: Uint8Array; + private readonly c2sLength: number; + private readonly total: number; + private readonly meta?: Record; + private readonly negotiated: number; + private readonly native: NativeDecoder; + private readonly r: BinaryReader; + private idCounter = 0; + private blockIndex = 0; + + /** + * @param combined the concatenated [c2s][s2c] byte buffer (rawData for the hex viewer) + * @param c2sLength byte length of the client→server portion (the split point) + */ + constructor(combined: Uint8Array, c2sLength: number, meta?: Record) { + this.combined = combined; + this.c2sLength = c2sLength; + this.total = combined.length; + this.meta = meta; + this.negotiated = this.computeNegotiatedVersion(); + // NativeDecoder reads `combined`; we drive its reader for all framing too, + // so packet-framing reads and block decoding share one offset cursor. + this.native = new NativeDecoder(combined, this.negotiated); + this.r = this.native.sharedReader; + } + + /** Build a ProtocolDecoder from a capture object (separate c2s / s2c). */ + static fromCapture(capture: ProtocolCapture): ProtocolDecoder { + const combined = new Uint8Array(capture.c2s.length + capture.s2c.length); + combined.set(capture.c2s, 0); + combined.set(capture.s2c, capture.c2s.length); + return new ProtocolDecoder(combined, capture.c2s.length, capture.meta); + } + + decode(): ParsedData { + const clientPackets = this.decodeClientStream(); + const serverPackets = this.decodeServerStream(); + + const clientSection: AstNode = { + id: this.nid(), + type: 'Protocol.ClientStream', + byteRange: { start: 0, end: this.c2sLength }, + value: clientPackets.length, + displayValue: `client → server · ${clientPackets.length} packet(s) · ${this.c2sLength}B`, + label: 'client → server', + children: clientPackets, + }; + const serverSection: AstNode = { + id: this.nid(), + type: 'Protocol.ServerStream', + byteRange: { start: this.c2sLength, end: this.total }, + value: serverPackets.length, + displayValue: `server → client · ${serverPackets.length} packet(s) · ${this.total - this.c2sLength}B`, + label: 'server → client', + children: serverPackets, + }; + + return { + format: this.format, + header: this.emptyHeader(), + totalBytes: this.total, + trailingNodes: [clientSection, serverSection], + metadata: { negotiatedVersion: this.negotiated, ...this.meta }, + }; + } + + // --- stream loops ------------------------------------------------------- + + private decodeClientStream(): AstNode[] { + const packets: AstNode[] = []; + if (this.c2sLength === 0) return packets; + + // 1. ClientHello is always first. + packets.push(this.guard(() => this.decodeClientHello(), 'client', packets)); + // 2. Addendum (no packet type byte), gated by the negotiated version. + if (this.negotiated >= F.ADDENDUM && this.r.offset < this.c2sLength) { + packets.push(this.guard(() => this.decodeAddendum(), 'client', packets)); + } + // 3. Remaining client packets (Query, Data, Ping, Cancel, ...). + while (this.r.offset < this.c2sLength) { + const before = this.r.offset; + packets.push(this.guard(() => this.decodeClientPacket(), 'client', packets)); + if (this.r.offset <= before) break; // no progress: stop to avoid a loop + } + return packets; + } + + private decodeServerStream(): AstNode[] { + const packets: AstNode[] = []; + if (this.total - this.c2sLength === 0) return packets; + while (this.r.offset < this.total) { + const before = this.r.offset; + packets.push(this.guard(() => this.decodeServerPacket(), 'server', packets)); + if (this.r.offset <= before) break; + } + return packets; + } + + /** + * Run one packet decode; on failure, emit an error node spanning the rest of + * the current direction and stop that stream. Keeps the UI usable on a + * partially-understood capture while letting tests assert zero error nodes. + */ + private guard(fn: () => AstNode, dir: 'client' | 'server', _packets: AstNode[]): AstNode { + const start = this.r.offset; + try { + return fn(); + } catch (err) { + const end = dir === 'client' ? this.c2sLength : this.total; + // Consume the remainder so the stream loop terminates. + this.r.skip(Math.max(0, end - this.r.offset)); + return { + id: this.nid(), + type: 'Protocol.DecodeError', + byteRange: { start, end }, + value: String(err instanceof Error ? err.message : err), + displayValue: `decode error: ${err instanceof Error ? err.message : err}`, + label: 'error', + }; + } + } + + // --- client packets ----------------------------------------------------- + + private decodeClientHello(): AstNode { + const start = this.r.offset; + const children: AstNode[] = []; + children.push(this.typeNode('ClientHello', ClientPacket.Hello)); + children.push(this.str('client_name').node); + children.push(this.vu('version_major').node); + children.push(this.vu('version_minor').node); + children.push(this.vu('protocol_version').node); + children.push(this.str('database').node); + children.push(this.str('user').node); + children.push(this.str('password').node); + return this.packet('ClientHello', start, children); + } + + private decodeAddendum(): AstNode { + const start = this.r.offset; + const children: AstNode[] = []; + // The Addendum has no packet type byte — fields go raw on the wire. + children.push(this.str('quota_key').node); + if (this.negotiated >= F.CHUNKED_PROTOCOL) { + children.push(this.str('proto_send_chunked').node); + children.push(this.str('proto_recv_chunked').node); + } + if (this.negotiated >= F.VERSIONED_PARALLEL_REPLICAS) { + children.push(this.vu('parallel_replicas_protocol_version').node); + } + if (this.negotiated >= F.VERSIONED_CLUSTER_FUNCTION) { + children.push(this.vu('cluster_function_protocol_version').node); + } + return this.packet('Addendum', start, children); + } + + private decodeClientPacket(): AstNode { + const start = this.r.offset; + const { value: type } = this.peekTypeOrThrow(); + switch (type) { + case ClientPacket.Query: + return this.decodeQuery(start); + case ClientPacket.Data: + return this.decodeDataPacket('Data', ClientPacket.Data, start); + case ClientPacket.Cancel: + return this.bodylessPacket('Cancel', ClientPacket.Cancel, start); + case ClientPacket.Ping: + return this.bodylessPacket('Ping', ClientPacket.Ping, start); + case ClientPacket.SSHChallengeRequest: + return this.bodylessPacket('SSHChallengeRequest', ClientPacket.SSHChallengeRequest, start); + case ClientPacket.SSHChallengeResponse: { + const children = [this.typeNode('SSHChallengeResponse', type), this.str('signature').node]; + return this.packet('SSHChallengeResponse', start, children); + } + default: + throw new Error(`unsupported client packet type ${type} at offset ${start}`); + } + } + + private decodeQuery(start: number): AstNode { + const children: AstNode[] = []; + children.push(this.typeNode('Query', ClientPacket.Query)); + children.push(this.str('query_id').node); + if (this.negotiated >= F.WRITE_CLIENT_INFO) { + children.push(this.decodeClientInfo()); + } + if (this.negotiated >= F.SETTINGS_AS_STRINGS) { + children.push(this.decodeSettingsList('settings')); + } else { + throw new Error( + `Query settings below v${F.SETTINGS_AS_STRINGS} (binary settings) are not supported; negotiated ${this.negotiated}`, + ); + } + if (this.negotiated >= F.INTERSERVER_EXTERNALLY_GRANTED_ROLES) { + children.push(this.str('external_roles').node); + } + if (this.negotiated >= F.INTERSERVER_SECRET) { + children.push(this.str('cluster_secret').node); + } + children.push(this.vu('stage').node); + children.push(this.vu('compression').node); + children.push(this.str('query_body').node); + if (this.negotiated >= F.PARAMETERS) { + children.push(this.decodeSettingsList('parameters')); + } + return this.packet('Query', start, children); + } + + private decodeClientInfo(): AstNode { + const start = this.r.offset; + const children: AstNode[] = []; + const queryKindRes = this.u8('query_kind'); + children.push(queryKindRes.node); + const queryKind = queryKindRes.value; + children.push(this.str('initial_user').node); + children.push(this.str('initial_query_id').node); + children.push(this.str('initial_address').node); + if (this.negotiated >= F.INITIAL_QUERY_START_TIME) { + children.push(this.i64('initial_time').node); // fixed-width 8 bytes + } + const ifaceRes = this.u8('query_interface'); + children.push(ifaceRes.node); + const iface = ifaceRes.value; + const isTcp = iface === 1; + if (isTcp) { + children.push(this.str('os_user').node); + children.push(this.str('client_hostname').node); + children.push(this.str('client_name').node); + children.push(this.vu('client_version_major').node); + children.push(this.vu('client_version_minor').node); + children.push(this.vu('client_protocol_version').node); + } + if (this.negotiated >= F.QUOTA_KEY_IN_CLIENT_INFO) { + children.push(this.str('quota_key').node); + } + if (this.negotiated >= F.DISTRIBUTED_DEPTH) { + children.push(this.vu('distributed_depth').node); + } + if (this.negotiated >= F.VERSION_PATCH && isTcp) { + children.push(this.vu('client_version_patch').node); + } + if (this.negotiated >= F.OPEN_TELEMETRY) { + children.push(this.decodeOpenTelemetry()); + } + if (this.negotiated >= F.PARALLEL_REPLICAS) { + children.push(this.vu('collaborate_with_initiator').node); + children.push(this.vu('count_participating_replicas').node); + children.push(this.vu('number_of_current_replica').node); + } + if (this.negotiated >= F.QUERY_AND_LINE_NUMBERS) { + children.push(this.vu('script_query_number').node); + children.push(this.vu('script_line_number').node); + } + if (this.negotiated >= F.JWT_IN_INTERSERVER) { + const jwtRes = this.u8('jwt_present'); + children.push(jwtRes.node); + if (jwtRes.value === 1) { + children.push(this.str('jwt').node); + } + } + return this.group('ClientInfo', start, children, `query_kind=${queryKind}, interface=${iface}`); + } + + private decodeOpenTelemetry(): AstNode { + const start = this.r.offset; + const children: AstNode[] = []; + const hasTraceRes = this.u8('has_trace'); + children.push(hasTraceRes.node); + if (hasTraceRes.value === 1) { + children.push(this.fixedBytes(16, 'trace_id', 'UInt128')); + children.push(this.fixedBytes(8, 'span_id', 'UInt64')); + children.push(this.str('trace_state').node); + children.push(this.u8('trace_flags').node); + } + return this.group('OpenTelemetry', start, children, hasTraceRes.value === 1 ? 'trace present' : 'no trace'); + } + + // --- server packets ----------------------------------------------------- + + private decodeServerPacket(): AstNode { + const start = this.r.offset; + const { value: type } = this.peekTypeOrThrow(); + switch (type) { + case ServerPacket.Hello: + return this.decodeServerHello(start); + case ServerPacket.Data: + return this.decodeDataPacket('Data', ServerPacket.Data, start); + case ServerPacket.Exception: + return this.decodeException(start); + case ServerPacket.Progress: + return this.decodeProgress(start); + case ServerPacket.Pong: + return this.bodylessPacket('Pong', ServerPacket.Pong, start); + case ServerPacket.EndOfStream: + return this.bodylessPacket('EndOfStream', ServerPacket.EndOfStream, start); + case ServerPacket.ProfileInfo: + return this.decodeProfileInfo(start); + case ServerPacket.Totals: + return this.decodeDataPacket('Totals', ServerPacket.Totals, start); + case ServerPacket.Extremes: + return this.decodeDataPacket('Extremes', ServerPacket.Extremes, start); + case ServerPacket.Log: + return this.decodeDataPacket('Log', ServerPacket.Log, start); + case ServerPacket.TableColumns: + return this.decodeTableColumns(start); + case ServerPacket.ProfileEvents: + return this.decodeDataPacket('ProfileEvents', ServerPacket.ProfileEvents, start); + case ServerPacket.TimezoneUpdate: { + const children = [this.typeNode('TimezoneUpdate', type), this.str('timezone').node]; + return this.packet('TimezoneUpdate', start, children); + } + case ServerPacket.SSHChallenge: { + const children = [this.typeNode('SSHChallenge', type), this.str('challenge').node]; + return this.packet('SSHChallenge', start, children); + } + default: + throw new Error(`unsupported server packet type ${type} at offset ${start}`); + } + } + + private decodeServerHello(start: number): AstNode { + const children: AstNode[] = []; + children.push(this.typeNode('ServerHello', ServerPacket.Hello)); + children.push(this.str('server_name').node); + children.push(this.vu('version_major').node); + children.push(this.vu('version_minor').node); + children.push(this.vu('protocol_version').node); + if (this.negotiated >= F.VERSIONED_PARALLEL_REPLICAS) { + // Wire position: immediately after protocol_version, before timezone. + children.push(this.vu('parallel_replicas_protocol_version').node); + } + if (this.negotiated >= F.TIMEZONE) { + children.push(this.str('timezone').node); + } + if (this.negotiated >= F.DISPLAY_NAME) { + children.push(this.str('display_name').node); + } + if (this.negotiated >= F.VERSION_PATCH) { + children.push(this.vu('version_patch').node); + } + if (this.negotiated >= F.CHUNKED_PROTOCOL) { + children.push(this.str('proto_send_chunked_srv').node); + children.push(this.str('proto_recv_chunked_srv').node); + } + if (this.negotiated >= F.PASSWORD_COMPLEXITY_RULES) { + children.push(this.decodePasswordRules()); + } + if (this.negotiated >= F.INTERSERVER_SECRET_V2) { + children.push(this.u64('nonce').node); + } + if (this.negotiated >= F.SERVER_SETTINGS) { + children.push(this.decodeSettingsList('server_settings')); + } + if (this.negotiated >= F.QUERY_PLAN_SERIALIZATION) { + children.push(this.vu('query_plan_serialization_version').node); + } + if (this.negotiated >= F.VERSIONED_CLUSTER_FUNCTION) { + children.push(this.vu('cluster_function_protocol_version').node); + } + // Forward-compat: some server builds (observed: 25.12.x, proto 54483) + // append extra trailing version VarUInt(s) after cluster_function that are + // absent from the public spec and source. Consume any trailing VarUInt + // that can't begin a valid post-hello server packet so the stream stays + // aligned. See docs/full_native_protocol_spec.md — this is a known gap. + while (this.r.offset < this.total) { + const peeked = this.peekVarUInt(); + if (peeked === null || VALID_FIRST_SERVER_PACKET.has(peeked)) break; + const node = this.vu('hello_tail_extra_version').node; + node.metadata = { specGap: true }; + children.push(node); + } + return this.packet('ServerHello', start, children); + } + + private decodePasswordRules(): AstNode { + const start = this.r.offset; + const { value: count, node: countNode } = this.vu('rule_count'); + const children: AstNode[] = [countNode]; + for (let i = 0; i < count; i++) { + const ruleStart = this.r.offset; + const pattern = this.str('pattern').node; + const message = this.str('message').node; + children.push(this.group(`rule[${i}]`, ruleStart, [pattern, message], '')); + } + return this.group('password_complexity_rules', start, children, `${count} rule(s)`); + } + + private decodeException(start: number): AstNode { + const children: AstNode[] = [this.typeNode('Exception', ServerPacket.Exception)]; + // Chain of nested exceptions: each ends with a has_nested Bool. + let depth = 0; + while (true) { + const exStart = this.r.offset; + const code = this.i32('code').node; + const name = this.str('name').node; + const message = this.str('message').node; + const stack = this.str('stack_trace').node; + const hasNestedRes = this.u8('has_nested'); + const hasNested = hasNestedRes.value; + const nestedFlag = hasNestedRes.node; + const exNode = this.group( + depth === 0 ? 'exception' : `nested_exception[${depth}]`, + exStart, + [code, name, message, stack, nestedFlag], + message.displayValue, + ); + children.push(exNode); + if (hasNested !== 1) break; + depth += 1; + if (depth > 64) break; // defensive bound + } + return this.packet('Exception', start, children); + } + + private decodeProgress(start: number): AstNode { + const children: AstNode[] = [this.typeNode('Progress', ServerPacket.Progress)]; + children.push(this.vu('rows').node); + children.push(this.vu('bytes').node); + children.push(this.vu('total_rows').node); + if (this.negotiated >= F.TOTAL_BYTES_IN_PROGRESS) { + children.push(this.vu('total_bytes').node); + } + if (this.negotiated >= F.WRITE_CLIENT_INFO) { + children.push(this.vu('wrote_rows').node); + children.push(this.vu('wrote_bytes').node); + } + if (this.negotiated >= F.SERVER_QUERY_TIME_IN_PROGRESS) { + children.push(this.vu('elapsed_ns').node); + } + return this.packet('Progress', start, children); + } + + private decodeProfileInfo(start: number): AstNode { + const children: AstNode[] = [this.typeNode('ProfileInfo', ServerPacket.ProfileInfo)]; + children.push(this.vu('rows').node); + children.push(this.vu('blocks').node); + children.push(this.vu('bytes').node); + children.push(this.bool('applied_limit').node); + children.push(this.vu('rows_before_limit').node); + children.push(this.bool('calculated_rows_before_limit').node); + if (this.negotiated >= F.ROWS_BEFORE_AGGREGATION) { + children.push(this.bool('applied_aggregation').node); + children.push(this.vu('rows_before_aggregation').node); + } + return this.packet('ProfileInfo', start, children); + } + + private decodeTableColumns(start: number): AstNode { + const children: AstNode[] = [this.typeNode('TableColumns', ServerPacket.TableColumns)]; + children.push(this.str('external_table').node); + children.push(this.str('columns_description').node); + return this.packet('TableColumns', start, children); + } + + // --- Data-family packets (table_name + Block) --------------------------- + + private decodeDataPacket(name: string, typeCode: number, start: number): AstNode { + const children: AstNode[] = [this.typeNode(name, typeCode)]; + children.push(this.str('table_name').node); + const block = this.native.decodeProtocolBlock(this.blockIndex++); + children.push(this.blockToAst(block)); + return this.packet(name, start, children); + } + + private blockToAst(block: BlockNode): AstNode { + const children: AstNode[] = [block.header.astNode]; + for (const col of block.columns) { + const colChildren: AstNode[] = [col.metadataNode, ...col.dataPrefixNodes, ...col.values]; + children.push({ + id: this.nid(), + type: col.typeString || 'Column', + byteRange: { start: col.metadataByteRange.start, end: col.dataByteRange.end }, + value: null, + displayValue: `${col.name}: ${col.typeString} · ${col.values.length} value(s)`, + label: col.name, + children: colChildren, + }); + } + return { + id: this.nid(), + type: 'Native.Block', + byteRange: block.byteRange, + value: { rows: block.rowCount, columns: block.columns.length }, + displayValue: `${block.rowCount} row(s) × ${block.columns.length} column(s)`, + label: 'block', + children, + }; + } + + // --- settings / parameters lists ---------------------------------------- + + private decodeSettingsList(label: string): AstNode { + const start = this.r.offset; + const children: AstNode[] = []; + let count = 0; + while (true) { + const keyStart = this.r.offset; + const { value: key, node: keyNode } = this.str('key'); + if (key === '') { + // Empty key = terminator (a single VarUInt 0). + keyNode.label = 'terminator'; + children.push(keyNode); + break; + } + const flags = this.vu('flags').node; + const value = this.str('value').node; + children.push(this.group(key, keyStart, [keyNode, flags, value], value.displayValue)); + count += 1; + if (count > 100000) throw new Error('settings list overflow'); + } + return this.group(label, start, children, `${count} entr${count === 1 ? 'y' : 'ies'}`); + } + + // --- version negotiation ------------------------------------------------ + + private computeNegotiatedVersion(): number { + const client = this.peekHelloVersion(this.combined.subarray(0, this.c2sLength)); + const server = this.peekHelloVersion(this.combined.subarray(this.c2sLength, this.total)); + if (client != null && server != null) return Math.min(client, server); + return client ?? server ?? 0; + } + + /** Read just the protocol_version out of a Hello at the start of `buf`. */ + private peekHelloVersion(buf: Uint8Array): number | null { + if (buf.length === 0) return null; + try { + const rr = new BinaryReader(buf); + const { value: type } = decodeLEB128(rr); + if (type !== 0) return null; // not a Hello (e.g. handshake Exception) + const { value: nameLen } = decodeLEB128(rr); + rr.skip(nameLen); + decodeLEB128(rr); // version_major + decodeLEB128(rr); // version_minor + return decodeLEB128(rr).value; // protocol_version + } catch { + return null; + } + } + + // --- primitive readers (each returns an AstNode + value) ---------------- + + private leaf(type: string, start: number, value: unknown, displayValue: string, label: string): AstNode { + return { id: this.nid(), type, byteRange: { start, end: this.r.offset }, value, displayValue, label }; + } + + private vu(label: string): { value: number; node: AstNode } { + const start = this.r.offset; + const { value } = decodeLEB128(this.r); + return { value, node: this.leaf('VarUInt', start, value, String(value), label) }; + } + + private str(label: string): { value: string; node: AstNode } { + const start = this.r.offset; + const { value: len } = decodeLEB128(this.r); + const { value: bytes } = this.r.readBytes(len); + const value = TEXT_DECODER.decode(bytes); + return { value, node: this.leaf('String', start, value, `"${value}"`, label) }; + } + + private u8(label: string): { value: number; node: AstNode } { + const start = this.r.offset; + const { value } = this.r.readUInt8(); + return { value, node: this.leaf('UInt8', start, value, String(value), label) }; + } + + private bool(label: string): { value: boolean; node: AstNode } { + const start = this.r.offset; + const { value } = this.r.readUInt8(); + return { value: value !== 0, node: this.leaf('Bool', start, value !== 0, value !== 0 ? 'true' : 'false', label) }; + } + + private i32(label: string): { value: number; node: AstNode } { + const start = this.r.offset; + const { value } = this.r.readInt32LE(); + return { value, node: this.leaf('Int32', start, value, String(value), label) }; + } + + private i64(label: string): { value: bigint; node: AstNode } { + const start = this.r.offset; + const { value } = this.r.readInt64LE(); + return { value, node: this.leaf('Int64', start, value, String(value), label) }; + } + + private u64(label: string): { value: bigint; node: AstNode } { + const start = this.r.offset; + const { value } = this.r.readUInt64LE(); + return { value, node: this.leaf('UInt64', start, value, String(value), label) }; + } + + private fixedBytes(n: number, label: string, typeName: string): AstNode { + const start = this.r.offset; + const { value } = this.r.readBytes(n); + const hex = Array.from(value).map((b) => b.toString(16).padStart(2, '0')).join(''); + return this.leaf(typeName, start, hex, `0x${hex}`, label); + } + + // --- node builders ------------------------------------------------------ + + private typeNode(name: string, code: number): AstNode { + // The leading VarUInt packet type code. Length is computed from the offset + // delta so multi-byte type codes (>=128) are tracked correctly. + const start = this.r.offset; + decodeLEB128(this.r); + return this.leaf('VarUInt', start, code, `${code} (${name})`, 'packet_type'); + } + + private peekTypeOrThrow(): { value: number } { + const v = this.peekVarUInt(); + if (v === null) throw new Error(`could not read packet type at offset ${this.r.offset}`); + return { value: v }; + } + + private peekVarUInt(): number | null { + const bytes = this.r.peekBytes(10); + let result = 0; + let shift = 0; + for (let i = 0; i < bytes.length; i++) { + const b = bytes[i]; + result += (b & 0x7f) * 2 ** shift; + if ((b & 0x80) === 0) return result; + shift += 7; + if (shift > 56) return null; + } + return null; + } + + /** A packet whose only content is its type code (Ping, Pong, Cancel, EndOfStream, ...). */ + private bodylessPacket(name: string, code: number, start: number): AstNode { + return this.packet(name, start, [this.typeNode(name, code)]); + } + + private packet(name: string, start: number, children: AstNode[]): AstNode { + return { + id: this.nid(), + type: `Protocol.${name}`, + byteRange: { start, end: this.r.offset }, + value: name, + displayValue: name, + label: name, + children, + }; + } + + private group(name: string, start: number, children: AstNode[], display: string): AstNode { + return { + id: this.nid(), + type: `Protocol.${name.replace(/[^A-Za-z0-9_]/g, '_')}`, + byteRange: { start, end: this.r.offset }, + value: name, + displayValue: display || name, + label: name, + children, + }; + } + + private emptyHeader(): HeaderNode { + const zero: ByteRange = { start: 0, end: 0 }; + return { byteRange: zero, columnCount: 0, columnCountRange: zero, columns: [] }; + } + + private nid(): string { + return `proto-${this.idCounter++}`; + } +} diff --git a/src/core/decoder/protocol-dump.ts b/src/core/decoder/protocol-dump.ts new file mode 100644 index 0000000..9b66120 --- /dev/null +++ b/src/core/decoder/protocol-dump.ts @@ -0,0 +1,59 @@ +import { ProtocolCapture } from './protocol-decoder'; + +/** + * Parser for the `.chproto` capture dump format written by the proxy harness + * (scripts/native-proxy.mjs). The format is: + * + * magic "CHPROTO1" (8 bytes ASCII) + * metaLen u32 LE + * meta metaLen bytes (UTF-8 JSON) + * segments repeated: [dir u8][len u32 LE][len bytes] + * + * where dir 0 = client→server, 1 = server→client. Segments of the same + * direction are concatenated into one contiguous stream (a packet may be split + * across TCP segments, so each direction must be decoded as one buffer). + */ +const MAGIC = 'CHPROTO1'; +const DIR_C2S = 0; +const DIR_S2C = 1; +const TEXT_DECODER = new TextDecoder(); + +export function parseChprotoDump(buf: Uint8Array): ProtocolCapture { + const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength); + const magic = TEXT_DECODER.decode(buf.subarray(0, MAGIC.length)); + if (magic !== MAGIC) { + throw new Error('not a CHPROTO dump (bad magic)'); + } + let pos = MAGIC.length; + const metaLen = view.getUint32(pos, true); + pos += 4; + const meta = JSON.parse(TEXT_DECODER.decode(buf.subarray(pos, pos + metaLen))) as Record; + pos += metaLen; + + const c2sChunks: Uint8Array[] = []; + const s2cChunks: Uint8Array[] = []; + while (pos < buf.length) { + const dir = view.getUint8(pos); + pos += 1; + const len = view.getUint32(pos, true); + pos += 4; + const chunk = buf.subarray(pos, pos + len); + pos += len; + if (dir === DIR_C2S) c2sChunks.push(chunk); + else if (dir === DIR_S2C) s2cChunks.push(chunk); + else throw new Error(`unknown segment direction ${dir}`); + } + + return { c2s: concat(c2sChunks), s2c: concat(s2cChunks), meta }; +} + +function concat(chunks: Uint8Array[]): Uint8Array { + const total = chunks.reduce((n, c) => n + c.length, 0); + const out = new Uint8Array(total); + let off = 0; + for (const c of chunks) { + out.set(c, off); + off += c.length; + } + return out; +} diff --git a/src/core/decoder/rowbinary-decoder.ts b/src/core/decoder/rowbinary-decoder.ts index 9636e4e..c9a6cd2 100644 --- a/src/core/decoder/rowbinary-decoder.ts +++ b/src/core/decoder/rowbinary-decoder.ts @@ -1,4 +1,5 @@ import { FormatDecoder } from './format-decoder'; +import { formatIPv6 } from './format-utils'; import { decodeLEB128 } from './leb128'; import { parseType } from '../parser/type-parser'; import { ClickHouseType, typeToString } from '../types/clickhouse-types'; @@ -168,6 +169,10 @@ export class RowBinaryDecoder extends FormatDecoder { case 'Bool': return this.decodeBool(); + // Nothing — one placeholder byte per row, value is always null + case 'Nothing': + return this.decodeNothing(); + // Date/Time case 'Date': return this.decodeDate(); @@ -552,6 +557,18 @@ export class RowBinaryDecoder extends FormatDecoder { }; } + // Nothing decoder — consumes one placeholder byte; the value is always null + private decodeNothing(): AstNode { + const { range } = this.reader.readUInt8(); + return { + id: this.generateId(), + type: 'Nothing', + byteRange: range, + value: null, + displayValue: 'ø', + }; + } + // Date/Time decoders private decodeDate(): AstNode { const { value, range } = this.reader.readUInt16LE(); @@ -706,12 +723,11 @@ export class RowBinaryDecoder extends FormatDecoder { private decodeIPv6(): AstNode { const { value: bytes, range } = this.reader.readBytes(16); - // Format as standard IPv6 - const groups: string[] = []; + const groups: number[] = []; for (let i = 0; i < 16; i += 2) { - groups.push(((bytes[i] << 8) | bytes[i + 1]).toString(16)); + groups.push((bytes[i] << 8) | bytes[i + 1]); } - const ip = groups.join(':'); + const ip = formatIPv6(groups); return { id: this.generateId(), diff --git a/src/core/parser/type-parser.ts b/src/core/parser/type-parser.ts index fad84fb..9b66a00 100644 --- a/src/core/parser/type-parser.ts +++ b/src/core/parser/type-parser.ts @@ -60,6 +60,7 @@ export function parseType(typeString: string): ClickHouseType { UUID: { kind: 'UUID' }, IPv4: { kind: 'IPv4' }, IPv6: { kind: 'IPv6' }, + Nothing: { kind: 'Nothing' }, Point: { kind: 'Point' }, Ring: { kind: 'Ring' }, Polygon: { kind: 'Polygon' }, diff --git a/src/core/types/ast.ts b/src/core/types/ast.ts index 28dd8fb..1739da9 100644 --- a/src/core/types/ast.ts +++ b/src/core/types/ast.ts @@ -135,4 +135,6 @@ export interface ParsedData { blocks?: BlockNode[]; /** Trailing protocol nodes not attached to data rows/blocks (for example terminal Native blocks) */ trailingNodes?: AstNode[]; + /** Optional decoder-specific metadata (for example the negotiated protocol version). */ + metadata?: Record; } diff --git a/src/core/types/clickhouse-types.ts b/src/core/types/clickhouse-types.ts index d0ab5d1..17d35f2 100644 --- a/src/core/types/clickhouse-types.ts +++ b/src/core/types/clickhouse-types.ts @@ -39,6 +39,8 @@ export type ClickHouseType = | { kind: 'IPv4' } | { kind: 'IPv6' } | { kind: 'Bool' } + // Unit type — the inner type of Nullable(Nothing) (e.g. SELECT NULL) + | { kind: 'Nothing' } // Enums | { kind: 'Enum8'; values: Map } | { kind: 'Enum16'; values: Map } @@ -109,6 +111,7 @@ export function typeToString(type: ClickHouseType): string { case 'IPv4': case 'IPv6': case 'Bool': + case 'Nothing': case 'Point': case 'Ring': case 'Polygon': @@ -292,6 +295,9 @@ export function getTypeColor(type: ClickHouseType): string { case 'Bool': return 'var(--type-bool)'; + case 'Nothing': + return 'var(--type-default)'; + // Advanced types case 'Variant': case 'Dynamic': diff --git a/src/core/types/formats.ts b/src/core/types/formats.ts index bb47492..3f5db7d 100644 --- a/src/core/types/formats.ts +++ b/src/core/types/formats.ts @@ -4,6 +4,8 @@ export enum ClickHouseFormat { RowBinaryWithNamesAndTypes = 'RowBinaryWithNamesAndTypes', Native = 'Native', + /** Native TCP protocol capture (packet stream), not an HTTP FORMAT. */ + NativeProtocol = 'NativeProtocol', } /** @@ -28,8 +30,14 @@ export const FORMAT_METADATA: Record = { }, [ClickHouseFormat.Native]: { id: ClickHouseFormat.Native, - displayName: 'Native', - description: 'Column-oriented binary format with blocks', + displayName: 'Native format (HTTP)', + description: 'Column-oriented Native format body fetched over HTTP', + supportsBlocks: true, + }, + [ClickHouseFormat.NativeProtocol]: { + id: ClickHouseFormat.NativeProtocol, + displayName: 'Native protocol + format (TCP)', + description: 'Full native TCP protocol packet stream (handshake, packets, and Native blocks) captured via proxy', supportsBlocks: true, }, }; diff --git a/src/store/store.ts b/src/store/store.ts index ec1c79d..9c0f921 100644 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -1,6 +1,7 @@ import { create } from 'zustand'; import { clickhouse, DEFAULT_QUERY } from '../core/clickhouse/client'; import { createDecoder } from '../core/decoder'; +import { parseChprotoDump } from '../core/decoder/protocol-dump'; import { AstNode, ParsedData } from '../core/types/ast'; import { ClickHouseFormat } from '../core/types/formats'; import { DEFAULT_NATIVE_PROTOCOL_VERSION } from '../core/types/native-protocol'; @@ -91,6 +92,14 @@ function getDefaultExpanded(parsedData: ParsedData): Set { parsedData.blocks?.forEach((_, i) => { expanded.add(`block-${i}`); }); + // Protocol captures: expand the two direction sections and each packet so + // the conversation timeline is visible, but leave packet fields collapsed. + if (parsedData.format === ClickHouseFormat.NativeProtocol) { + parsedData.trailingNodes?.forEach((section) => { + expanded.add(section.id); + section.children?.forEach((packet) => expanded.add(packet.id)); + }); + } return expanded; } @@ -151,6 +160,16 @@ export const useStore = create((set, get) => ({ set(getLoadingState()); try { + if (format === ClickHouseFormat.NativeProtocol) { + // Capture the full native TCP packet stream via the proxy harness + // (desktop only) and decode the conversation, not just one format body. + const { combined, c2sLength, timing } = await clickhouse.captureProtocol(query); + const decoder = createDecoder(combined, format, { protocolC2SLength: c2sLength }); + const parsed = decoder.decode(); + set(getSuccessState(combined, parsed, timing)); + return; + } + const { data, timing } = await clickhouse.query({ query, format, nativeProtocolVersion }); const decoder = createDecoder(data, format, { nativeProtocolVersion }); const parsed = decoder.decode(); @@ -168,6 +187,22 @@ export const useStore = create((set, get) => ({ try { const arrayBuffer = await file.arrayBuffer(); const data = new Uint8Array(arrayBuffer); + + // A .chproto capture (or the NativeProtocol format) is decoded as a + // protocol packet stream. The dump carries the c2s/s2c split itself. + if (file.name.endsWith('.chproto') || format === ClickHouseFormat.NativeProtocol) { + const capture = parseChprotoDump(data); + const combined = new Uint8Array(capture.c2s.length + capture.s2c.length); + combined.set(capture.c2s, 0); + combined.set(capture.s2c, capture.c2s.length); + const decoder = createDecoder(combined, ClickHouseFormat.NativeProtocol, { + protocolC2SLength: capture.c2s.length, + }); + const parsed = decoder.decode(); + set(getSuccessState(combined, parsed, null)); + return; + } + const decoder = createDecoder(data, format, { nativeProtocolVersion }); const parsed = decoder.decode(); set(getSuccessState(data, parsed, null)); diff --git a/vite.config.ts b/vite.config.ts index 7351f88..485e1a2 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,5 +1,7 @@ import { defineConfig } from 'vite' import react from '@vitejs/plugin-react' +// @ts-expect-error - plain ESM helper, no type declarations +import { captureServerPlugin } from './scripts/capture-middleware.mjs' const isElectron = !!process.env.ELECTRON; @@ -37,7 +39,7 @@ export default defineConfig(async () => { } return { - plugins: [react(), ...electronPlugins], + plugins: [react(), captureServerPlugin(), ...electronPlugins], server: { proxy: { '/clickhouse': {