diff --git a/Claude.md b/Claude.md new file mode 100644 index 0000000..9343c4b --- /dev/null +++ b/Claude.md @@ -0,0 +1,1144 @@ +# Policy-zig - Project Overview + +## Architecture + +This is a Zig project implementing a computing runtime with a hybrid +architectural approach: + +- **Data-Oriented Design (DoD)**: Primary focus for performance-critical paths +- **Functional Programming**: For composability and testability +- **Object-Oriented Programming**: Where it provides clarity without sacrificing + performance + +## Data-Oriented Design Principles + +### Core Philosophy + +**The CPU is fast, but memory is slow.** All design decisions must optimize for +memory access patterns and cache coherency. **Predictable memory usage is the +goal.** Each binary should have a predictable memory footprint based on +throughput. + +### Key Strategies + +#### 1. Identify and Optimize Uniform Data + +- Group similar data together in memory +- Minimize the size of each individual item +- Process data in bulk rather than one item at a time + +#### 2. Use Indexes Instead of Pointers + +```zig +// BAD: Pointer-heavy approach (64 bits per reference) +const Node = struct { + data: Data, + next: ?*Node, + parent: ?*Node, +}; + +// GOOD: Index-based approach (typically 32 bits or less) +const NodeId = u32; +const Node = struct { + data: Data, + next: ?NodeId, + parent: ?NodeId, +}; +const NodeStorage = std.ArrayList(Node); +``` + +#### 3. Manage Type Safety with Compact Representations + +```zig +// Use strongly-typed handles instead of raw integers +const HandleType = enum { request, response, connection }; +fn Handle(comptime T: HandleType) type { + return enum(u32) { _ }; +} + +const RequestHandle = Handle(.request); +const ResponseHandle = Handle(.response); +``` + +#### 4. Store Booleans Separately + +SKIP THIS FOR NOW. + +#### 5. Eliminate Padding with Struct-of-Arrays + +```zig +// BAD: Array-of-Structs (AoS) - poor cache utilization +const Connection = struct { + id: u64, // 8 bytes + status: u8, // 1 byte + 3 bytes padding + port: u16, // 2 bytes + 2 bytes padding + address: [16]u8, // 16 bytes +}; +const connections = std.ArrayList(Connection); + +// GOOD: Struct-of-Arrays (SoA) - excellent cache utilization +const Connection = struct { + id: u64, // 8 bytes + status: u8, // 1 byte + 3 bytes padding + port: u16, // 2 bytes + 2 bytes padding + address: [16]u8, // 16 bytes +}; +const connections = std.MultiArrayList(Connection); + +``` + +#### 6. Use Hash Maps for Sparse Data + +```zig +// When only a small subset of entities has certain data +const SparseMetadata = std.AutoHashMap(EntityId, Metadata); +``` + +#### 7. Encoding Instead of Polymorphism + +```zig +// BAD: Runtime polymorphism (vtable overhead) +const Handler = struct { + vtable: *const VTable, + data: *anyopaque, +}; + +// GOOD: Tagged unions with explicit types +const HandlerType = enum { http, websocket, grpc }; +const Handler = union(HandlerType) { + http: HttpHandler, + websocket: WebSocketHandler, + grpc: GrpcHandler, + + fn process(self: *Handler, data: []const u8) void { + switch (self.*) { + .http => |*h| h.processHttp(data), + .websocket => |*w| w.processWs(data), + .grpc => |*g| g.processGrpc(data), + } + } +}; +``` + +## Module System + +### Philosophy + +The codebase is organized into **composable modules** that can be selectively +combined to create different distributions. Each module should: + +1. Be self-contained with minimal dependencies +2. Export a clear public API through its root file +3. Define its own types and data structures +4. Be independently testable + +### Structure + +``` +src/ +├── core/ # Core runtime primitives +│ ├── allocator.zig +│ ├── event_loop.zig +│ └── scheduler.zig +├── network/ # Network stack +│ ├── tcp.zig +│ ├── udp.zig +│ └── http.zig +├── storage/ # Storage layer +│ ├── cache.zig +│ └── persistence.zig +├── security/ # Security primitives +│ ├── tls.zig +│ └── auth.zig +└── distributions/ # Distribution-specific compositions + ├── minimal.zig + ├── full.zig + └── custom.zig +``` + +### Distribution Composition + +Each distribution manually selects and composes the modules it needs: + +```zig +// distributions/minimal.zig +const core = @import("../core/event_loop.zig"); +const network = @import("../network/tcp.zig"); + +pub fn main() !void { + var runtime = try core.Runtime.init(); + defer runtime.deinit(); + + var tcp_server = try network.TcpServer.init(&runtime); + defer tcp_server.deinit(); + + try runtime.run(); +} +``` + +## Code Organization Best Practices + +### 1. Separate Hot and Cold Data + +```zig +// Hot data: accessed frequently +const EntityHot = struct { + position: Vec3, + velocity: Vec3, +}; + +// Cold data: accessed rarely +const EntityCold = struct { + name: []const u8, + metadata: Metadata, +}; + +const EntityId = u32; +const hot_data = std.ArrayList(EntityHot); +const cold_data = std.AutoHashMap(EntityId, EntityCold); +``` + +### 2. Batch Processing + +```zig +// Process data in batches for better cache utilization +fn updatePositions(entities: []Entity, dt: f32) void { + for (entities) |*entity| { + entity.position.x += entity.velocity.x * dt; + entity.position.y += entity.velocity.y * dt; + entity.position.z += entity.velocity.z * dt; + } +} +``` + +### 3. Memory Alignment + +```zig +// Align data structures for optimal CPU access +const CacheLine = 64; // bytes +const alignas = CacheLine; + +const PerformanceCritical = struct { + // Hot data aligned to cache line + data: [16]u32 align(alignas), +}; +``` + +### 4. Arena Allocators for Temporary Data + +```zig +fn processRequest(arena: *std.heap.ArenaAllocator, request: Request) !Response { + // All temporary allocations use arena + const temp_buffer = try arena.allocator().alloc(u8, 1024); + // No need to free - arena will be reset + + // ... process request ... + + return response; +} +``` + +## Testing Strategy + +- **Unit tests**: Test individual functions with `test` blocks +- **Integration tests**: Test module interactions +- **Benchmark tests**: Verify performance characteristics using `std.time` +- **Memory tests**: Use `std.testing.allocator` to detect leaks + +## Performance Guidelines + +1. **Profile before optimizing** - Use `std.time.Timer` and platform profilers +2. **Minimize allocations** - Reuse buffers where possible +3. **Prefer stack over heap** - Use fixed-size buffers when bounds are known +4. **Avoid indirection** - Direct function calls over function pointers when + possible +5. **Think in terms of data transformations** - Input → Process → Output + +## Functional Programming Aspects + +- Use pure functions where possible (no side effects) +- Leverage comptime for zero-cost abstractions +- Prefer explicit error handling with `!` and `catch` +- Use const by default, mut when necessary + +## When to Use OOP + +Object-oriented patterns are acceptable when: + +- They improve code clarity without performance cost +- The domain naturally maps to objects (e.g., Connection, Server) +- Methods help organize related functionality +- State encapsulation provides safety + +**Always measure**: If OOP introduces indirection or hurts cache coherency, +refactor to DoD. + +## Build System + +The project uses Zig's build system (`build.zig`) to: + +- Define multiple build targets (distributions) +- Manage dependencies +- Configure compilation options +- Run tests + +## Getting Started + +When modifying or extending this project: + +1. Identify the module(s) you need to change +2. Consider the data layout and access patterns +3. Prefer transforming contiguous arrays over scattered object updates +4. Write tests to verify correctness +5. Benchmark to verify performance +6. Update relevant distributions if needed + +## Questions to Ask Before Coding + +1. How will this data be accessed? (Sequential, random, sparse?) +2. What is the typical working set size? +3. Can I process this in batches? +4. Are there hot/cold splits in this data? +5. Can I use indexes instead of pointers? +6. Does this need to be in the hot path? + +--- + +Remember: **Optimize for data locality first, algorithmic complexity second.** + +--- + +# Zig 0.15.x std.io Breaking Changes Reference + +> **HISTORICAL (2026-06):** the codebase is now fully Zig 0.16-native — std.Io +> is an injected interface (io parameter everywhere), the HTTP data plane rides +> std.Io.net + std.http.Server with structured cancellation (Io.Group), and +> request bodies stream through src/pipeline/. The notes below describe the +> 0.14→0.15 migration era and are kept for context only; verify any API against +> zigdoc before relying on it. + +**Zig 0.15.x introduced "Writergate"—a complete redesign of std.io Reader and +Writer interfaces that fundamentally changes how I/O operations work.** The new +interfaces replace generic `anytype`-based APIs with concrete types that +integrate buffering directly into the interface itself. This change delivers +45-49% performance improvements but requires explicit buffer management and +manual flushing. All existing code using std.io readers and writers will break +and must be migrated. + +## Why this matters for Tero policy-zig + +These changes are **extremely breaking** (Andrew Kelley's words) and affect +every file, network, and console I/O operation. The new architecture eliminates +the generic `std.io.GenericReader` and `std.io.GenericWriter` types, requiring +buffers to be provided at instantiation time. Code that worked in Zig 0.14.x +will fail to compile or produce silent failures (missing output due to unflushed +buffers) in 0.15.x. This redesign is permanent and foundational for Zig's future +async/await implementation—there is no going back to the old interfaces. + +Understanding these changes is critical for Tero policy-zig because the project +needs current, accurate I/O patterns. The old patterns are now deprecated or +completely removed. Future AI assistants must know: **you cannot copy interface +instances, you must always flush writers, and buffers are mandatory**. These +aren't optional optimizations—they're fundamental requirements of the new API. + +--- + +## The fundamental architectural shift + +Zig 0.15.x moves from generic, implementation-defined buffering to concrete, +interface-level buffering. The **buffer now lives in the interface itself** +(above the vtable), not in the implementation. This "buffer above vtable" design +enables the hot path—most I/O operations—to work directly on the buffer without +vtable calls, while only calling into the vtable when the buffer fills or +empties. + +**Old architecture (0.14.x):** Generic types like `std.io.GenericReader(T)` and +`std.io.GenericWriter(T)` wrapped underlying resources. Buffering was optional +via separate `bufferedReader()` and `bufferedWriter()` wrappers. Every function +accepting a reader or writer became generic with `anytype` parameters, poisoning +struct types. + +**New architecture (0.15.x):** Concrete `std.Io.Reader` and `std.Io.Writer` +types with integrated buffers. No more generics for basic I/O. Buffer is a +required field in the interface struct itself, making buffering the default +behavior rather than an opt-in wrapper. + +The performance gains are substantial: **45.7% faster wall time**, 49% fewer CPU +cycles, and 24.4% fewer instructions in I/O-heavy operations. Debug mode +compilation is roughly 5x faster using the self-hosted x86 backend. The +trade-off is explicit complexity—developers must manage buffers and remember to +flush. + +--- + +## Critical breaking change 1: Explicit buffer requirements + +Every reader and writer instantiation **must provide a buffer**. There is no +default allocation, no hidden memory management. This is Zig philosophy: no +hidden control flow, no surprise allocations. + +**Code that breaks:** + +```zig +// 0.14.x - worked fine +const stdout = std.io.getStdOut().writer(); +try stdout.print("Hello\n", .{}); +``` + +**Required pattern in 0.15.x:** + +```zig +// Must allocate buffer explicitly +var stdout_buffer: [4096]u8 = undefined; +var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); +const stdout = &stdout_writer.interface; +try stdout.print("Hello\n", .{}); +try stdout.flush(); // CRITICAL: Must flush! +``` + +**Buffer sizing guidelines:** + +- **4096 bytes (one page):** Standard for general I/O operations +- **8192 bytes:** Better for high-throughput scenarios +- **std.crypto.tls.max_ciphertext_record_len:** Required minimum for TLS + operations +- **Empty buffer `&.{}`:** Opt-out to unbuffered mode (not recommended) + +Buffers can be stack-allocated for short-lived operations or heap-allocated for +long-running resources. The official recommendation is to **consider making +frequently-used buffers (like stdout) global** to avoid repeated stack +allocation overhead. + +--- + +## Critical breaking change 2: The flush requirement + +Buffered data **does not automatically commit**. Forgetting `flush()` causes +silent failures where programs run successfully but produce no output. This is +the single most common migration bug. + +**Why flushing is mandatory:** The interface accumulates data in its buffer to +reduce system calls. Until you call `flush()`, the data remains in memory, never +reaching the file descriptor, socket, or console. + +**When you must flush:** + +- Before program exit +- After writing data you expect to see immediately +- Before switching between read and write operations on the same resource +- Before passing control to code that might read what you wrote +- After writing to stdout/stderr if you need immediate visibility + +**Example of silent failure:** + +```zig +var buffer: [4096]u8 = undefined; +var writer = std.fs.File.stdout().writer(&buffer); +const stdout = &writer.interface; +try stdout.print("Important message\n", .{}); +// Program ends - buffer never flushed, output never appears! +``` + +**Correct pattern:** + +```zig +var buffer: [4096]u8 = undefined; +var writer = std.fs.File.stdout().writer(&buffer); +const stdout = &writer.interface; +try stdout.print("Important message\n", .{}); +try stdout.flush(); // Now output appears +``` + +--- + +## Critical breaking change 3: The @fieldParentPtr footgun + +The new interfaces use `@fieldParentPtr` internally to recover the parent +wrapper struct from the interface field. This creates a **critical invariant: +you must never copy interface instances**. + +**What breaks:** + +```zig +var file_reader = file.reader(&buf); +const reader = file_reader.interface; // WRONG - copies the interface +while (reader.takeDelimiterExclusive('\n')) |line| { + // Crashes with "switch on corrupt value" or undefined behavior +} +``` + +**Correct pattern:** + +```zig +var file_reader = file.reader(&buf); +const reader = &file_reader.interface; // RIGHT - pointer to interface +while (reader.takeDelimiterExclusive('\n')) |line| { + // Works correctly +} +``` + +**Why this happens:** When you copy the interface, the vtable methods call +`@fieldParentPtr` expecting to find the parent struct at a specific memory +offset. But the copied interface is no longer at that offset relative to the +parent, causing memory corruption or crashes. + +**Rule of thumb:** Interface types are always passed as pointers +(`*std.Io.Reader`, `*std.Io.Writer`). Never use them as values. + +--- + +## New Reader interface and methods + +The `std.Io.Reader` type replaces `std.io.GenericReader` and all reader-related +generic types. It's a concrete struct with three fields: + +```zig +pub const Reader = struct { + vtable: *const VTable, + buffer: []u8, // Ring buffer + seek: usize = 0, // Bytes consumed from buffer + end: usize = 0, // End of buffered data +}; +``` + +**Key methods that replace old APIs:** + +**`takeDelimiterExclusive(delimiter: u8)`** replaces +`readUntilDelimiterOrEof()`: + +```zig +while (reader.takeDelimiterExclusive('\n')) |line| { + // Process line (delimiter not included) +} else |err| switch (err) { + error.EndOfStream => {}, // Normal end of stream + error.StreamTooLong => {}, // Line exceeds buffer size + error.ReadFailed => return err, // Actual I/O error +} +``` + +**`stream(writer: *Writer, limit: StreamLimit)`** for copying data: + +```zig +// Copy up to 1024 bytes from reader to writer +const n = try reader.stream(writer, .limited(1024)); + +// Copy until end of stream +const n = try reader.stream(writer, .until_end); +``` + +**`peek()` and `take()`** for zero-copy buffer access: + +```zig +const available = reader.peek(); // View buffered data without consuming +const slice = try reader.take(100); // Take exactly 100 bytes +``` + +**`toss(n: usize)`** to discard data: + +```zig +// Skip delimiter byte after streamDelimiter +try reader.toss(1); +``` + +**Removed methods from 0.14.x:** + +- `readUntilDelimiterOrEof()` → Use `takeDelimiterExclusive()` +- `readAll()` → Use `stream()` with appropriate limit +- `readNoEof()` → Use `take()` +- Generic variants → All replaced by concrete methods + +--- + +## New Writer interface and methods + +The `std.Io.Writer` type replaces `std.io.GenericWriter` with a concrete +implementation: + +```zig +pub const Writer = struct { + vtable: *const VTable, + buffer: []u8, + end: usize = 0, // Current buffer fill level +}; +``` + +**Essential methods:** + +**`writeAll(bytes: []const u8)`** for writing complete slices: + +```zig +try writer.writeAll("Complete message\n"); +``` + +**`print(comptime fmt: []const u8, args: anytype)`** for formatted output: + +```zig +try writer.print("Value: {d}\n", .{42}); +``` + +**`flush()`** to commit buffered data: + +```zig +try writer.flush(); // Calls drain() on implementation +``` + +**`drain()`** (vtable method) for custom implementations—writes buffered data to +underlying resource. + +**New specialized writers:** + +**`std.Io.Writer.Allocating`** for dynamic allocation: + +```zig +var writer = std.Io.Writer.Allocating.init(allocator); +defer writer.deinit(); +try writer.writer.writeAll("Data"); +const result = writer.written(); // Get accumulated slice +``` + +**`std.Io.Writer.Discarding`** for counting without storage: + +```zig +var counter = std.Io.Writer.Discarding{}; +try counter.writer.writeAll(data); +const bytes_written = counter.count; +``` + +**`std.Io.Writer.fixed(buffer: []u8)`** for fixed buffers: + +```zig +var buf: [1024]u8 = undefined; +var writer = std.Io.Writer.fixed(&buf); +try writer.writeAll("data"); +``` + +--- + +## Stream interfaces are gone + +The `std.io.SeekableStream` type and related abstractions **have been completely +removed**. There is no direct replacement—use specific concrete types instead. + +**Migration paths:** + +**For file operations:** + +- Use `*std.fs.File.Reader` with positional read support +- Use `*std.fs.File.Writer` with positional write support +- These types memoize file information (size, position) for efficient seeking + +**For memory operations:** + +- Use `std.ArrayListUnmanaged(u8)` as a growable buffer +- Use `std.Io.Writer.fixed()` for fixed-size memory buffers + +**For network operations:** + +- `std.net.Stream.reader()` and `std.net.Stream.writer()` now return + `std.fs.File.Reader` and `std.fs.File.Writer` wrappers +- Must provide buffers to these methods + +**Example migration:** + +```zig +// 0.14.x - had SeekableStream +var seekable = file.seekableStream(); +try seekable.seekTo(100); + +// 0.15.x - use File.Reader with positional reads +var buf: [4096]u8 = undefined; +var reader = file.reader(&buf); +// Positional reads are automatic when file supports them +``` + +--- + +## File operations now require explicit buffering + +The `std.fs.File` type's reader and writer methods **require buffer +parameters**. + +**Old pattern (0.14.x):** + +```zig +const file = try std.fs.cwd().openFile("data.txt", .{}); +const reader = file.reader(); // No buffer needed +``` + +**New pattern (0.15.x):** + +```zig +const file = try std.fs.cwd().openFile("data.txt", .{}); +var read_buffer: [4096]u8 = undefined; +var file_reader = file.reader(&read_buffer); +const reader = &file_reader.interface; +``` + +**File.Reader benefits:** + +- Memoizes file size and position +- Automatically uses positional reads when available (seeking becomes no-ops) +- Supports `sendFile()` for zero-copy file-to-file transfers +- Falls back gracefully to streaming when positional I/O unavailable + +**File.Writer benefits:** + +- Similar memoization for write operations +- Integrated buffering reduces system calls +- Positional writes when supported + +**Removed file APIs:** + +- `writeFileAll()` → Use `File.Writer` methods +- `writeFileAllUnseekable()` → Use `File.Writer` methods +- Direct `WriteFileOptions` → Pass buffer to `writer()` method + +**Changed APIs:** + +- `std.fs.Dir.atomicFile()` now requires `write_buffer` in options +- `std.fs.Dir.copyFile()` can no longer fail with `error.OutOfMemory` (uses + stack-allocated buffers) + +--- + +## Complete list of std.io namespace changes + +**Fully removed (no deprecation period):** + +- `std.io.SeekableStream` → Use concrete file/memory types +- `std.io.BitReader` → Removed entirely +- `std.io.BitWriter` → Removed entirely +- `std.io.LimitedReader` → Use `Reader.stream()` with `.limited()` +- `std.fifo.LinearFifo` → Replaced by Reader/Writer as ring buffers +- `std.RingBuffer` → Obsolete with new interfaces +- All ring buffer implementations (5 different types consolidated) + +**Deprecated (warnings, will be removed):** + +- `std.io.GenericReader` → Use `std.Io.Reader` +- `std.io.GenericWriter` → Use `std.Io.Writer` +- `std.io.AnyReader` → Use `std.Io.Reader` +- `std.io.AnyWriter` → Use `std.Io.Writer` +- `std.io.bufferedReader()` → Buffering now built-in +- `std.io.bufferedWriter()` → Buffering now built-in + +**New APIs replacing old patterns:** + +- `takeDelimiterExclusive()` replaces `readUntilDelimiterOrEof()` +- `stream()` replaces `readAll()` and related methods +- `Reader.pull()` (vtable) replaces custom read implementations +- `Writer.drain()` (vtable) replaces custom write implementations + +**Migration adapter (temporary):** + +```zig +// For gradual migration only - has known bugs +fn foo(old_writer: anytype) !void { + var adapter = old_writer.adaptToNewApi(&.{}); + const w: *std.Io.Writer = &adapter.new_interface; + try w.print("{s}", .{"example"}); +} +``` + +This adapter is **not recommended for production** and may not work in all cases +(GitHub issue #24483 documents limitations). + +--- + +## HTTP Client and Server changes + +Both `std.http.Client` and `std.http.Server` underwent major restructuring due +to the I/O changes. + +**HTTP Server migration:** + +```zig +// 0.14.x +var read_buffer: [8000]u8 = undefined; +var server = std.http.Server.init(connection, &read_buffer); + +// 0.15.x - separate buffers for read and write +var recv_buffer: [4000]u8 = undefined; +var send_buffer: [4000]u8 = undefined; +var conn_reader = connection.stream.reader(&recv_buffer); +var conn_writer = connection.stream.writer(&send_buffer); +var server = std.http.Server.init(conn_reader.interface(), &conn_writer.interface); +``` + +**HTTP Client migration:** + +```zig +// 0.14.x +var server_header_buffer: [1024]u8 = undefined; +var req = try client.open(.GET, uri, .{ + .server_header_buffer = &server_header_buffer, +}); +defer req.deinit(); +try req.send(); +try req.wait(); +const body_reader = try req.reader(); + +// 0.15.x - new request/response pattern +var req = try client.request(.GET, uri, .{}); +defer req.deinit(); +try req.sendBodiless(); +var response = try req.receiveHead(&.{}); + +// Process headers before reading body +var it = response.head.iterateHeaders(); +while (it.next()) |header| { + // Headers become invalid after reader() call +} + +var reader_buffer: [100]u8 = undefined; +const body_reader = response.reader(&reader_buffer); +``` + +**Key architectural improvement:** HTTP modules no longer depend on `std.net`. +They operate purely on Reader/Writer interfaces, making them more testable and +flexible. + +--- + +## TLS client requires four separate buffers + +The `std.crypto.tls.Client` type now requires **four distinct buffers** for +operation: + +```zig +// 1. Network stream read buffer +var stream_read_buf: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined; +var stream_reader = stream.reader(&stream_read_buf); + +// 2. Network stream write buffer +var stream_write_buf: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined; +var stream_writer = stream.writer(&stream_write_buf); + +// 3. TLS client read buffer +var tls_read_buf: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined; + +// 4. TLS client write buffer +var tls_write_buf: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined; + +var tls_client = try std.crypto.tls.Client.init( + stream_reader.interface(), + &stream_writer.interface, + .{ + .ca = .{.bundle = bundle}, + .host = .{ .explicit = "example.com" }, + .read_buffer = &tls_read_buf, + .write_buffer = &tls_write_buf, + }, +); +``` + +This complexity has been controversial in the community, but reflects the +layered nature of TLS: the outer layer buffers network I/O, the inner layer +buffers TLS protocol operations. + +**Benefit:** `std.crypto.tls.Client` no longer depends on `std.net` or `std.fs`, +operating purely on abstract I/O interfaces. + +--- + +## Compression API restructured + +The `std.compress.flate` module underwent major changes: + +**Decompression (still supported):** + +```zig +var decompress_buffer: [std.compress.flate.max_window_len]u8 = undefined; +var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &decompress_buffer); +const decompress_reader: *std.Io.Reader = &decompress.reader; + +// Or stream directly to writer +var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{}); +const n = try decompress.streamRemaining(writer); +``` + +**Compression (temporarily removed):** Deflate compression functionality was +removed in 0.15.x. The official release notes state this is temporary, +prioritizing language design over standard library completeness. Third-party +compression libraries are recommended for now. + +**Container formats:** The API now requires explicit container parameter +(`.zlib`, `.gzip`, `.raw`) instead of inferring from context. + +--- + +## Format string changes require migration + +Custom format implementations must be updated to the new signature: + +**Old format method (0.14.x):** + +```zig +pub fn format( + this: @This(), + comptime format_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, +) !void { + try writer.print("Custom: {}", .{this.value}); +} +``` + +**New format method (0.15.x):** + +```zig +pub fn format( + this: @This(), + writer: *std.Io.Writer +) std.Io.Writer.Error!void { + try writer.print("Custom: {}", .{this.value}); +} +``` + +**Format specifier changes:** + +- **`{}`** is now ambiguous if type has a `format()` method (compilation error) +- **`{f}`** explicitly calls the `format()` method +- **`{any}`** explicitly skips the `format()` method, using default + representation + +**Why this changed:** Prevents silent bugs where adding or removing a format +method changes output behavior without any code changes at call sites. + +**Migration strategy:** Use `zig build -freference-trace` to find all format +string issues. The compiler will show exact locations where format specifiers +need updating. + +--- + +## New best practices for Zig 0.15.x I/O + +**1. Default to buffering with appropriate sizes** + +Use 4096 bytes (one memory page) as the standard buffer size for general I/O. +Increase to 8192 or more for high-throughput operations. Use +`std.crypto.tls.max_ciphertext_record_len` for TLS operations. + +```zig +// Standard pattern +var buffer: [4096]u8 = undefined; +var writer = file.writer(&buffer); +``` + +**2. Always flush before visibility is needed** + +Flush writers before program exit, before reading back from the same resource, +and whenever output must be immediately visible. + +```zig +try writer.print("Status: {s}\n", .{status}); +try writer.flush(); // Make status visible now +``` + +**3. Use pointers for interfaces, never copy** + +Always reference interfaces via pointer to avoid the @fieldParentPtr footgun. + +```zig +const writer = &file_writer.interface; // RIGHT +``` + +**4. Consider global buffers for frequently-used I/O** + +For stdout/stderr used throughout a program, making the buffer global avoids +repeated stack allocation and simplifies code. + +```zig +var stdout_buffer: [8192]u8 = undefined; + +pub fn main() !void { + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + const stdout = &stdout_writer.interface; + // Use stdout throughout program +} +``` + +**5. Handle all error cases from new Reader methods** + +The new delimiter methods return distinct error types that should be handled +explicitly. + +```zig +while (reader.takeDelimiterExclusive('\n')) |line| { + // Process line +} else |err| switch (err) { + error.EndOfStream => {}, // Normal completion + error.StreamTooLong => return err, // Line exceeded buffer + error.ReadFailed => return err, // I/O failure +} +``` + +**6. Use Writer.Allocating for unbounded data** + +When reading lines that may exceed buffer size, use allocating writer: + +```zig +var line_writer = std.Io.Writer.Allocating.init(allocator); +defer line_writer.deinit(); + +while (reader.streamDelimiter(&line_writer.writer, '\n')) |_| { + const line = line_writer.written(); + // Process line + line_writer.clearRetainingCapacity(); + try reader.toss(1); // Skip delimiter +} else |err| if (err != error.EndOfStream) return err; +``` + +**7. Accept *std.Io.Reader and *std.Io.Writer in public APIs** + +Use concrete interface types in function signatures instead of `anytype`: + +```zig +pub fn processData(reader: *std.Io.Reader, writer: *std.Io.Writer) !void { + // No longer forces calling code to be generic +} +``` + +--- + +## Common migration patterns + +**Pattern: Stdout printing** + +```zig +// Allocate buffer once (can be global) +var stdout_buffer: [4096]u8 = undefined; +var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); +const stdout = &stdout_writer.interface; + +// Use throughout program +try stdout.print("Message: {s}\n", .{message}); +try stdout.flush(); // Don't forget! +``` + +**Pattern: Reading file line by line** + +```zig +const file = try std.fs.cwd().openFile("data.txt", .{}); +defer file.close(); + +var buffer: [8192]u8 = undefined; +var file_reader = file.reader(&buffer); +const reader = &file_reader.interface; + +while (reader.takeDelimiterExclusive('\n')) |line| { + // Process each line +} else |err| if (err != error.EndOfStream) return err; +``` + +**Pattern: Writing to file** + +```zig +const file = try std.fs.cwd().createFile("output.txt", .{}); +defer file.close(); + +var buffer: [4096]u8 = undefined; +var file_writer = file.writer(&buffer); +const writer = &file_writer.interface; + +try writer.writeAll("Content\n"); +try writer.flush(); // Ensure data written +``` + +**Pattern: Network socket I/O** + +```zig +const stream = try std.net.tcpConnectToHost(allocator, "example.com", 80); +defer stream.close(); + +var read_buf: [4096]u8 = undefined; +var reader = stream.reader(&read_buf); + +var write_buf: [4096]u8 = undefined; +var writer = stream.writer(&write_buf); + +try writer.interface.writeAll("GET / HTTP/1.0\r\n\r\n"); +try writer.interface.flush(); + +// Read response +while (reader.interface.takeDelimiterExclusive('\n')) |line| { + // Process response line +} else |err| if (err != error.EndOfStream) return err; +``` + +--- + +## What's coming in Zig 0.16.x + +The std.io changes in 0.15.x are **foundational for the return of async/await** +expected in 0.16.x or later releases. The roadmap includes: + +**Complete std.Io interface:** All blocking operations (file system, networking, +timers, synchronization) will require an `Io` instance, similar to how memory +allocation requires an `Allocator` instance. + +**Event loops as first-class citizens:** The new I/O abstraction enables +different concurrency models (async/await, thread pools, blocking I/O) to work +with the same library code. + +**Better testing and reliability:** I/O as an interface parameter enables +mocking, instrumentation, and resource leak detection. + +**Quote from Andrew Kelley:** "All code that performs I/O will need access to an +Io instance, similar to how all code that allocates memory needs access to an +Allocator instance." + +The 0.15.x changes, while painful, establish the architecture necessary for Zig +to support modern async I/O patterns without language-level async keywords. This +is a deliberate choice to keep async in the standard library rather than making +it a language feature. + +--- + +## Documentation resources for Tero policy-zig + +**Official sources:** + +- Zig 0.15.1 Release Notes: + https://ziglang.org/download/0.15.1/release-notes.html +- Standard Library Documentation: https://ziglang.org/documentation/0.15.2/std/ +- Andrew Kelley's "Don't Forget To Flush" talk (Systems Distributed 2025) + +**Community resources:** + +- "migrating to zig 0.15: the roadblocks nobody warned you about" (sngeth.com) +- "Zig 0.15.1 I/O Overhaul" (dev.to/bkataru) +- "I'm too dumb for Zig's new IO interface" (openmymind.net) - honest account of + migration difficulties +- "Inside Zig's New Writer" (joegm.github.io) - technical deep dive +- Ziggit.dev forum discussions on migration challenges + +**GitHub:** + +- Writergate PR #24329 with full technical rationale +- Active issue tracker for bugs and policy-zig cases + +--- + +## Migration checklist for Tero policy-zig + +For updating your claude.md documentation to prevent outdated patterns: + +- [ ] Document that all readers/writers require explicit buffer allocation +- [ ] Emphasize the mandatory flush requirement (most common bug) +- [ ] Warn about @fieldParentPtr footgun (never copy interfaces) +- [ ] Update all code examples from 0.14.x patterns to 0.15.x patterns +- [ ] Replace `std.io.GenericReader/Writer` with `std.Io.Reader/Writer` +- [ ] Replace `readUntilDelimiterOrEof()` with `takeDelimiterExclusive()` +- [ ] Document that std.io.SeekableStream is removed +- [ ] Update HTTP client/server examples with new multi-buffer patterns +- [ ] Note TLS client requires four buffers minimum +- [ ] Document format string changes ({} → {f} or {any}) +- [ ] Add examples showing proper error handling for StreamTooLong +- [ ] Include buffer sizing guidelines (4096 minimum, TLS requires + max_ciphertext_record_len) +- [ ] Show Writer.Allocating pattern for unbounded data +- [ ] Emphasize that compression functionality was temporarily removed + +This comprehensive reference should enable AI models to provide accurate, +current guidance for Zig 0.15.x I/O operations in the Tero policy-zig project. diff --git a/README.md b/README.md index 3b7d31a..366b1f5 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ src/ trace_sampler.zig # Trace sampling rate_limiter.zig # Rate limiting hyperscan.zig # Vectorscan/Hyperscan C bindings + extensions/ # Policy extensions (spec v1.6.0): s3-dump handler observability/ # Event bus, spans, formatters proto/ # Generated protobuf definitions ``` @@ -161,23 +162,83 @@ src/ This gives O(k \* n) performance where k = unique matcher keys and n = input text length, independent of policy count. +## Supported Extensions + +Per policy-spec conformance item 9, this implementation supports the following +extension `type`/`version` pairs (see `design/extensions.md` and the +`extensions` module): + +| type | versions | notes | +| --------------------- | -------- | ------------------------------------------------------------------------------------------ | +| `com.usetero/s3-dump` | 1.0.x | batched ndjson objects via z3; failed uploads retried in-memory (bounded), idempotent keys | + +Extensions declared by a policy but not supported (or with an unresolvable +target) are skipped fail-open and reported via `PolicySyncStatus.errors`; the +policy's core match/keep/transform still applies. + +Enabling extensions in a consumer (the `extensions` module owns the handlers +and adapts them to policy_zig's seams): + +```zig +const ext = @import("extensions"); + +// One Extensions object owns every enabled handler. Encoders are per-signal +// because the engine hands the sink an opaque record it can't serialize. +var exts = ext.Extensions.init(allocator, .{ .log = encodeLog }); +defer exts.deinit(); + +// Enable s3-dump and configure targets directly on the returned handle. +// Credentials come from the consumer (env/secrets) — never from policies. +const s3 = exts.enableS3Dump(allocator, .{}, .{ + .access_key_id = key, + .secret_access_key = secret, +}); +try s3.addTarget(io, "eu-bucket", + \\{"bucket": "tero-waste", "region": "eu-west-1", "prefix": "dumps/"} +); + +// One call wires the resolver (snapshot compile) + sync hooks (advertised on +// each provider). subscribe() then pushes the hooks to every provider — the +// HTTP provider advertises capabilities and receives broadcast targets; a +// file provider accepts and ignores them. +exts.register(®istry); +try registry.subscribe(.{ .http = http_provider }); + +// Per evaluate: pass the sink so selected records are delivered. +_ = engine.evaluate(.log, &accessor, &record, &buf, .{ + .io = io, + .extension_sink = exts.sink(), +}); + +// From a background loop: flush uploads and read the result as metrics. +const r = exts.flush(io, .{}); +// r.objects_uploaded / .records_dropped / .backlog_bytes → your metrics client +``` + +`com.usetero/s3-dump`'s upload path is verified two ways: `zig build test` +(covered by `task test`) includes a hermetic test against an in-process HTTP +stub — no network or docker needed — and `task test:s3-e2e` spins up a real +MinIO container via docker and round-trips an upload through it. + ## Task Commands This project uses [Task](https://taskfile.dev/) for common operations: -| Command | Description | -| -------------------- | ------------------------------------------------ | -| `task` | Build (debug) | -| `task test` | Run all tests | -| `task build:release` | Build with ReleaseFast | -| `task build:safe` | Build with ReleaseSafe | -| `task format` | Format source files | -| `task format:check` | Check formatting | -| `task lint` | Run linting checks | -| `task clean` | Clean build artifacts | -| `task do` | Run all pre-commit checks (format + lint + test) | -| `task signoff` | Full signoff (do + build:safe + gh signoff) | -| `task ci:setup` | Install CI/dev dependencies | +| Command | Description | +| -------------------- | --------------------------------------------------------------- | +| `task` | Build (debug) | +| `task test` | Run all tests | +| `task test:s3-e2e` | Real-storage s3-dump smoke test against MinIO (requires docker) | +| `task bench:s3` | s3-dump scale benchmarks against MinIO (requires docker) | +| `task build:release` | Build with ReleaseFast | +| `task build:safe` | Build with ReleaseSafe | +| `task format` | Format source files | +| `task format:check` | Check formatting | +| `task lint` | Run linting checks | +| `task clean` | Clean build artifacts | +| `task do` | Run all pre-commit checks (format + lint + test) | +| `task signoff` | Full signoff (do + build:safe + gh signoff) | +| `task ci:setup` | Install CI/dev dependencies | ## License diff --git a/Taskfile.yml b/Taskfile.yml index 97db170..5f57bb5 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -71,6 +71,78 @@ tasks: cmds: - zig build test --summary all + test:s3-e2e: + desc: "Real-storage smoke test for the s3-dump extension against a local MinIO container (requires docker)" + cmds: + - | + if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for task test:s3-e2e" + exit 1 + fi + CONTAINER=policy-zig-minio-e2e + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + docker run -d --rm --name "$CONTAINER" \ + -p 9000:9000 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data >/dev/null + trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1' EXIT + + echo "Waiting for MinIO..." + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; then + break + fi + if [ "$i" = 30 ]; then + echo "MinIO did not become healthy in time" + docker logs "$CONTAINER" || true + exit 1 + fi + sleep 1 + done + + AWS_ACCESS_KEY_ID=minioadmin \ + AWS_SECRET_ACCESS_KEY=minioadmin \ + S3_ENDPOINT=http://127.0.0.1:9000 \ + S3_BUCKET=policy-zig-e2e \ + zig build test-s3-e2e --summary all + + bench:s3: + desc: "s3-dump scale benchmarks against a local MinIO container (requires docker)" + cmds: + - | + if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for task bench:s3" + exit 1 + fi + CONTAINER=policy-zig-minio-bench + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + docker run -d --rm --name "$CONTAINER" \ + -p 9000:9000 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data >/dev/null + trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1' EXIT + + echo "Waiting for MinIO..." + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; then + break + fi + if [ "$i" = 30 ]; then + echo "MinIO did not become healthy in time" + docker logs "$CONTAINER" || true + exit 1 + fi + sleep 1 + done + + AWS_ACCESS_KEY_ID=minioadmin \ + AWS_SECRET_ACCESS_KEY=minioadmin \ + S3_ENDPOINT=http://127.0.0.1:9000 \ + S3_BUCKET=policy-zig-bench \ + zig build bench-s3 + format: desc: "Format all Zig source files" aliases: [fmt, f] diff --git a/build.zig b/build.zig index f030754..263fd28 100644 --- a/build.zig +++ b/build.zig @@ -44,6 +44,23 @@ pub fn build(b: *std.Build) void { mod.link_libc = true; mod.linkSystemLibrary("hs", .{}); + // Extensions module (spec v1.6.0): concrete extension handlers (s3-dump). + // Separate module so policy_zig stays free of I/O and of the z3 + // dependency; only consumers that enable extensions link it. + const z3_dep = b.dependency("z3", .{ + .target = target, + .optimize = optimize, + }); + const ext_mod = b.addModule("extensions", .{ + .root_source_file = b.path("src/extensions/root.zig"), + .target = target, + .imports = &.{ + .{ .name = "proto", .module = proto_mod }, + .{ .name = "policy_zig", .module = mod }, + .{ .name = "s3", .module = z3_dep.module("s3") }, + }, + }); + // Tests const mod_tests = b.addTest(.{ .root_module = mod, @@ -59,9 +76,60 @@ pub fn build(b: *std.Build) void { }); const run_o11y_tests = b.addRunArtifact(o11y_tests); + // Extensions module tests (need libc/hyperscan transitively via policy_zig) + const ext_tests = b.addTest(.{ + .root_module = ext_mod, + }); + ext_tests.root_module.link_libc = true; + ext_tests.root_module.linkSystemLibrary("hs", .{}); + const run_ext_tests = b.addRunArtifact(ext_tests); + const test_step = b.step("test", "Run tests"); test_step.dependOn(&run_mod_tests.step); test_step.dependOn(&run_o11y_tests.step); + test_step.dependOn(&run_ext_tests.step); + + // Real-storage s3-dump smoke test (needs a running S3-compatible + // server — see `task test:s3-e2e`). Deliberately NOT part of `test`: + // it requires network I/O and external state, unlike every other test. + const s3_e2e_tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("src/extensions/s3_minio_test.zig"), + .target = target, + .imports = &.{ + .{ .name = "proto", .module = proto_mod }, + .{ .name = "policy_zig", .module = mod }, + .{ .name = "s3", .module = z3_dep.module("s3") }, + }, + }), + }); + s3_e2e_tests.root_module.link_libc = true; + s3_e2e_tests.root_module.linkSystemLibrary("hs", .{}); + const run_s3_e2e_tests = b.addRunArtifact(s3_e2e_tests); + const s3_e2e_step = b.step("test-s3-e2e", "Run the s3-dump smoke test against a real S3-compatible server"); + s3_e2e_step.dependOn(&run_s3_e2e_tests.step); + + // s3-dump scale benchmarks (needs MinIO — see `task bench:s3`). + // ReleaseFast like the other benchmarks; excluded from `test`. A plain + // executable, not a test: it reports numbers, not pass/fail. + const s3_bench = b.addExecutable(.{ + .name = "bench-s3", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/extensions/s3_bench.zig"), + .target = target, + .optimize = .ReleaseFast, + .imports = &.{ + .{ .name = "proto", .module = proto_mod }, + .{ .name = "policy_zig", .module = mod }, + .{ .name = "s3", .module = z3_dep.module("s3") }, + }, + }), + }); + s3_bench.root_module.link_libc = true; + s3_bench.root_module.linkSystemLibrary("hs", .{}); + const run_s3_bench = b.addRunArtifact(s3_bench); + const s3_bench_step = b.step("bench-s3", "Run s3-dump scale benchmarks against a real S3-compatible server"); + s3_bench_step.dependOn(&run_s3_bench.step); // Benchmarks const zbench_dep = b.dependency("zbench", .{ .target = target, .optimize = .ReleaseFast }); @@ -96,6 +164,8 @@ pub fn build(b: *std.Build) void { .destination_directory = b.path("src/proto"), .source_files = &.{ b.path("proto/tero/policy/v1/policy.proto"), + b.path("proto/tero/policy/v1/extension.proto"), + b.path("proto/tero/policy/v1/tero_extensions.proto"), b.path("proto/tero/policy/v1/log.proto"), b.path("proto/tero/policy/v1/metric.proto"), b.path("proto/tero/policy/v1/trace.proto"), diff --git a/build.zig.zon b/build.zig.zon index fd71d56..44478c2 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -50,6 +50,10 @@ .url = "git+https://github.com/hendriknielaender/zBench.git#b2b89c475e3ef1bb2bd71255c80478a82d3e0ca8", .hash = "zbench-0.13.0-YTdc7xVBAQCCMC-IdLLuotBeiNNNm8k9Pi2V4VYqrwfI", }, + .z3 = .{ + .url = "git+https://codeberg.org/fellowtraveler/z3#83b1f8d6222188f39c09966b8572bc9168813d04", + .hash = "z3-0.4.1-N25-cH-fAQAbj6ABBY0Bq85Kv-d7sdIvANgLCE9NCaEF", + }, }, // Specifies the set of files and directories that are included in this package. // Only files and directories listed here are included in the `hash` that diff --git a/design/extensions.md b/design/extensions.md new file mode 100644 index 0000000..7101df9 --- /dev/null +++ b/design/extensions.md @@ -0,0 +1,562 @@ +# Extensions package + `com.usetero/s3-dump` + +Design for implementing policy-spec v1.6.0 extensions in policy-zig, and the +first concrete extension: dumping telemetry to S3-compatible storage. + +Status: **implemented** (see `src/extensions/`, engine dispatch in +`src/policy/policy_engine.zig`, bindings in `src/policy/matcher_index.zig`). +Deviations from the text below, made during implementation: + +- The resolver seam passes `io` first (`resolve(io, ctx, signal, policy_id, + ext)`) per ziglint's io-ordering rule, and resolution returns an opaque + `{handler, slot}` pair — the slot is the s3-dump batch index, assigned at + compile time so dispatch is an array lookup. +- Per-signal `encode` callbacks are registered once on `Extensions.init` + rather than carried in each `DeliveredRecord` (the engine passes only + `(signal, record ctx)` across the module seam). +- Credentials are consumer-provided at `S3Dump.init` (no built-in env + discovery — z3 has none either, and the edge binary owns its env). +- Extension sync hooks (a two-fn seam like `StatsCollector`) live in the + generic provider layer (`provider.zig`); `Provider.setExtensionSyncHooks` + dispatches per variant (`.http` wires, `.file`/`.testing` no-op), and + `registry.subscribe` pushes them to every provider automatically — the + consumer wires them once via `registry.setExtensionSyncHooks` (or + `Extensions.register`), not per provider. +- `Extensions.enableS3Dump(alloc, opts, creds) *S3Dump` constructs the + handler in place and returns its pointer, so targets are configured on the + handler directly (no move, no `s3Dump().?` reach-back). +- `Extensions.register(®istry)` wires resolver + sync hooks in one call. +- All batching knobs live in `S3Dump.Options`, a plain-data struct with + defaults, deserializable from a config file. + +## What the spec requires (v1.6.0) + +From `spec.md` §Extensions and `extension.proto` / `tero_extensions.proto`: + +- A policy MAY carry `extensions: [{type, version, config, mode}]`. `type` is + reverse-FQDN (`com.usetero/s3-dump`), `version` is semver, `config` is + **opaque bytes** the engine never interprets, `mode` selects a traffic slice. +- Pipeline order is `Match → Keep → Extension dispatch → Transform`. Extensions + receive **pre-transform** records and MUST NOT change the keep or transform + outcome. The keep verdict used for classification is the record's **final + pipeline outcome** across all policies (that's what makes `mode: dropped` on + a `keep: .01%` policy receive the ~99.99% waste). +- Modes, relative to the extension's policy: `kept`, `dropped`, `unmatched` + (disjoint partition), `matched` = kept+dropped (default), `all` = everything. +- Handshake: the client advertises `ClientMetadata.supported_extensions` + (`ExtensionCapability{type, min_version, repeated bytes config}`); the + provider MAY broadcast `SyncResponse.extension_configs` + (`ExtensionConfig{type, repeated bytes config}`). For s3-dump the capability + descriptors are serialized `ExtensionTargetRef`s and the broadcast entries + are serialized `ExtensionTarget`s. +- Rules: fail-open (an unsupported/unsatisfiable extension is skipped; core + match/keep/transform still applies; SHOULD reject the policy only when the + extension is *required* for its behavior — never true for a dump), execute + off the hot path with non-blocking operations, and document supported + `type`/`version` pairs (conformance item 9). +- `ExtensionTarget{kind, name, config}` is a pre-configured named destination; + policies reference it by `ExtensionTargetRef{kind, name}` and never carry + credentials or connection details inline. + +## Goals / non-goals + +**v1 goals** + +- Generic extension mechanism: handler tagged union, engine dispatch hook, + compile-stage validation, capability advertisement, config broadcast + handling. +- One extension: `com.usetero/s3-dump` v1.0.0, all three signals, ndjson + objects, S3-compatible endpoints (AWS, MinIO, R2) via the + [z3](https://codeberg.org/fellowtraveler/z3) client. + +**Non-goals (v1)** + +- Columnar object formats (OTel Arrow, Vortex, Parquet) — see + [Data format](#data-format-edge-native-rows-not-columnar); gzip; multipart + upload (z3 supports it if batches ever need it); upload retry queues. +- Post-transform delivery (spec leaves it type-defined; we don't need it). +- Any second extension type. The mechanism is generic but we build exactly one. + +## Prerequisite: proto update + +We deliberately skipped the extension protos when updating to v1.6.0. First +step is finishing that: + +1. Vendor `extension.proto`, `tero_extensions.proto`, and the v1.6.0 + `policy.proto` (adds `Policy.extensions = 20`, + `ClientMetadata.supported_extensions = 4`, + `SyncResponse.extension_configs = 7`) into `proto/tero/policy/v1/`. +2. `zig build -Dgen-proto=true gen-proto`. + +No engine behavior changes from this alone — unknown fields were already +ignored on the wire. + +## Package layout + +A new module `extensions`, mirroring how `observability` is wired: + +``` +src/extensions/ + root.zig // ExtensionHandler tagged union, dispatch fn, re-exports + s3_dump.zig // com.usetero/s3-dump handler +``` + +One new dependency in `build.zig.zon`: +[z3](https://codeberg.org/fellowtraveler/z3) — pure-Zig async S3 client (MIT, +min Zig 0.16, zero deps beyond std). It covers SigV4 signing, path- and +virtual-host-style requests, S3-compatible endpoints (tested against R2), and +multipart uploads if batches ever outgrow single PUTs. This replaces the +hand-rolled `sigv4.zig` + `std.http` plumbing from an earlier draft. Only the +`extensions` module imports it — `policy_zig` stays dependency-free. + +`build.zig`: + +```zig +const ext_mod = b.addModule("extensions", .{ + .root_source_file = b.path("src/extensions/root.zig"), + .imports = &.{ proto, policy_zig, observability }, +}); +``` + +Rationale for a separate module rather than folding into `policy_zig`: the +engine stays free of I/O and of any knowledge of concrete extension types +(spec: config is opaque to the engine), and consumers that don't use +extensions link nothing new. + +## Core mechanism + +### 1. Engine dispatch hook (`policy_zig` change — the only one) + +Dispatch must happen *inside* `evaluate`: after the final keep decision is +known but **before transforms run** (spec ordering). Post-`evaluate` dispatch +by the consumer would see redacted records, so that's out. + +Add an optional sink to the per-call evaluate options, alongside `io`: + +```zig +/// One function pointer + ctx — the same shape as the accessor primitives, +/// NOT a vtable (Claude.md: encoding over polymorphism; the fn pointer here +/// exists only to break the module cycle — policy_zig cannot import the +/// extensions module that implements it). All real dispatch happens behind +/// it via the ExtensionHandler tagged union. +pub const ExtensionSink = struct { + ctx: *anyopaque, + /// Called once per (record, policy-with-extensions) pair whose mode + /// selects this record, between keep resolution and transforms. + deliver: *const fn ( + ctx: *anyopaque, + io: ?std.Io, + telemetry: DeliveredRecord, // signal tag + consumer ctx + encode fn + binding: *const ExtensionBinding, // policy index + resolved handler tag + config + mode + slice: SliceTag, // kept | dropped | unmatched + ) void, +}; +``` + +- `evaluate(..., .{ .io = io, .extension_sink = null })` — a null sink is a + single branch; zero cost for existing consumers. +- The engine classifies per binding using state it already has: + `matched(policy)` (from `MatchState`) and the final decision. `kept` = + matched ∧ keep, `dropped` = matched ∧ drop, `unmatched` = ¬matched; + `matched`/`all` are unions resolved at compile time into "which slices fire". +- `UNMATCHED`/`ALL` bindings must be visited even when the record matched no + policy, so the binding list is iterated unconditionally — but only when the + snapshot has any bindings (one `bindings.len == 0` check keeps the common + case free). +- Delivery passes the consumer `ctx` + accessor-adjacent `encode` fn (below), + not bytes: the engine has no serialized form of the record. + +### 2. Compile-stage bindings and validation + +Per the "validate during compilation" convention, `IndexBuilder` grows an +extensions pass: + +- Collect `ExtensionBinding{policy_index, handler: ExtensionTag, mode, config}` + for every enabled policy into the snapshot (plain slice, iterated at + dispatch). The `type` string is resolved to the `ExtensionTag` enum **once, + here** — dispatch never compares strings (Claude.md: encode, don't point). +- Validation, fail-open per spec rule 5: an extension whose `type` has no + registered handler, whose `version` exceeds the handler's supported range, + or whose `config` the handler rejects (e.g. unknown target ref) is **skipped + and reported** via the existing `recordError` path → `PolicySyncStatus.errors`. + The policy itself stays valid — a dump extension is never "required for the + policy's behavior". +- Handlers get a `validateConfig(config: []const u8) error{...}!void` chance at + compile time so bad configs surface at sync, not at dispatch. + +### 3. Handlers: tagged union, not a vtable + +Extension types are a closed set the implementation must declare anyway (spec +rule 4), so runtime polymorphism buys nothing. Per Claude.md §"Encoding +Instead of Polymorphism", handlers are a tagged union with switch dispatch: + +```zig +pub const ExtensionTag = enum { s3_dump }; + +pub const ExtensionHandler = union(ExtensionTag) { + s3_dump: S3Dump, + + pub fn typeId(tag: ExtensionTag) []const u8 { + return switch (tag) { .s3_dump => "com.usetero/s3-dump" }; + } + + pub fn validateConfig(self: *const ExtensionHandler, config: []const u8) !void { + switch (self.*) { .s3_dump => |*h| try h.validateConfig(config) } + } + + /// Hot-path entry. MUST be non-blocking: append to a batch and return. + pub fn deliver(self: *ExtensionHandler, rec: DeliveredRecord, config: []const u8, slice: SliceTag) void { + switch (self.*) { .s3_dump => |*h| h.deliver(rec, config, slice) } + } + + /// SyncResponse.extension_configs entries for this type. + pub fn configure(self: *ExtensionHandler, entries: []const []const u8) void { ... } + + /// Capability descriptors for ClientMetadata.supported_extensions. + pub fn capabilities(self: *const ExtensionHandler, allocator: Allocator) ![]const []const u8 { ... } +}; +``` + +There is no registry object: the consumer constructs an +`Extensions = struct { handlers: std.EnumArray(ExtensionTag, ?ExtensionHandler) }` +with the handlers it enables, and the dispatch path is an array index by the +binding's pre-resolved tag — no string lookup, no pointer chase. Adding a +second extension type = one new enum tag + one new union arm; the compiler +enforces exhaustiveness everywhere. + +The only fn pointers in the design are the engine-side `ExtensionSink` +(one function, to break the policy_zig ↔ extensions module cycle — same +pattern as accessors) and the consumer's `encode` callback (unavoidable: the +record type is the consumer's). + +### 4. Record encoding — consumer contract + +The engine reads records through accessors and never owns bytes, so a dump +extension cannot serialize a record by itself. `DeliveredRecord` therefore +carries a consumer-wired encoder: + +```zig +pub const DeliveredRecord = struct { + signal: TelemetryType, + /// Borrowed. Valid ONLY for the duration of the deliver() call. + ctx: *const anyopaque, + /// Render the record (pre-transform) to bytes, e.g. OTLP JSON. + encode: *const fn (ctx: *const anyopaque, writer: *std.Io.Writer) anyerror!void, +}; +``` + +Consumers that already hold OTLP protobuf can emit it directly; others emit +whatever textual form they want in the object. The library does not prescribe +the record schema inside the dump — the extension prescribes the *container* +(ndjson: one encoded record per line). + +### Record lifetime and hot-path cost — copy at delivery, never hold + +This is the load-bearing invariant of the whole design: + +**Handlers encode synchronously inside `deliver()` and own only the encoded +bytes.** The batch buffer receives a self-contained ndjson line; the moment +`deliver()` returns, no extension holds any reference to the consumer's +record. Record memory has exactly the lifetime it has today — free/reuse it +when `evaluate` returns. Nothing is pinned until flush. + +Deferring the copy isn't an optimization we declined — it's impossible to do +correctly: + +1. **The spec requires pre-transform delivery**, and the engine applies + transforms (redact/remove/rename) *in place, immediately after dispatch, + in the same `evaluate` call*. A handler that stashed a reference and read + it at flush time would observe redacted/renamed data. +2. Holding references until flush (up to `max_batch_age` = 30 s) would pin + entire OTLP batches in memory on the edge — the exact memory-pressure + failure this design exists to avoid. + +This is also how we satisfy the spec's "dispatches copies" wording: the copy +is the encoded line; borrowing `ctx` for the duration of one synchronous call +is just how the copy gets made without a second intermediate allocation. + +What the hot path actually pays, worst to best case: + +| situation | cost per record | +| -------------------------------------- | ----------------------------------------- | +| no sink / no bindings in snapshot | one null/len check | +| bindings exist, no mode selects record | slice classification (a few compares) | +| record selected | one `encode` + append into batch buffer | +| record selected, sealed backlog full | capacity check only — **encode skipped**, drop counted | + +The selected-record cost is a bounded, allocation-free serialize (µs-scale +for a typical log record) on the processing thread — a memcpy-class +operation, never I/O, never a lock held across I/O. The capacity check runs +*before* encoding, so a destination outage degrades to a counter increment, +not wasted serialization. And for the flagship `mode: dropped` use, the +records paying the encode are ones the pipeline was discarding anyway. + +If that encode ever shows up in a profile, the knobs are consumer-side: a +cheaper `encode` (e.g. memcpy of pre-serialized bytes the consumer already +has), or narrower modes/matchers so fewer records are selected. A zero-copy +"raw bytes" fast path on `DeliveredRecord` is possible later without changing +this invariant. + +### 5. Sync plumbing (`provider_http`) + +- **Advertise**: when building `ClientMetadata`, iterate the enabled handlers + in the `Extensions` enum array, ask each for `capabilities()`, and fill + `supported_extensions` with + `{type, min_version, config}`. For s3-dump each descriptor is a serialized + `ExtensionTargetRef` for every target the client can currently reach. +- **Receive**: on `SyncResponse`, route each `extension_configs` entry to + `handler.configure(entries)`. For s3-dump the entries are serialized + `ExtensionTarget`s, merged with locally configured targets (broadcast wins + on `(kind, name)` collision; spec says clients merge). + +## `com.usetero/s3-dump` v1.0.0 + +### Policy-facing config + +Exactly what the spec's example shows — a target reference, nothing else: + +```yaml +extensions: + - type: com.usetero/s3-dump + version: 1.0.0 + mode: dropped + config: + target: { kind: s3, name: eu-bucket } +``` + +`config` bytes = serialized `ExtensionTargetRef`. `validateConfig` decodes it +and checks the target exists (locally configured or broadcast); unknown target +→ skip + sync error, per fail-open. + +### Target config (`ExtensionTarget.config` for kind `s3`) + +The spec leaves this kind-defined. Define it as JSON (matches the rest of the +repo's config surface and avoids another proto): + +```json +{ + "endpoint": "https://s3.eu-west-1.amazonaws.com", // any S3-compatible URL + "region": "eu-west-1", + "bucket": "tero-waste", + "prefix": "dumps/", + "force_path_style": false +} +``` + +**Credentials are never in the target or the policy.** The consumer supplies a +credentials callback at handler construction +(`fn () ?Credentials{access_key, secret_key, session_token}`); the default +reads `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`/`AWS_SESSION_TOKEN`. No +credentials → deliveries for that target are counted-and-dropped (fail-open). + +### Batching — the core of the handler + +Records are never uploaded individually. One S3 PUT per record would melt both +the pipeline and the S3 bill; the unit of upload is a **batch**, and the hot +path's only job is appending to one (spec rule 6: off the hot path, +non-blocking). + +**Batch key**: `(target, signal, policy_id)`. Signal and policy separate +naturally in storage (waste analysis is per-policy), and the object key below +encodes both. The number of batches is bounded by +`configured targets × 3 signals × policies-with-s3-dump` — small, known at +snapshot compile time. + +**Hot path** (`deliver()`): find the open batch — bindings are compiled per +snapshot, so each binding carries its batch index; array lookup, no hashing +per record — check capacity **first** (full backlog → count the drop and +return, no encode), then run the consumer `encode` into the batch's append +buffer, write the `\n`, bump the record count. Under a per-batch mutex; no +I/O, no allocation beyond the batch buffer. See +[Record lifetime](#record-lifetime-and-hot-path-cost--copy-at-delivery-never-hold) +for why the encode happens here and not later. + +**Batch limits** (defaults, overridable at handler construction): + +| limit | default | on hit | +| ------------------- | ------- | ---------------------------------------- | +| `max_batch_bytes` | 4 MiB | batch is sealed, ready for flush | +| `max_batch_records` | 10 000 | batch is sealed, ready for flush | +| `max_batch_age` | 30 s | checked in `flush()`, seals stale batches | +| `max_sealed_bytes` | 32 MiB | total sealed backlog — new records **drop** | + +A record arriving when the sealed backlog is at `max_sealed_bytes` is dropped +and counted (`records_dropped` on the event bus). Backpressure never +propagates to telemetry processing — a dump destination outage costs waste +records, never pipeline latency. + +**Flush** — the consumer drives I/O, per this repo's io-threading rule +(`std.Io` passed per call, never stored): `flush(io: std.Io) FlushResult` +seals any over-age open batches (age from `io.now`), uploads each sealed +batch as one object, and releases its buffer. The consumer calls it from a +background task/timer; the handler never spawns threads or sleeps. +Double-buffering (open batch swaps onto the sealed list) keeps `deliver()` +appending while `flush` uploads. + +### Data format: edge-native rows, not columnar + +Considered: dump in whatever form the edge already has (consumer-encoded +rows), vs. re-encoding into a columnar format (OTel Arrow / OTAP, Vortex, +Parquet). Decision: **edge-native rows at the edge; columnar belongs in a +downstream compaction job, if ever.** + +- **No Zig implementation exists** for Arrow, OTAP, or Vortex. Adopting one + means FFI to a Rust/C++ library or writing a columnar encoder — either + dwarfs the rest of this design for a waste stream that is written often and + read rarely. +- **Columnar encoding is a batch-side workload.** Good compression needs large + row groups, dictionary state, and schema unification across records — CPU + and memory spent on the edge, on the hot-adjacent path, for exactly the + records the policy decided were not worth keeping. The edge's job is to get + bytes out cheaply; fail-open dumping must stay near-free. +- **Rows rehydrate; columns analyze.** The primary consumer of a waste dump is + "replay these records back into the pipeline" (incident retro, sampling + regret). Consumer-encoded OTLP rows replay directly. If cheap analytics over + the dumps becomes real, run a server-side compaction job + (S3 → Parquet/Vortex) where cores are cheap and batches are huge — the + standard lake pattern, and it needs zero changes at the edge. +- The consumer `encode` callback already exists because the engine reads + records through accessors and owns no serialized form. A columnar format + would force a much fatter contract (full typed schema per record) on every + consumer. + +So the object body is rows in whatever form the edge is working with — +recommended `encode` output is OTLP JSON, but the container doesn't care. + +### Object format and key + +One object per batch: + +- Body: ndjson — each line is one record as produced by the consumer `encode`. +- Key: `{prefix}{signal}/{yyyy}/{mm}/{dd}/{hh}/{policy_id}-{unix_nanos}-{seq}.ndjson` + Time from `io.now` at flush; `seq` is a per-process counter (no uuid dep). +- `Content-Type: application/x-ndjson`. + +### Upload + +- One `z3` `putObject(bucket, key, body, options)` per sealed batch, over the + flush-supplied `io`. z3 handles SigV4, path- vs virtual-host-style (our + `force_path_style` maps directly), and custom endpoints. The sealed batch is + a contiguous buffer, so content length and payload hash are trivially + available (z3's no-chunked-signing limitation doesn't bite us). +- z3 does not do credential discovery — fine, because our design already + sources credentials from the consumer callback / env (see above) and passes + them at client init. One z3 client per target per flush (lazily created), + so std.http.Client's connection pool is reused across that flush's uploads. +- Target configs are snapshotted into a flush-local arena under the mutex; + uploads never read live target state (a concurrent broadcast `configure` + can therefore never invalidate memory mid-upload). +- Failure handling: z3's `max_attempts` only retries connection + establishment — send/receive failures and 5xx responses are not retried by + the client. So a failed upload is **requeued** onto the sealed backlog and + retried on subsequent flushes, bounded by the same `max_sealed_bytes` cap + (overflow drops, counted). The object key is fixed at seal time, so + retries are idempotent same-key PUTs — a request that succeeded + server-side before the failure surfaced overwrites itself instead of + duplicating. Records are lost only when the backlog cap overflows; there + is still no unbounded queue and no disk spill. + + +### Capability advertisement + +`capabilities()` returns one serialized `ExtensionTargetRef{kind: "s3", name}` +per configured target, so the provider only sends policies whose extensions +this client can satisfy. + +## Scale characteristics (measured) + +`task bench:s3` runs scale benchmarks against a local MinIO container +(ReleaseFast, 191-byte records, Apple Silicon dev machine — treat as shape, +not absolutes): + +| scenario | result | +| --- | --- | +| deliver, 1 thread | ~16M records/s (~2.9 GiB/s) — ~62ns/record | +| deliver, 4 threads | ~11.5M records/s aggregate — **slower than 1 thread** | +| flush, 64KiB objects | ~250–320 objects/s, ~16–20 MiB/s | +| flush, 1MiB objects | ~100 objects/s, ~100 MiB/s | +| flush, 4MiB objects | ~76–80 objects/s, ~135–142 MiB/s | +| fan-out, 64 policies × 128KiB | ~260 objects/s (one flush ≈ 0.25s) | +| outage flush, max_attempts=1 | ~1ms (connect-refused + requeue is free) | +| outage flush, max_attempts=3 | ~2.3–2.8s for 8 objects (~290ms/object of retry sleeps) | + +What this says about scale: + +1. **The hot path is not the bottleneck.** Deliver costs ~62ns/record — + comparable to a whole engine evaluation (~40–60ns) and paid only by + records a mode selects. +2. **The single dump-wide mutex is a real ceiling under concurrent + delivery**: 4 threads are ~30% *slower* in aggregate than 1. Irrelevant at + realistic waste rates (1M records/s is ~6% of the ceiling), but per-batch + mutexes are the upgrade path if a many-threaded consumer attaches + `mode: all` extensions to high-volume signals. +3. **Object size dominates upload throughput** (~3–4ms fixed cost per PUT: + SigV4 + HTTP round trip). 64KiB objects move ~16 MiB/s; 4MiB objects move + ~140 MiB/s. Keep `max_batch_bytes` ≥ 1MiB and avoid fragmenting batches + across very many (policy × signal × target) combinations. Uploads within + a flush are serial; hundreds of batches per flush cycle means seconds of + flush latency. +4. **z3's in-request connect retries are redundant with our requeue** and + serialize ~100ms sleeps per object into the flush during outages — + hence `max_attempts` defaults to 1: a failed batch just waits for the + next flush instead of stalling the current one. + +## Conformance note + +README gains a "Supported extensions" table — required by conformance item 9: + +| type | versions | notes | +| --------------------- | -------- | ---------------------------------- | +| `com.usetero/s3-dump` | 1.0.x | ndjson objects, z3 client, no retries | + +## Testing + +- **Classification**: engine-level table tests — one policy per mode × record + {matched-kept, matched-dropped, unmatched}, assert exactly the spec's slice + reaches a recording fake sink; assert transforms observe untouched records + after a sink that mutates nothing; assert final decision is identical with + and without a sink (MUST NOT change outcome). +- **Cross-policy drop**: policy A `keep: none`, policy B `keep: all` with + `mode: dropped` extension — B's extension receives the record (final-outcome + semantics). +- **Validation**: unknown type / bad version / unknown target → policy still + active, extension skipped, error string in `PolicySyncStatus.errors`. +- **Upload (hermetic)**: `src/extensions/s3_stub_test.zig` runs flush against + an in-process `std.http.Server` stub over a real socket — asserts method, + path-style URL/key layout, the SigV4 auth header, the payload-sha256 header + S3 uses for integrity verification, and exact ndjson body bytes; a second + test scripts a 500 response and asserts the batch is requeued under the + identical object key and the retry succeeds. Runs in every + `zig build test`, no docker or network required. +- **Upload (real backend)**: `src/extensions/s3_minio_test.zig`, built as the + separate `zig build test-s3-e2e` step (excluded from `test` — needs a live + server) and driven end-to-end by `task test:s3-e2e`, which starts a MinIO + container via docker, waits for its health check, runs the test, and tears + the container down (`trap ... EXIT`). Skips itself (`error.SkipZigTest`) if + the AWS_* / S3_ENDPOINT env vars aren't set, so it's inert when built + without the task. Verifies against the real backend's own APIs — lists the + uploaded prefix, fetches the object, and diffs its body byte-for-byte — + rather than trusting our own response parsing. + +## Rollout + +1. Vendor v1.6.0 extension protos + regen (no behavior change). +2. `extensions` module: `ExtensionHandler` tagged union, sink, binding + compilation + validation in `IndexBuilder`, engine dispatch hook. + Fake-sink tests. +3. Vendor the z3 dependency; smoke-test `putObject` against MinIO. +4. `s3_dump.zig`: config/target parsing, batching, flush/upload, stub tests. +5. `provider_http`: capability advertisement + `extension_configs` routing. +6. README conformance table + consumer wiring example. + +Each step lands independently; 1–2 are useful on their own (dispatch mechanism +with no handlers registered is inert). + +## Open questions + +1. **`mode: all`/`unmatched` cost**: these force sink calls (and encodes) for + every record of the signal. Fine mechanically, but do we want a + per-snapshot cap or a warning event when such a policy is loaded? +2. **S3 target JSON vs proto**: JSON chosen above for authoring ergonomics; + if the tero backend already defines an S3 target proto, use that instead — + the bytes are opaque to everything but the handler either way. diff --git a/proto/tero/policy/v1/extension.proto b/proto/tero/policy/v1/extension.proto new file mode 100644 index 0000000..809a766 --- /dev/null +++ b/proto/tero/policy/v1/extension.proto @@ -0,0 +1,96 @@ +syntax = "proto3"; + +package tero.policy.v1; + +option go_package = "github.com/usetero/policy/gen/go/tero/policy/v1"; + +// ============================================================================= +// Extensions +// ============================================================================= +// +// Extensions are a fully generic mechanism for attaching implementation-specific +// behavior to a policy. The policy engine matches telemetry using the core +// target (log/metric/trace), computes the final keep outcome, then dispatches +// selected records to the handler registered for the extension `type` before +// transforms run. All extension payloads are opaque bytes: the engine never +// interprets them, so any implementation can define its own extension types +// without changing the core schema. +// +// tero's own extension types (e.g. the s3-dump destination) live in +// tero_extensions.proto. + +// ExtensionMode selects which slice of the telemetry stream the engine delivers +// to an extension, relative to its policy. {KEPT, DROPPED, UNMATCHED} are +// disjoint and partition the stream; MATCHED and ALL are convenience unions. +enum ExtensionMode { + EXTENSION_MODE_UNSPECIFIED = 0; // treated as MATCHED + + // Matched the policy and survived the keep stage. + EXTENSION_MODE_KEPT = 1; + + // Matched the policy but was removed by the keep stage (dropped, sampled out, + // or rate limited). Its keep verdict is the record's final pipeline outcome, + // which another matching policy may have caused. This is the "dump waste" mode. + EXTENSION_MODE_DROPPED = 2; + + // Did not match the policy's matchers. + EXTENSION_MODE_UNMATCHED = 3; + + // Matched the policy, regardless of keep outcome (KEPT + DROPPED). Default. + EXTENSION_MODE_MATCHED = 4; + + // Every record of the signal type, regardless of match or keep + // (KEPT + DROPPED + UNMATCHED). + EXTENSION_MODE_ALL = 5; +} + +// Extension attaches implementation-specific behavior to a policy. +// +// Implementations MUST declare which extension types they support. An extension +// with an unrecognized type is handled per the spec (fail-open, or reject policy +// load when the extension is required for the policy's behavior). +message Extension { + // Type identifier using reverse-FQDN notation (e.g. "com.usetero/s3-dump"). + // Implementations use this to route to the correct handler. + string type = 1; + + // Version of the extension schema (semver, e.g. "1.0.0"). + string version = 2; + + // Opaque, type-defined configuration payload, deserialized by the handler + // registered for `type`. The policy engine does not interpret it. + bytes config = 3; + + // Which slice of traffic the engine delivers to this extension. Defaults to + // MATCHED. + ExtensionMode mode = 4; +} + +// ExtensionCapability declares a client's support for one extension type. The +// repeated `config` entries are opaque, type-defined capability descriptors — +// for example, the destinations the client can route to. The policy engine does +// not interpret them; the extension handler does. Providers use this to avoid +// sending policies whose extensions the client cannot satisfy. +message ExtensionCapability { + // Extension type identifier (e.g. "com.usetero/s3-dump"). + string type = 1; + + // Minimum supported extension version (semver, inclusive). + string min_version = 2; + + // Opaque, type-defined capability descriptors. For tero's s3-dump these are + // serialized ExtensionTargetRefs (see tero_extensions.proto). + repeated bytes config = 3; +} + +// ExtensionConfig broadcasts opaque, type-defined configuration for one +// extension type to clients (via SyncResponse.extension_configs). Each `config` +// entry is parsed by the handler registered for `type`. For tero's s3-dump +// these are serialized ExtensionTargets (see tero_extensions.proto). +message ExtensionConfig { + // Extension type identifier (e.g. "com.usetero/s3-dump"). + string type = 1; + + // Opaque, type-defined configuration payloads. + repeated bytes config = 2; +} diff --git a/proto/tero/policy/v1/policy.proto b/proto/tero/policy/v1/policy.proto index 02d8d8a..b1ddbca 100644 --- a/proto/tero/policy/v1/policy.proto +++ b/proto/tero/policy/v1/policy.proto @@ -4,6 +4,7 @@ package tero.policy.v1; import "google/api/annotations.proto"; import "opentelemetry/proto/common/v1/common.proto"; +import "tero/policy/v1/extension.proto"; import "tero/policy/v1/log.proto"; import "tero/policy/v1/metric.proto"; import "tero/policy/v1/trace.proto"; @@ -49,6 +50,10 @@ message Policy { MetricTarget metric = 11; TraceTarget trace = 12; } + + // Implementation-specific extensions attached to this policy. Each extension + // references a pre-configured ExtensionTarget by (kind, name). + repeated Extension extensions = 20; } // ============================================================================= @@ -83,6 +88,11 @@ message ClientMetadata { // * service.namespace // * service.version repeated opentelemetry.proto.common.v1.KeyValue resource_attributes = 3; + + // Extension types this client supports, and the target kinds/names it can + // route each type to. Providers SHOULD only send policies whose extensions + // reference targets advertised here. + repeated ExtensionCapability supported_extensions = 4; } // TransformStageStatus reports hits and misses for a single transform stage. @@ -160,6 +170,13 @@ message SyncResponse { // Error message if sync failed string error_message = 6; + + // Opaque, type-defined extension configuration broadcast to the client, keyed + // by extension type. The policy engine does not interpret it; each extension + // handler parses its own entries. For tero's s3-dump this carries the + // available ExtensionTargets. A client merges these with any locally + // configured extension state. + repeated ExtensionConfig extension_configs = 7; } // ============================================================================= diff --git a/proto/tero/policy/v1/tero_extensions.proto b/proto/tero/policy/v1/tero_extensions.proto new file mode 100644 index 0000000..2df6401 --- /dev/null +++ b/proto/tero/policy/v1/tero_extensions.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package tero.policy.v1; + +option go_package = "github.com/usetero/policy/gen/go/tero/policy/v1"; + +// ============================================================================= +// tero Extension Types +// ============================================================================= +// +// Concrete extension types defined by tero, carried inside the opaque `config` +// bytes of the generic extension messages in extension.proto. These are not +// part of the core policy mechanism — other implementations define their own. + +// ExtensionTarget is a pre-configured, named destination an extension can route +// telemetry to (for example, an S3 bucket or a downstream OTLP endpoint). +// +// Targets are configured out of band on the implementation, or broadcast to +// clients via SyncResponse.extension_configs (serialized into the s3-dump +// extension's config). Policies reference a target by (kind, name) via +// ExtensionTargetRef and never carry its configuration inline. +message ExtensionTarget { + // Destination kind, e.g. "s3" or "otlp". + string kind = 1; + + // Name, unique within a kind, e.g. "eu-bucket". Used as the reference key. + string name = 2; + + // Opaque, kind-defined configuration (e.g. region, bucket, prefix). + bytes config = 3; +} + +// ExtensionTargetRef references a pre-configured ExtensionTarget by identity. +// Carried inside an Extension's config (to name where it routes) and inside an +// ExtensionCapability's config (to advertise reachable destinations). The +// referenced target MUST be known to the implementation — locally configured or +// received via SyncResponse.extension_configs. +message ExtensionTargetRef { + string kind = 1; + string name = 2; +} diff --git a/src/extensions/root.zig b/src/extensions/root.zig new file mode 100644 index 0000000..1912133 --- /dev/null +++ b/src/extensions/root.zig @@ -0,0 +1,450 @@ +//! Policy extensions (spec v1.6.0): concrete extension handlers and the +//! adapters that plug them into policy_zig's resolver/sink seams. +//! +//! Handlers are a tagged union, not a vtable (see Claude.md "Encoding Instead +//! of Polymorphism") — extension types are a closed set the implementation +//! must declare anyway (spec rule 4). Adding a type = one enum tag + one +//! union arm; the compiler enforces exhaustiveness everywhere. + +const std = @import("std"); +const policy = @import("policy_zig"); +const proto = @import("proto"); + +const s3_dump_mod = @import("./s3_dump.zig"); +pub const S3Dump = s3_dump_mod.S3Dump; +pub const EncodeFn = s3_dump_mod.EncodeFn; +pub const TeroS3DumpTag = "com.usetero/s3-dump"; + +pub const ExtensionTag = enum(u8) { + s3_dump, + + pub fn typeId(tag: ExtensionTag) []const u8 { + return switch (tag) { + .s3_dump => TeroS3DumpTag, + }; + } + + /// Highest supported major version per type (semver gate: an extension + /// declaring a different major is unsupported). + pub fn supportedMajor(tag: ExtensionTag) usize { + return switch (tag) { + .s3_dump => 1, + }; + } + + fn fromTypeId(type_id: []const u8) ?ExtensionTag { + inline for (comptime std.enums.values(ExtensionTag)) |tag| { + if (std.mem.eql(u8, type_id, comptime tag.typeId())) return tag; + } + return null; + } +}; + +pub const ExtensionHandler = union(ExtensionTag) { + s3_dump: S3Dump, + + pub fn deinit(self: *ExtensionHandler) void { + switch (self.*) { + .s3_dump => |*h| h.deinit(), + } + } +}; + +/// Owns the enabled handlers and adapts them to policy_zig's fn-pointer +/// seams (`ExtensionResolver` at snapshot compile time, `ExtensionSink` at +/// evaluate time). Encoders are wired per signal at construction because the +/// record types behind `*const anyopaque` are the consumer's. +/// +/// Usage: +/// var exts = Extensions.init(allocator, .{ .log = encodeLog }); +/// defer exts.deinit(); +/// const s3 = exts.enableS3Dump(allocator, .{}, creds); +/// try s3.addTarget(io, "eu-bucket", target_json); +/// exts.register(®istry); // resolver + sync hooks, once +/// try registry.subscribe(.{ .http = http_provider }); // hooks auto-pushed +/// // per evaluate call: .{ .io = io, .extension_sink = exts.sink() } +/// // periodically: exts.flush(io, .{}); +pub const Extensions = struct { + pub const Encoders = struct { + log: ?EncodeFn = null, + metric: ?EncodeFn = null, + trace: ?EncodeFn = null, + }; + + allocator: std.mem.Allocator, + handlers: std.EnumArray(ExtensionTag, ?ExtensionHandler), + encoders: Encoders, + + pub fn init(allocator: std.mem.Allocator, encoders: Encoders) Extensions { + return .{ + .allocator = allocator, + .handlers = .initFill(null), + .encoders = encoders, + }; + } + + pub fn deinit(self: *Extensions) void { + defer self.* = undefined; + var it = self.handlers.iterator(); + while (it.next()) |entry| { + if (entry.value.*) |*handler| handler.deinit(); + } + } + + /// Enable a handler. Takes ownership; replaces (and deinits) any handler + /// already enabled for the same tag. + pub fn enable(self: *Extensions, handler: ExtensionHandler) void { + const tag = std.meta.activeTag(handler); + if (self.handlers.getPtr(tag).*) |*old| old.deinit(); + self.handlers.set(tag, handler); + } + + /// Enable the s3-dump handler, constructing it directly in place, and + /// return a stable pointer so targets are configured on the handler + /// itself — no move, no `s3Dump().?` reach-back: + /// const s3 = exts.enableS3Dump(allocator, .{}, creds); + /// try s3.addTarget(io, "eu-bucket", target_json); + /// The pointer is valid for the life of this `Extensions` (don't move it). + pub fn enableS3Dump( + self: *Extensions, + allocator: std.mem.Allocator, + options: S3Dump.Options, + credentials: ?S3Dump.Credentials, + ) *S3Dump { + self.enable(.{ .s3_dump = S3Dump.init(allocator, options, credentials) }); + return self.s3Dump().?; + } + + pub fn s3Dump(self: *Extensions) ?*S3Dump { + if (self.handlers.getPtr(.s3_dump).*) |*handler| return &handler.s3_dump; + return null; + } + + /// Wire this Extensions into a registry in one call: the resolver (used + /// at snapshot compile) and the sync hooks (pushed to providers on + /// subscribe). Call before `registry.subscribe(...)`. The per-evaluate + /// `sink()` and the background `flush()` are still the consumer's to wire. + pub fn register(self: *Extensions, reg: *policy.Registry) void { + reg.setExtensionResolver(self.resolver()); + reg.setExtensionSyncHooks(self.syncHooks()); + } + + // ========================================================================= + // policy_zig seams + // ========================================================================= + + pub fn resolver(self: *Extensions) policy.ExtensionResolver { + return .{ .ctx = self, .resolve = resolveFn }; + } + + pub fn sink(self: *Extensions) policy.ExtensionSink { + return .{ .ctx = self, .deliver = deliverFn }; + } + + fn resolveFn( + io: std.Io, + ctx: *anyopaque, + signal: policy.TelemetryType, + policy_id: []const u8, + extension: *const proto.policy.Extension, + ) ?policy.ExtensionResolution { + const self: *Extensions = @ptrCast(@alignCast(ctx)); + const tag = ExtensionTag.fromTypeId(extension.type) orelse return null; + const version = std.SemanticVersion.parse(extension.version) catch return null; + if (version.major != tag.supportedMajor()) return null; + + switch (tag) { + .s3_dump => { + const handler = self.s3Dump() orelse return null; + const slot = handler.resolve(io, signal, policy_id, extension.config) orelse return null; + return .{ .handler = @intFromEnum(tag), .slot = slot }; + }, + } + } + + fn deliverFn( + ctx: *anyopaque, + io: ?std.Io, + signal: policy.TelemetryType, + record: *const anyopaque, + binding: *const policy.ExtensionBinding, + slice: policy.ExtensionSlice, + ) void { + _ = slice; // mode selection already applied by the engine + const self: *Extensions = @ptrCast(@alignCast(ctx)); + const encode = switch (signal) { + .log => self.encoders.log, + .metric => self.encoders.metric, + .trace => self.encoders.trace, + } orelse return; + const tag = std.enums.fromInt(ExtensionTag, binding.handler) orelse return; + switch (tag) { + .s3_dump => if (self.s3Dump()) |handler| { + handler.deliver(io, binding.slot, record, encode); + }, + } + } + + // ========================================================================= + // Sync plumbing helpers + // ========================================================================= + + /// Adapter for HttpProvider.setExtensionSyncHooks: advertises + /// capabilities in sync requests and routes broadcast extension configs + /// from sync responses. + pub fn syncHooks(self: *Extensions) policy.ExtensionSyncHooks { + return .{ + .ctx = self, + .capabilities = capabilitiesHook, + .apply_configs = applyConfigsHook, + }; + } + + fn capabilitiesHook( + io: std.Io, + ctx: *anyopaque, + arena: std.mem.Allocator, + ) anyerror![]proto.policy.ExtensionCapability { + const self: *Extensions = @ptrCast(@alignCast(ctx)); + return self.supportedExtensions(io, arena); + } + + fn applyConfigsHook( + io: std.Io, + ctx: *anyopaque, + configs: []const proto.policy.ExtensionConfig, + ) void { + const self: *Extensions = @ptrCast(@alignCast(ctx)); + self.applyExtensionConfigs(io, configs); + } + + /// Route `SyncResponse.extension_configs` entries to their handlers. + pub fn applyExtensionConfigs(self: *Extensions, io: std.Io, configs: []const proto.policy.ExtensionConfig) void { + for (configs) |config| { + const tag = ExtensionTag.fromTypeId(config.type) orelse continue; + switch (tag) { + .s3_dump => if (self.s3Dump()) |handler| { + handler.configure(io, config.config.items); + }, + } + } + } + + /// Build `ClientMetadata.supported_extensions` for the enabled handlers. + /// All memory (including descriptor bytes) is allocated with `arena` — + /// pass an arena allocator and free it wholesale. + pub fn supportedExtensions(self: *Extensions, io: std.Io, arena: std.mem.Allocator) ![]proto.policy.ExtensionCapability { + var out: std.ArrayList(proto.policy.ExtensionCapability) = .empty; + var it = self.handlers.iterator(); + while (it.next()) |entry| { + if (entry.value.* == null) continue; + const descriptors: []const []const u8 = switch (entry.key) { + .s3_dump => try self.s3Dump().?.capabilities(io, arena), + }; + var config_list: std.ArrayList([]const u8) = .empty; + try config_list.appendSlice(arena, descriptors); + try out.append(arena, .{ + .type = entry.key.typeId(), + .min_version = "1.0.0", + .config = config_list, + }); + } + return out.toOwnedSlice(arena); + } + + /// Flush all handlers that buffer for upload. Forward to the consumer's + /// background loop. + pub fn flush(self: *Extensions, io: std.Io, opts: S3Dump.FlushOptions) S3Dump.FlushResult { + if (self.s3Dump()) |handler| return handler.flush(io, opts); + return .{}; + } +}; + +test { + _ = @import("./s3_dump.zig"); + _ = @import("./s3_stub_test.zig"); +} + +// ============================================================================= +// Integration test: engine → resolver → sink → batch +// ============================================================================= + +const testing = std.testing; +const test_io = std.Options.debug_io; + +const TestLogRecord = struct { + body: []const u8, + + fn typedValue(ctx: *const anyopaque, field: policy.FieldRef) ?policy.TypedValue { + const self: *const TestLogRecord = @ptrCast(@alignCast(ctx)); + return switch (field) { + .log_field => |lf| if (lf == .LOG_FIELD_BODY) .{ .string = self.body } else null, + else => null, + }; + } + + const accessor: policy.LogAccessor = .{ .typed_value = typedValue }; + + fn encode(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + const self: *const TestLogRecord = @ptrCast(@alignCast(record)); + try writer.print("{{\"body\":\"{s}\"}}", .{self.body}); + } +}; + +fn encodeTargetRefAlloc(allocator: std.mem.Allocator, kind: []const u8, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = kind, .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +test "end to end: dropped records land in the s3-dump batch as ndjson" { + const allocator = testing.allocator; + const o11y = policy.observability; + + var exts = Extensions.init(allocator, .{ .log = TestLogRecord.encode }); + defer exts.deinit(); + // Configure the target directly on the handler pointer — no reach-back. + const s3 = exts.enableS3Dump(allocator, .{}, null); + try s3.addTarget( + test_io, + "eu-bucket", + \\{"bucket": "waste", "region": "eu-west-1"} + , + ); + + // Policy: drop payment logs, dump the waste to eu-bucket. + const ref_bytes = try encodeTargetRefAlloc(allocator, "s3", "eu-bucket"); + defer allocator.free(ref_bytes); + var pol: proto.policy.Policy = .{ + .id = try allocator.dupe(u8, "drop-payments"), + .name = try allocator.dupe(u8, "drop-payments"), + .enabled = true, + .target = .{ .log = .{ .keep = try allocator.dupe(u8, "none") } }, + }; + try pol.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "payment") }, + }); + try pol.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, "com.usetero/s3-dump"), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, ref_bytes), + .mode = .EXTENSION_MODE_DROPPED, + }); + defer pol.deinit(allocator); + + var noop_bus: o11y.NoopEventBus = undefined; + noop_bus.init(test_io); + var registry = policy.Registry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + exts.register(®istry); // resolver + sync hooks in one call + try registry.updatePolicies(&.{pol}, "test", .file); + + const engine = policy.PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var dropped_record: TestLogRecord = .{ .body = "payment failed" }; + const r1 = engine.evaluate(.log, &TestLogRecord.accessor, &dropped_record, &policy_id_buf, .{ + .io = test_io, + .extension_sink = exts.sink(), + }); + try testing.expectEqual(policy.FilterDecision.drop, r1.decision); + + var kept_record: TestLogRecord = .{ .body = "healthcheck" }; + _ = engine.evaluate(.log, &TestLogRecord.accessor, &kept_record, &policy_id_buf, .{ + .io = test_io, + .extension_sink = exts.sink(), + }); + + // Only the dropped record was batched, encoded by the consumer encoder. + const dump = exts.s3Dump().?; + try testing.expectEqual(@as(usize, 1), dump.batches.items.len); + try testing.expectEqualStrings( + "{\"body\":\"payment failed\"}\n", + dump.batches.items[0].buf.items, + ); + + // Capability advertisement covers the configured target. + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const caps = try exts.supportedExtensions(test_io, arena.allocator()); + try testing.expectEqual(@as(usize, 1), caps.len); + try testing.expectEqualStrings(TeroS3DumpTag, caps[0].type); + try testing.expectEqual(@as(usize, 1), caps[0].config.items.len); +} + +test "Extensions: applyExtensionConfigs routes broadcast targets by type" { + const allocator = testing.allocator; + var exts = Extensions.init(allocator, .{}); + defer exts.deinit(); + exts.enable(.{ .s3_dump = S3Dump.init(allocator, .{}, null) }); + + var aw = std.Io.Writer.Allocating.init(allocator); + defer aw.deinit(); + const target: proto.policy.ExtensionTarget = .{ + .kind = "s3", + .name = "bcast-bucket", + .config = "{\"bucket\": \"waste\"}", + }; + try target.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + const target_bytes = try list.toOwnedSlice(allocator); + defer allocator.free(target_bytes); + + var entries: std.ArrayList([]const u8) = .empty; + defer entries.deinit(allocator); + try entries.append(allocator, target_bytes); + + // A config for an unknown type is ignored; the s3-dump one lands. + const configs = [_]proto.policy.ExtensionConfig{ + .{ .type = "vendor/unknown", .config = entries }, + .{ .type = TeroS3DumpTag, .config = entries }, + }; + exts.applyExtensionConfigs(test_io, &configs); + + try testing.expectEqual(@as(usize, 1), exts.s3Dump().?.targets.items.len); + try testing.expectEqualStrings("bcast-bucket", exts.s3Dump().?.targets.items[0].name); +} + +test "Extensions.register wires resolver + sync hooks; subscribe pushes hooks per provider" { + const allocator = testing.allocator; + const o11y = policy.observability; + + var exts = Extensions.init(allocator, .{ .log = TestLogRecord.encode }); + defer exts.deinit(); + _ = exts.enableS3Dump(allocator, .{}, null); + + var noop_bus: o11y.NoopEventBus = undefined; + noop_bus.init(test_io); + var registry = policy.Registry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + + // One call wires both seams onto the registry. + exts.register(®istry); + try testing.expect(registry.extension_resolver != null); + try testing.expect(registry.extension_sync_hooks != null); + + // A file provider accepts the hooks and ignores them (no control plane): + // the union dispatch must not crash and leaves nothing to observe. + const file_provider = try policy.FileProvider.init(allocator, test_io, noop_bus.eventBus(), .{ + .id = "local", + .path = "does-not-exist.json", + }); + defer file_provider.deinit(); + const file_prov: policy.Provider = .{ .file = file_provider }; + file_prov.setExtensionSyncHooks(registry.extension_sync_hooks.?); + + // An HTTP provider actually stores them — this is what `subscribe` pushes. + const http_provider = try policy.HttpProvider.init(allocator, test_io, noop_bus.eventBus(), .{ + .id = "control-plane", + .url = "http://127.0.0.1:1/policies", + }); + defer http_provider.deinit(); + try testing.expect(http_provider.extension_hooks == null); + const http_prov: policy.Provider = .{ .http = http_provider }; + http_prov.setExtensionSyncHooks(registry.extension_sync_hooks.?); + try testing.expect(http_provider.extension_hooks != null); +} diff --git a/src/extensions/s3_bench.zig b/src/extensions/s3_bench.zig new file mode 100644 index 0000000..442847e --- /dev/null +++ b/src/extensions/s3_bench.zig @@ -0,0 +1,284 @@ +//! Scale benchmarks for the s3-dump extension against a real S3-compatible +//! backend (MinIO). Built as the `bench-s3` step (ReleaseFast executable, +//! excluded from `zig build test`) and driven by `task bench:s3`, which +//! starts MinIO in docker, runs this, and tears down. +//! +//! Scenarios, chosen to expose where scale hurts: +//! A. deliver hot path, 1 thread — encode+append+mutex cost per record +//! B. deliver hot path, 4 threads — the single-mutex contention ceiling +//! C. upload throughput vs object size — small-object overhead on flush +//! D. 64-policy fan-out — per-object cost with client reuse +//! E. outage cost per flush — z3 connect-retry sleeps serialize flush + +const std = @import("std"); +const proto = @import("proto"); +const s3 = @import("s3"); +const s3_dump_mod = @import("./s3_dump.zig"); +const S3Dump = s3_dump_mod.S3Dump; + +// Bench allocations are large and churny; the debug testing allocator would +// dominate the measurement. smp_allocator is the ReleaseFast-appropriate +// general-purpose allocator. +const gpa = std.heap.smp_allocator; + +const Env = struct { + access_key: []const u8, + secret_key: []const u8, + endpoint: []const u8, + bucket: []const u8, + + fn load(environ: std.process.Environ) ?Env { + return .{ + .access_key = environ.getPosix("AWS_ACCESS_KEY_ID") orelse return null, + .secret_key = environ.getPosix("AWS_SECRET_ACCESS_KEY") orelse return null, + .endpoint = environ.getPosix("S3_ENDPOINT") orelse "http://127.0.0.1:9000", + .bucket = environ.getPosix("S3_BUCKET") orelse "policy-zig-bench", + }; + } +}; + +/// ~200-byte log-shaped line, the typical record size for a waste dump. +const record_line = "{\"timestamp\":\"2026-07-08T12:00:00Z\",\"severity\":\"DEBUG\",\"body\":\"GET /api/v1/checkout/cart 200 requested by user\",\"service.name\":\"checkout-api\",\"http.request.id\":\"01890a5d-ac96-774b-b84e-fe56\"}"; + +fn encodeRecord(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + _ = record; + try writer.writeAll(record_line); +} + +fn encodeTargetRef(allocator: std.mem.Allocator, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = "s3", .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +fn nowNs(io: std.Io) i128 { + return std.Io.Timestamp.now(io, .awake).nanoseconds; +} + +fn mibPerSec(bytes: usize, elapsed_ns: i128) f64 { + const secs = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; + return @as(f64, @floatFromInt(bytes)) / (1024.0 * 1024.0) / secs; +} + +fn perSec(count: usize, elapsed_ns: i128) f64 { + const secs = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; + return @as(f64, @floatFromInt(count)) / secs; +} + +/// Build a dump with one target and `policies` resolved slots. Caller deinits. +fn makeDump( + io: std.Io, + env: Env, + options: S3Dump.Options, + endpoint_override: ?[]const u8, + policies: u32, +) !struct { dump: S3Dump, slots: []u32 } { + var dump = S3Dump.init(gpa, options, .{ + .access_key_id = env.access_key, + .secret_access_key = env.secret_key, + }); + errdefer dump.deinit(); + + const target_json = try std.fmt.allocPrint( + gpa, + \\{{"endpoint": "{s}", "region": "us-east-1", "bucket": "{s}", "prefix": "bench/", "force_path_style": true}} + , + .{ endpoint_override orelse env.endpoint, env.bucket }, + ); + defer gpa.free(target_json); + try dump.addTarget(io, "bench-target", target_json); + + const ref = try encodeTargetRef(gpa, "bench-target"); + defer gpa.free(ref); + + const slots = try gpa.alloc(u32, policies); + errdefer gpa.free(slots); + for (slots, 0..) |*slot, i| { + var id_buf: [32]u8 = undefined; + const policy_id = try std.fmt.bufPrint(&id_buf, "bench-policy-{d:0>3}", .{i}); + slot.* = dump.resolve(io, .log, policy_id, ref) orelse return error.ResolveFailed; + } + return .{ .dump = dump, .slots = slots }; +} + +fn deliverWorker(dump: *S3Dump, io: std.Io, slot: u32, count: usize) void { + for (0..count) |_| { + dump.deliver(io, slot, @ptrCast(&record_line), encodeRecord); + } +} + +pub fn main(init: std.process.Init) !void { + const env = Env.load(init.minimal.environ) orelse { + std.debug.print( + "bench-s3 needs a running S3-compatible server:\n" ++ + " AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (required)\n" ++ + " S3_ENDPOINT (default http://127.0.0.1:9000)\n" ++ + " S3_BUCKET (default policy-zig-bench)\n" ++ + "or run `task bench:s3` to do all of it via docker.\n", + .{}, + ); + return error.MissingS3Environment; + }; + + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + // Ensure the bucket exists. + var admin = try s3.S3Client.init(gpa, .{ + .access_key_id = env.access_key, + .secret_access_key = env.secret_key, + .endpoint = env.endpoint, + .virtual_host_style = false, + }, .{ .io = io }); + defer admin.deinit(); + var create_resp = try admin.createBucket(env.bucket, .{}); + create_resp.deinit(); + + std.debug.print( + "\ns3-dump scale benchmarks ({s}, record={d}B)\n" ++ + "{s}\n", + .{ env.endpoint, record_line.len, "-" ** 78 }, + ); + + // ------------------------------------------------------------------- + // A + B: deliver hot path, 1 vs 4 threads (no uploads; memory only). + // ------------------------------------------------------------------- + const deliver_records: usize = 200_000; + const deliver_opts: S3Dump.Options = .{ .max_sealed_bytes = 1 << 30 }; + inline for (.{ 1, 4 }) |workers| { + var made = try makeDump(io, env, deliver_opts, null, workers); + defer made.dump.deinit(); + defer gpa.free(made.slots); + + const per_worker = deliver_records / workers; + const start = nowNs(io); + if (workers == 1) { + deliverWorker(&made.dump, io, made.slots[0], per_worker); + } else { + var group: std.Io.Group = .init; + var spawn_failed = false; + for (made.slots) |slot| { + group.concurrent(io, deliverWorker, .{ &made.dump, io, slot, per_worker }) catch { + spawn_failed = true; + break; + }; + } + // Await before any early return so no worker outlives the dump. + group.await(io) catch {}; + if (spawn_failed) return error.ConcurrencyUnavailable; + } + const elapsed = nowNs(io) - start; + + const dropped = made.dump.records_dropped.load(.monotonic); + std.debug.print( + "deliver {d} thread(s) {d:>8} recs {d:>11.0} recs/s {d:>8.1} MiB/s drops={d}\n", + .{ + workers, + deliver_records, + perSec(deliver_records, elapsed), + mibPerSec(deliver_records * (record_line.len + 1), elapsed), + dropped, + }, + ); + } + + // ------------------------------------------------------------------- + // C: upload throughput vs object size (~32MiB total per size). + // ------------------------------------------------------------------- + const total_upload_bytes: usize = 32 << 20; + inline for (.{ 64 << 10, 1 << 20, 4 << 20 }) |object_bytes| { + var made = try makeDump(io, env, .{ + .max_batch_bytes = object_bytes, + .max_sealed_bytes = 1 << 30, + .max_attempts = 1, + }, null, 1); + defer made.dump.deinit(); + defer gpa.free(made.slots); + + const records = total_upload_bytes / (record_line.len + 1); + deliverWorker(&made.dump, io, made.slots[0], records); + + const start = nowNs(io); + const result = made.dump.flush(io, .{ .force = true }); + const elapsed = nowNs(io) - start; + + std.debug.print( + "flush {d:>5} KiB objects {d:>8} objs {d:>11.1} objs/s {d:>8.1} MiB/s failed={d}\n", + .{ + object_bytes / 1024, + result.objects_uploaded, + perSec(result.objects_uploaded, elapsed), + mibPerSec(total_upload_bytes, elapsed), + result.objects_failed, + }, + ); + } + + // ------------------------------------------------------------------- + // D: 64-policy fan-out — one flush uploading 64 small objects. + // ------------------------------------------------------------------- + { + const fanout_policies: u32 = 64; + const per_policy_bytes: usize = 128 << 10; + var made = try makeDump(io, env, .{ .max_sealed_bytes = 1 << 30, .max_attempts = 1 }, null, fanout_policies); + defer made.dump.deinit(); + defer gpa.free(made.slots); + + const records_per_policy = per_policy_bytes / (record_line.len + 1); + for (made.slots) |slot| { + deliverWorker(&made.dump, io, slot, records_per_policy); + } + + const start = nowNs(io); + const result = made.dump.flush(io, .{ .force = true }); + const elapsed = nowNs(io) - start; + + std.debug.print( + "fan-out 64 policies {d:>8} objs {d:>11.1} objs/s {d:>8.1} MiB/s failed={d}\n", + .{ + result.objects_uploaded, + perSec(result.objects_uploaded, elapsed), + mibPerSec(fanout_policies * per_policy_bytes, elapsed), + result.objects_failed, + }, + ); + } + + // ------------------------------------------------------------------- + // E: outage cost — dead endpoint, 8 batches, per-flush wall time. + // z3 sleeps ~100ms between connect retries, serialized per object, so + // max_attempts multiplies directly into flush latency during an outage. + // ------------------------------------------------------------------- + inline for (.{ 1, 3 }) |attempts| { + var made = try makeDump(io, env, .{ + .max_sealed_bytes = 1 << 30, + .max_attempts = attempts, + }, "http://127.0.0.1:9", 8); + defer made.dump.deinit(); + defer gpa.free(made.slots); + + for (made.slots) |slot| { + deliverWorker(&made.dump, io, slot, 16); + } + + const start = nowNs(io); + const result = made.dump.flush(io, .{ .force = true }); + const elapsed = nowNs(io) - start; + + std.debug.print( + "outage flush (attempts={d}) {d:>7} objs {d:>10.1} ms/flush requeued={d}\n", + .{ + attempts, + result.objects_failed, + @as(f64, @floatFromInt(elapsed)) / std.time.ns_per_ms, + result.objects_requeued, + }, + ); + } + + std.debug.print("{s}\n", .{"-" ** 78}); +} diff --git a/src/extensions/s3_dump.zig b/src/extensions/s3_dump.zig new file mode 100644 index 0000000..1dd3d11 --- /dev/null +++ b/src/extensions/s3_dump.zig @@ -0,0 +1,863 @@ +//! `com.usetero/s3-dump` extension handler (policy spec v1.6.0). +//! +//! Routes records selected by a policy extension's mode to a pre-configured +//! S3-compatible destination as batched ndjson objects. See +//! design/extensions.md for the full design. +//! +//! Invariants: +//! - `deliver` copies (encodes) synchronously and holds no reference to the +//! record after it returns; the hot path never performs network I/O. +//! - A destination outage costs pipeline latency never: the sealed-backlog +//! cap bounds all buffered memory, and past it records drop (counted). +//! - Failed uploads are requeued (within the same cap) and retried on later +//! flushes under the SAME object key, so retries are idempotent. Records +//! are lost only when the backlog cap overflows, never silently. +//! - I/O happens only in consumer-driven `flush(io)`, per this repo's rule +//! that `std.Io` is passed per call and never stored. + +const std = @import("std"); +const policy = @import("policy_zig"); +const proto = @import("proto"); +const s3 = @import("s3"); + +/// Renders one record to bytes (one ndjson line, no trailing newline). +/// Wired by the consumer per signal — the record type is the consumer's. +pub const EncodeFn = *const fn (record: *const anyopaque, writer: *std.Io.Writer) anyerror!void; + +pub const S3Dump = struct { + /// All knobs, as plain data with defaults so a consumer can deserialize + /// them straight from a config file (e.g. std.json) later. + pub const Options = struct { + /// Open batch is sealed for upload once it reaches this size. + max_batch_bytes: usize = 4 << 20, + /// Open batch is sealed for upload once it holds this many records. + max_batch_records: u32 = 10_000, + /// flush() seals a non-empty open batch older than this. + max_batch_age_ms: u64 = 30_000, + /// Cap on sealed-but-not-uploaded bytes; past it new records drop. + max_sealed_bytes: usize = 32 << 20, + /// Per-request attempts passed to the S3 client. The client only + /// retries connection establishment, sleeping ~100ms between tries + /// serialized per object — measured at ~290ms/object of added flush + /// latency during an outage (`task bench:s3`). Since failed batches + /// are requeued and retried on the next flush anyway, the default + /// keeps flush fast; raise it only if the next-flush wait is too + /// coarse for transient connect blips. + max_attempts: usize = 1, + content_type: []const u8 = "application/x-ndjson", + }; + + /// Static credentials, consumer-provided (e.g. read from env or a + /// credentials file by the edge binary). Never carried in policies or + /// targets. Null → uploads are counted as failed and batches dropped. + pub const Credentials = struct { + access_key_id: []const u8, + secret_access_key: []const u8, + }; + + /// Kind-defined config carried in `ExtensionTarget.config` for kind + /// "s3", authored as JSON. + pub const TargetConfig = struct { + endpoint: ?[]const u8 = null, + region: []const u8 = "us-east-1", + bucket: []const u8, + prefix: []const u8 = "", + force_path_style: bool = true, + }; + + const Target = struct { + name: []const u8, + parsed: std.json.Parsed(TargetConfig), + }; + + /// One open batch per (target, signal, policy). Created at resolve time + /// (snapshot compile) so `deliver` is an array index, never a hash or + /// string compare. Batches persist across snapshot rebuilds so pending + /// data survives policy updates. + const Batch = struct { + target: u16, + signal: policy.TelemetryType, + policy_id: []const u8, + buf: std.ArrayList(u8) = .empty, + records: u32 = 0, + /// Monotonic (.awake) time of the first record, for age sealing. + first_ns: i128 = 0, + }; + + const Sealed = struct { + target: u16, + signal: policy.TelemetryType, + /// Borrowed from the owning Batch (batches live until deinit). + policy_id: []const u8, + /// Object key, fixed at seal time so retried uploads are idempotent + /// (same key, identical content). Owned. + key: []u8, + buf: []u8, + records: u32, + }; + + /// Flush-local copy of a target's connection config, duped into the + /// flush arena under the mutex. Uploads read only snapshots, so a + /// concurrent addTarget/configure (provider sync thread) can never + /// invalidate memory mid-upload. + const TargetSnapshot = struct { + bucket: []const u8, + region: []const u8, + endpoint: ?[]const u8, + virtual_host_style: bool, + }; + + /// PUT responses are tiny (empty or an XML error); never buffer more. + const max_response_body: usize = 64 * 1024; + + pub const FlushOptions = struct { + /// Seal and upload all open batches regardless of age (shutdown). + force: bool = false, + }; + + /// Everything a consumer needs to turn one `flush()` call into metrics: + /// counters (deltas since the previous flush) plus one gauge + /// (`backlog_bytes`, a point-in-time snapshot, not a delta) for + /// early-warning before the `max_sealed_bytes` cap starts dropping data. + /// The library emits no metrics itself — this struct is the seam; the + /// edge decides where counts and gauge go (OTel, Prometheus, statsd, ...). + pub const FlushResult = struct { + objects_uploaded: u32 = 0, + /// Objects that failed to upload this flush. Unless the backlog cap + /// forced a drop they are requeued and retried on the next flush. + objects_failed: u32 = 0, + /// Of the failed objects, how many were requeued for retry. + objects_requeued: u32 = 0, + records_uploaded: u64 = 0, + records_failed: u64 = 0, + /// Records dropped since the previous flush: at delivery time + /// (backlog full, encode failure, null io) or because a failed batch + /// could not be requeued within the backlog cap. + records_dropped: u64 = 0, + /// Object body bytes successfully PUT this flush. Feeds a + /// throughput/cost counter (S3 PUT + egress pricing scales with this). + bytes_uploaded: u64 = 0, + /// Object body bytes in batches that failed to upload this flush + /// (whether requeued or dropped). + bytes_failed: u64 = 0, + /// GAUGE, not a delta: sealed-but-not-yet-uploaded bytes remaining + /// after this flush (open batches are excluded — they aren't at risk + /// of the drop cliff yet). Alert on this approaching + /// `Options.max_sealed_bytes`: that's the leading indicator of the + /// destination falling behind, before `records_dropped` climbs. + backlog_bytes: usize = 0, + }; + + allocator: std.mem.Allocator, + options: Options, + credentials: ?Credentials, + mutex: std.Io.Mutex = .init, + targets: std.ArrayList(Target) = .empty, + batches: std.ArrayList(*Batch) = .empty, + sealed: std.ArrayList(Sealed) = .empty, + sealed_bytes: usize = 0, + records_dropped: std.atomic.Value(u64) = .init(0), + object_seq: std.atomic.Value(u64) = .init(0), + + pub fn init(allocator: std.mem.Allocator, options: Options, credentials: ?Credentials) S3Dump { + return .{ + .allocator = allocator, + .options = options, + .credentials = credentials, + }; + } + + pub fn deinit(self: *S3Dump) void { + defer self.* = undefined; + for (self.batches.items) |batch| { + batch.buf.deinit(self.allocator); + self.allocator.free(batch.policy_id); + self.allocator.destroy(batch); + } + self.batches.deinit(self.allocator); + for (self.sealed.items) |s| { + self.allocator.free(s.buf); + self.allocator.free(s.key); + } + self.sealed.deinit(self.allocator); + for (self.targets.items) |*t| { + self.allocator.free(t.name); + t.parsed.deinit(); + } + self.targets.deinit(self.allocator); + } + + // ========================================================================= + // Target configuration + // ========================================================================= + + /// Add or replace a named s3 target. `config_json` is the kind-defined + /// JSON TargetConfig. Used for local (out-of-band) configuration; the + /// broadcast path (`configure`) lands here too, so broadcast wins on a + /// name collision. + pub fn addTarget(self: *S3Dump, io: std.Io, name: []const u8, config_json: []const u8) !void { + const parsed = try std.json.parseFromSlice(TargetConfig, self.allocator, config_json, .{ + .ignore_unknown_fields = true, + .allocate = .alloc_always, + }); + errdefer parsed.deinit(); + const name_copy = try self.allocator.dupe(u8, name); + errdefer self.allocator.free(name_copy); + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + for (self.targets.items) |*t| { + if (std.mem.eql(u8, t.name, name)) { + self.allocator.free(t.name); + t.parsed.deinit(); + t.* = .{ .name = name_copy, .parsed = parsed }; + return; + } + } + try self.targets.append(self.allocator, .{ .name = name_copy, .parsed = parsed }); + } + + /// Ingest broadcast target entries (`SyncResponse.extension_configs` for + /// this type): each entry is a serialized ExtensionTarget whose `config` + /// is JSON TargetConfig. Non-s3 kinds and malformed entries are skipped + /// (fail-open). + pub fn configure(self: *S3Dump, io: std.Io, entries: []const []const u8) void { + for (entries) |bytes| { + var reader = std.Io.Reader.fixed(bytes); + var target = proto.policy.ExtensionTarget.decode(&reader, self.allocator) catch continue; + defer target.deinit(self.allocator); + if (!std.mem.eql(u8, target.kind, "s3")) continue; + self.addTarget(io, target.name, target.config) catch continue; + } + } + + /// One serialized ExtensionTargetRef per configured target — the + /// capability descriptors for `ClientMetadata.supported_extensions`. + /// Caller owns the returned slices (allocated with `allocator`). + pub fn capabilities(self: *S3Dump, io: std.Io, allocator: std.mem.Allocator) ![]const []const u8 { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + var out: std.ArrayList([]const u8) = .empty; + errdefer { + for (out.items) |item| allocator.free(item); + out.deinit(allocator); + } + for (self.targets.items) |t| { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = "s3", .name = t.name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + try out.append(allocator, try list.toOwnedSlice(allocator)); + } + return try out.toOwnedSlice(allocator); + } + + // ========================================================================= + // Resolution (snapshot compile time, cold) + // ========================================================================= + + /// Resolve one policy extension config (a serialized ExtensionTargetRef) + /// to a batch slot. Null → unsupported/invalid, the caller skips the + /// extension fail-open. The same (target, signal, policy) triple always + /// resolves to the same slot, so batches survive snapshot rebuilds. + pub fn resolve( + self: *S3Dump, + io: std.Io, + signal: policy.TelemetryType, + policy_id: []const u8, + config: []const u8, + ) ?u32 { + var reader = std.Io.Reader.fixed(config); + var ref = proto.policy.ExtensionTargetRef.decode(&reader, self.allocator) catch return null; + defer ref.deinit(self.allocator); + if (!std.mem.eql(u8, ref.kind, "s3")) return null; + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + const target_index: u16 = blk: { + for (self.targets.items, 0..) |t, i| { + if (std.mem.eql(u8, t.name, ref.name)) break :blk @intCast(i); + } + return null; + }; + + for (self.batches.items, 0..) |batch, i| { + if (batch.target == target_index and batch.signal == signal and + std.mem.eql(u8, batch.policy_id, policy_id)) + { + return @intCast(i); + } + } + + const batch = self.allocator.create(Batch) catch return null; + const id_copy = self.allocator.dupe(u8, policy_id) catch { + self.allocator.destroy(batch); + return null; + }; + batch.* = .{ .target = target_index, .signal = signal, .policy_id = id_copy }; + self.batches.append(self.allocator, batch) catch { + self.allocator.free(id_copy); + self.allocator.destroy(batch); + return null; + }; + return @intCast(self.batches.items.len - 1); + } + + // ========================================================================= + // Delivery (hot path — encode + append only, never I/O) + // ========================================================================= + + pub fn deliver( + self: *S3Dump, + io_opt: ?std.Io, + slot: u32, + record: *const anyopaque, + encode: EncodeFn, + ) void { + const io = io_opt orelse return self.drop(); + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + // Stale/foreign slot: fail-open. + if (slot >= self.batches.items.len) return self.drop(); + const batch = self.batches.items[slot]; + + // Seal a full batch first — and check the sealed-backlog cap BEFORE + // encoding, so a destination outage degrades to a counter increment. + const must_seal = batch.buf.items.len >= self.options.max_batch_bytes or + batch.records >= self.options.max_batch_records; + if (must_seal) { + if (self.sealed_bytes + batch.buf.items.len > self.options.max_sealed_bytes) { + return self.drop(); + } + self.sealLocked(io, batch) catch return self.drop(); + } + + // Encode directly into the batch buffer — one write, no intermediate + // copy. A failed or oversized encode is rolled back by truncating to + // the pre-record length, so the batch is never corrupted. + const len_before = batch.buf.items.len; + var aw = std.Io.Writer.Allocating.fromArrayList(self.allocator, &batch.buf); + var write_failed = false; + encode(record, &aw.writer) catch { + write_failed = true; + }; + if (!write_failed) aw.writer.writeByte('\n') catch { + write_failed = true; + }; + batch.buf = aw.toArrayList(); + if (write_failed or batch.buf.items.len - len_before > self.options.max_batch_bytes) { + // A rogue oversized record may have grown the buffer far past + // anything a legal batch needs; release that memory rather than + // pinning it on this slot forever. Legal batches never need more + // than ~2x max_batch_bytes (seal-before-encode + one record). + if (batch.buf.capacity > self.options.max_batch_bytes * 2) { + batch.buf.shrinkAndFree(self.allocator, len_before); + } else { + batch.buf.shrinkRetainingCapacity(len_before); + } + return self.drop(); + } + batch.records += 1; + if (batch.first_ns == 0) { + batch.first_ns = std.Io.Timestamp.now(io, .awake).nanoseconds; + } + } + + fn drop(self: *S3Dump) void { + _ = self.records_dropped.fetchAdd(1, .monotonic); + } + + /// Move the open batch's buffer onto the sealed list and reset the + /// batch. The object key is fixed here, at seal time: a batch that fails + /// to upload retries under the SAME key, so a PUT that actually + /// succeeded server-side before the failure surfaced is overwritten with + /// identical content instead of duplicated under a second key. + fn sealLocked(self: *S3Dump, io: std.Io, batch: *Batch) !void { + if (batch.buf.items.len == 0) return; + const prefix = self.targets.items[batch.target].parsed.value.prefix; + const key = try self.buildKey(io, prefix, batch.signal, batch.policy_id); + errdefer self.allocator.free(key); + const buf = try batch.buf.toOwnedSlice(self.allocator); + errdefer self.allocator.free(buf); + try self.sealed.append(self.allocator, .{ + .target = batch.target, + .signal = batch.signal, + .policy_id = batch.policy_id, + .key = key, + .buf = buf, + .records = batch.records, + }); + self.sealed_bytes += buf.len; + batch.records = 0; + batch.first_ns = 0; + } + + /// Requeue a failed batch for the next flush, bounded by the same + /// backlog cap as delivery-time sealing. Returns false when the cap (or + /// OOM) forced a drop instead. + fn requeue(self: *S3Dump, io: std.Io, sealed_batch: Sealed) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + const fits = self.sealed_bytes + sealed_batch.buf.len <= self.options.max_sealed_bytes; + if (fits) { + if (self.sealed.append(self.allocator, sealed_batch)) |_| { + self.sealed_bytes += sealed_batch.buf.len; + return true; + } else |_| {} + } + _ = self.records_dropped.fetchAdd(sealed_batch.records, .monotonic); + self.allocator.free(sealed_batch.buf); + self.allocator.free(sealed_batch.key); + return false; + } + + // ========================================================================= + // Flush (consumer-driven I/O) + // ========================================================================= + + /// Seal over-age (or, with `force`, all) open batches and upload every + /// sealed batch as one object. A failed upload is requeued for the next + /// flush (bounded by max_sealed_bytes) rather than dropped — z3 only + /// retries connection establishment, not send/receive failures or 5xx + /// responses, so handler-level retry is what makes a transient + /// destination outage lossless. + pub fn flush(self: *S3Dump, io: std.Io, opts: FlushOptions) FlushResult { + var result: FlushResult = .{}; + + const now: i128 = std.Io.Timestamp.now(io, .awake).nanoseconds; + const max_age_ns: i128 = @as(i128, self.options.max_batch_age_ms) * std.time.ns_per_ms; + + var arena_state = std.heap.ArenaAllocator.init(self.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var no_snapshots = [0]?TargetSnapshot{}; + var no_clients = [0]?s3.S3Client{}; + + // Phase 1 (locked): seal due batches, steal the sealed list, and + // snapshot target configs into the flush arena. Uploads read only + // the snapshots, never `self.targets`. + self.mutex.lockUncancelable(io); + for (self.batches.items) |batch| { + if (batch.buf.items.len == 0) continue; + const expired = batch.first_ns != 0 and now - batch.first_ns >= max_age_ns; + if (opts.force or expired) { + // OOM here just leaves the batch open for the next flush. + self.sealLocked(io, batch) catch {}; + } + } + const to_upload = self.sealed.toOwnedSlice(self.allocator) catch { + self.mutex.unlock(io); + result.records_dropped = self.records_dropped.swap(0, .monotonic); + return result; + }; + self.sealed_bytes = 0; + const snapshots: []?TargetSnapshot = + arena.alloc(?TargetSnapshot, self.targets.items.len) catch no_snapshots[0..]; + for (self.targets.items, 0..) |t, i| { + if (i >= snapshots.len) break; + snapshots[i] = snapshotTarget(arena, t.parsed.value) catch null; + } + self.mutex.unlock(io); + defer self.allocator.free(to_upload); + + // Phase 2 (unlocked): create one client per target used this flush, + // then upload. Client setup is a distinct step from the PUT — one + // client per target so the std.http.Client connection pool is reused + // across all of this flush's uploads to that target. + const clients = self.makeClients(io, arena, snapshots, to_upload, no_clients[0..]); + defer for (clients) |*c| { + if (c.*) |*client| client.deinit(); + }; + + for (to_upload) |sealed_batch| { + const uploaded = ok: { + const client = clientFor(clients, sealed_batch.target) orelse break :ok false; + break :ok self.uploadOne(client, snapshots[sealed_batch.target].?, &sealed_batch); + }; + if (uploaded) { + result.objects_uploaded += 1; + result.records_uploaded += sealed_batch.records; + result.bytes_uploaded += sealed_batch.buf.len; + self.allocator.free(sealed_batch.buf); + self.allocator.free(sealed_batch.key); + } else { + result.objects_failed += 1; + result.records_failed += sealed_batch.records; + result.bytes_failed += sealed_batch.buf.len; + if (self.requeue(io, sealed_batch)) result.objects_requeued += 1; + } + } + + result.records_dropped = self.records_dropped.swap(0, .monotonic); + // Gauge read AFTER requeue has put failed-but-kept batches back onto + // the backlog, so it reflects what's actually still queued. + // `sealed_bytes` is plain state guarded by `mutex` (like the rest of + // this struct), not an atomic — take the lock rather than tear it. + self.mutex.lockUncancelable(io); + result.backlog_bytes = self.sealed_bytes; + self.mutex.unlock(io); + return result; + } + + fn snapshotTarget(arena: std.mem.Allocator, cfg: TargetConfig) !TargetSnapshot { + return .{ + .bucket = try arena.dupe(u8, cfg.bucket), + .region = try arena.dupe(u8, cfg.region), + .endpoint = if (cfg.endpoint) |e| try arena.dupe(u8, e) else null, + .virtual_host_style = !cfg.force_path_style, + }; + } + + /// Create one S3 client per target that has a batch to upload this flush + /// (deduped — several batches for one target share a client, hence one + /// connection pool). Indexed by target, parallel to `snapshots`. A target + /// with no credentials, no snapshot, or a failed client init is left null; + /// its batches then fail the upload and requeue. Clients are owned by the + /// caller (freed via the flush's `defer`), allocated in the flush arena. + fn makeClients( + self: *S3Dump, + io: std.Io, + arena: std.mem.Allocator, + snapshots: []const ?TargetSnapshot, + to_upload: []const Sealed, + fallback: []?s3.S3Client, + ) []?s3.S3Client { + const clients = arena.alloc(?s3.S3Client, snapshots.len) catch return fallback; + for (clients) |*c| c.* = null; + + const creds = self.credentials orelse return clients; // no creds → no clients + for (to_upload) |sealed_batch| { + const target = sealed_batch.target; + if (target >= clients.len or clients[target] != null) continue; + const snap = snapshots[target] orelse continue; + clients[target] = s3.S3Client.init(self.allocator, .{ + .access_key_id = creds.access_key_id, + .secret_access_key = creds.secret_access_key, + .region = snap.region, + .endpoint = snap.endpoint, + .virtual_host_style = snap.virtual_host_style, + }, .{ .io = io }) catch continue; + } + return clients; + } + + /// The client for a target, or null if none was created for it (out of + /// range, or setup skipped it — see `makeClients`). + fn clientFor(clients: []?s3.S3Client, target: u16) ?*s3.S3Client { + if (target >= clients.len) return null; + return if (clients[target]) |*c| c else null; + } + + /// PUT one sealed batch as a single object. No client lifecycle here — the + /// client is created up front by `makeClients`. + fn uploadOne( + self: *S3Dump, + client: *s3.S3Client, + snap: TargetSnapshot, + sealed_batch: *const Sealed, + ) bool { + var response = client.putObject(snap.bucket, sealed_batch.key, sealed_batch.buf, .{ + .content_type = self.options.content_type, + .request = .{ + .max_attempts = self.options.max_attempts, + .max_body_size = max_response_body, + }, + }) catch return false; + defer response.deinit(); + return response.http_head.status.class() == .success; + } + + /// `{prefix}{signal}/{yyyy}/{mm}/{dd}/{hh}/{policy_id}-{unix_nanos}-{seq}.ndjson` + fn buildKey( + self: *S3Dump, + io: std.Io, + prefix: []const u8, + signal: policy.TelemetryType, + policy_id: []const u8, + ) ![]u8 { + const wall_ns: i128 = std.Io.Timestamp.now(io, .real).nanoseconds; + const secs: u64 = @intCast(@divTrunc(wall_ns, std.time.ns_per_s)); + const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = secs }; + const year_day = epoch_secs.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const seq = self.object_seq.fetchAdd(1, .monotonic); + return std.fmt.allocPrint( + self.allocator, + "{s}{s}/{d:0>4}/{d:0>2}/{d:0>2}/{d:0>2}/{s}-{d}-{d}.ndjson", + .{ + prefix, + @tagName(signal), + year_day.year, + month_day.month.numeric(), + @as(u32, month_day.day_index) + 1, + epoch_secs.getDaySeconds().getHoursIntoDay(), + policy_id, + wall_ns, + seq, + }, + ); + } +}; + +// ============================================================================= +// Tests +// ============================================================================= + +const testing = std.testing; +const test_io = std.Options.debug_io; + +fn encodeTargetRef(allocator: std.mem.Allocator, kind: []const u8, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = kind, .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +const target_json = + \\{"endpoint": "http://127.0.0.1:1", "region": "eu-west-1", "bucket": "waste", "prefix": "dumps/"} +; + +fn testEncodeRecord(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + const msg: *const []const u8 = @ptrCast(@alignCast(record)); + try writer.writeAll(msg.*); +} + +test "S3Dump: resolve validates kind and target, slots are stable" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const good = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(good); + const bad_kind = try encodeTargetRef(testing.allocator, "gcs", "eu-bucket"); + defer testing.allocator.free(bad_kind); + const bad_name = try encodeTargetRef(testing.allocator, "s3", "nope"); + defer testing.allocator.free(bad_name); + + try testing.expect(dump.resolve(test_io, .log, "p1", bad_kind) == null); + try testing.expect(dump.resolve(test_io, .log, "p1", bad_name) == null); + + const slot = dump.resolve(test_io, .log, "p1", good).?; + // Same triple → same slot (snapshot rebuilds keep their batches). + try testing.expectEqual(slot, dump.resolve(test_io, .log, "p1", good).?); + // Different policy or signal → different slot. + try testing.expect(dump.resolve(test_io, .log, "p2", good).? != slot); + try testing.expect(dump.resolve(test_io, .trace, "p1", good).? != slot); +} + +test "S3Dump: deliver batches records as ndjson lines" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec1: []const u8 = "{\"body\":\"one\"}"; + const rec2: []const u8 = "{\"body\":\"two\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec1), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec2), testEncodeRecord); + + const batch = dump.batches.items[slot]; + try testing.expectEqual(@as(u32, 2), batch.records); + try testing.expectEqualStrings("{\"body\":\"one\"}\n{\"body\":\"two\"}\n", batch.buf.items); + try testing.expectEqual(@as(u64, 0), dump.records_dropped.load(.monotonic)); +} + +test "S3Dump: flush result carries byte counters and a backlog gauge" { + var dump = S3Dump.init(testing.allocator, .{}, null); // no credentials → upload fails + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; // 14 bytes + '\n' = 15 + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + + const result = dump.flush(test_io, .{ .force = true }); + + // Failed upload: bytes counted as failed, none as uploaded, and the + // gauge reflects the batch sitting in the backlog after requeue. + try testing.expectEqual(@as(u64, 30), result.bytes_failed); + try testing.expectEqual(@as(u64, 0), result.bytes_uploaded); + try testing.expectEqual(@as(usize, 30), result.backlog_bytes); + try testing.expectEqual(dump.sealed_bytes, result.backlog_bytes); + + // A flush with nothing to do reports a zero gauge, not the last value. + var empty_dump = S3Dump.init(testing.allocator, .{}, null); + defer empty_dump.deinit(); + const empty_result = empty_dump.flush(test_io, .{ .force = true }); + try testing.expectEqual(@as(usize, 0), empty_result.backlog_bytes); +} + +test "S3Dump: batch seals at record limit; backlog cap drops instead of growing" { + var dump = S3Dump.init(testing.allocator, .{ + .max_batch_records = 2, + .max_sealed_bytes = 40, + }, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "0123456789"; // 11 bytes per line with \n + // Two records fill the batch; the third seals it (22 bytes ≤ 40) and + // starts a new one. + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + try testing.expectEqual(@as(usize, 1), dump.sealed.items.len); + try testing.expectEqual(@as(u32, 2), dump.sealed.items[0].records); + try testing.expectEqual(@as(u32, 1), dump.batches.items[slot].records); + + // Fill and roll again: sealing would put the backlog at 44 > 40, so the + // record that forces the seal is dropped before encoding. + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + try testing.expectEqual(@as(u64, 1), dump.records_dropped.load(.monotonic)); + try testing.expectEqual(@as(usize, 1), dump.sealed.items.len); +} + +test "S3Dump: failed upload is requeued under a stable key, not dropped" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + const batch_len = dump.batches.items[slot].buf.items.len; + + // No credentials → upload fails → the batch stays queued for retry. + const r1 = dump.flush(test_io, .{ .force = true }); + try testing.expectEqual(@as(u32, 0), r1.objects_uploaded); + try testing.expectEqual(@as(u32, 1), r1.objects_failed); + try testing.expectEqual(@as(u32, 1), r1.objects_requeued); + try testing.expectEqual(@as(u64, 1), r1.records_failed); + try testing.expectEqual(@as(u64, 0), r1.records_dropped); + try testing.expectEqual(@as(usize, 1), dump.sealed.items.len); + try testing.expectEqual(batch_len, dump.sealed_bytes); + try testing.expectEqual(@as(usize, 0), dump.batches.items[slot].buf.items.len); + + // The key was fixed at seal time and survives across retries, so a + // retried PUT is idempotent. Snapshot it, retry, compare. + const key_copy = try testing.allocator.dupe(u8, dump.sealed.items[0].key); + defer testing.allocator.free(key_copy); + try testing.expect(std.mem.startsWith(u8, key_copy, "dumps/log/")); + try testing.expect(std.mem.endsWith(u8, key_copy, ".ndjson")); + + const r2 = dump.flush(test_io, .{ .force = true }); + try testing.expectEqual(@as(u32, 1), r2.objects_failed); + try testing.expectEqual(@as(u32, 1), r2.objects_requeued); + try testing.expectEqual(@as(usize, 1), dump.sealed.items.len); + try testing.expectEqualStrings(key_copy, dump.sealed.items[0].key); +} + +test "S3Dump: requeue drops when the backlog cap would be exceeded" { + // Cap smaller than one batch: the failed upload cannot be requeued. + var dump = S3Dump.init(testing.allocator, .{ .max_sealed_bytes = 4 }, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + + const result = dump.flush(test_io, .{ .force = true }); + try testing.expectEqual(@as(u32, 1), result.objects_failed); + try testing.expectEqual(@as(u32, 0), result.objects_requeued); + try testing.expectEqual(@as(u64, 1), result.records_dropped); + try testing.expectEqual(@as(usize, 0), dump.sealed.items.len); + try testing.expectEqual(@as(usize, 0), dump.sealed_bytes); +} + +fn testEncodePartialFail(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + _ = record; + try writer.writeAll("partial garbage"); + return error.EncodeFailed; +} + +test "S3Dump: failed encode rolls the batch back to its previous contents" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + // Partial write, then failure: everything it wrote must be truncated. + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodePartialFail); + + const batch = dump.batches.items[slot]; + try testing.expectEqual(@as(u32, 1), batch.records); + try testing.expectEqualStrings("{\"body\":\"one\"}\n", batch.buf.items); + try testing.expectEqual(@as(u64, 1), dump.records_dropped.load(.monotonic)); +} + +test "S3Dump: non-forced flush leaves young batches open" { + var dump = S3Dump.init(testing.allocator, .{ .max_batch_age_ms = 60_000 }, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + + const result = dump.flush(test_io, .{}); + try testing.expectEqual(@as(u32, 0), result.objects_uploaded + result.objects_failed); + try testing.expectEqual(@as(u32, 1), dump.batches.items[slot].records); +} + +test "S3Dump: configure ingests broadcast targets; capabilities round-trips refs" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + + // Broadcast one s3 target and one foreign-kind target (skipped). + var aw = std.Io.Writer.Allocating.init(testing.allocator); + defer aw.deinit(); + const bcast: proto.policy.ExtensionTarget = .{ .kind = "s3", .name = "us-bucket", .config = target_json }; + try bcast.encode(&aw.writer, testing.allocator); + var list = aw.toArrayList(); + const bcast_bytes = try list.toOwnedSlice(testing.allocator); + defer testing.allocator.free(bcast_bytes); + + dump.configure(test_io, &.{ bcast_bytes, "not-a-proto" }); + try testing.expectEqual(@as(usize, 1), dump.targets.items.len); + + const caps = try dump.capabilities(test_io, testing.allocator); + defer { + for (caps) |c| testing.allocator.free(c); + testing.allocator.free(caps); + } + try testing.expectEqual(@as(usize, 1), caps.len); + var reader = std.Io.Reader.fixed(caps[0]); + var ref = try proto.policy.ExtensionTargetRef.decode(&reader, testing.allocator); + defer ref.deinit(testing.allocator); + try testing.expectEqualStrings("s3", ref.kind); + try testing.expectEqualStrings("us-bucket", ref.name); +} diff --git a/src/extensions/s3_minio_test.zig b/src/extensions/s3_minio_test.zig new file mode 100644 index 0000000..bf55fab --- /dev/null +++ b/src/extensions/s3_minio_test.zig @@ -0,0 +1,135 @@ +//! Real-storage smoke test for the s3-dump extension, run against a local +//! MinIO container. Not part of `zig build test` — it needs a running +//! backend, so it's a separate build step (`zig build test-s3-e2e`) driven by +//! `task test:s3-e2e`, which starts MinIO via docker, runs this, and tears +//! down. See `src/extensions/s3_stub_test.zig` for the hermetic equivalent +//! that runs in every `zig build test`. +//! +//! Verifies the whole path against a real implementation of the S3 API that +//! the stub only approximates: bucket creation, SigV4 accepted by a real +//! server, and the object actually readable back afterward with +//! byte-identical content. The stub test (s3_stub_test.zig) covers retry +//! mechanics under scripted failures; this test covers "does a real +//! S3-compatible server accept what we send and return what we sent". + +const std = @import("std"); +const proto = @import("proto"); +const s3 = @import("s3"); +const s3_dump_mod = @import("./s3_dump.zig"); +const S3Dump = s3_dump_mod.S3Dump; + +const testing = std.testing; + +fn requiredEnv(environ: std.process.Environ, name: []const u8) ?[]const u8 { + return environ.getPosix(name); +} + +const Env = struct { + access_key: []const u8, + secret_key: []const u8, + endpoint: []const u8, + bucket: []const u8, + + fn load() ?Env { + const environ = std.testing.environ; + return .{ + .access_key = requiredEnv(environ, "AWS_ACCESS_KEY_ID") orelse return null, + .secret_key = requiredEnv(environ, "AWS_SECRET_ACCESS_KEY") orelse return null, + .endpoint = requiredEnv(environ, "S3_ENDPOINT") orelse "http://127.0.0.1:9000", + .bucket = requiredEnv(environ, "S3_BUCKET") orelse "policy-zig-e2e", + }; + } +}; + +fn encodeTargetRef(allocator: std.mem.Allocator, kind: []const u8, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = kind, .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +fn encodeRecord(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + const msg: *const []const u8 = @ptrCast(@alignCast(record)); + try writer.writeAll(msg.*); +} + +test "e2e minio: real upload round-trips through a real S3-compatible server" { + const env = Env.load() orelse return error.SkipZigTest; + const allocator = testing.allocator; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + // Ensure the bucket exists (idempotent: MinIO returns 409 if it's already + // there, which z3's own test suite treats as success too). + var admin = try s3.S3Client.init(allocator, .{ + .access_key_id = env.access_key, + .secret_access_key = env.secret_key, + .endpoint = env.endpoint, + .virtual_host_style = false, + }, .{ .io = io }); + defer admin.deinit(); + var create_resp = try admin.createBucket(env.bucket, .{}); + defer create_resp.deinit(); + try testing.expect(create_resp.http_head.status == .ok or create_resp.http_head.status == .conflict); + + // Drive the extension exactly as the engine would: resolve a target, + // deliver two records, flush for real. + var dump = S3Dump.init(allocator, .{ .max_attempts = 1 }, .{ + .access_key_id = env.access_key, + .secret_access_key = env.secret_key, + }); + defer dump.deinit(); + + const target_json = try std.fmt.allocPrint( + allocator, + \\{{"endpoint": "{s}", "region": "us-east-1", "bucket": "{s}", "prefix": "e2e/", "force_path_style": true}} + , + .{ env.endpoint, env.bucket }, + ); + defer allocator.free(target_json); + try dump.addTarget(io, "minio", target_json); + + const ref = try encodeTargetRef(allocator, "s3", "minio"); + defer allocator.free(ref); + const slot = dump.resolve(io, .log, "e2e-policy", ref).?; + + const rec1: []const u8 = "{\"body\":\"first record\"}"; + const rec2: []const u8 = "{\"body\":\"second record\"}"; + dump.deliver(io, slot, @ptrCast(&rec1), encodeRecord); + dump.deliver(io, slot, @ptrCast(&rec2), encodeRecord); + + const result = dump.flush(io, .{ .force = true }); + try testing.expectEqual(@as(u32, 1), result.objects_uploaded); + try testing.expectEqual(@as(u32, 0), result.objects_failed); + try testing.expectEqual(@as(u64, 2), result.records_uploaded); + + // Verify against the real backend, not our own bookkeeping: list the + // prefix, fetch the object, and check the bytes really landed. + var listing = try admin.listObjects(env.bucket, .{ .prefix = "e2e/log/" }); + defer listing.deinit(); + // A fresh container (the task flow) holds exactly our object; a + // long-lived server may hold leftovers from direct runs, so find ours + // by key substring rather than assuming it's the only one. + const key = blk: { + for (listing.objects) |obj| { + if (std.mem.indexOf(u8, obj.key, "e2e-policy-") != null) break :blk obj.key; + } + return error.UploadedObjectNotFound; + }; + + var get_resp = try admin.getObject(env.bucket, key, .{}); + defer get_resp.deinit(); + try testing.expectEqual(std.http.Status.ok, get_resp.http_head.status); + try testing.expectEqualStrings( + "{\"body\":\"first record\"}\n{\"body\":\"second record\"}\n", + get_resp.body, + ); + + // Clean up so repeated runs don't accumulate objects. + var del_resp = try admin.deleteObject(env.bucket, key, .{}); + del_resp.deinit(); +} diff --git a/src/extensions/s3_stub_test.zig b/src/extensions/s3_stub_test.zig new file mode 100644 index 0000000..e2165e3 --- /dev/null +++ b/src/extensions/s3_stub_test.zig @@ -0,0 +1,262 @@ +//! End-to-end verification of the s3-dump upload path against an in-process +//! HTTP server that speaks just enough S3 — no docker, no external network, +//! runs in plain `zig build test`. Verifies the bytes on the wire: method, +//! path-style URL and key layout, SigV4 signing, the payload hash S3 uses for +//! integrity verification, and the requeue-then-retry durability path. +//! +//! For a real-storage smoke test against MinIO, see `task test:s3-e2e`. + +const std = @import("std"); +const proto = @import("proto"); +const s3_dump_mod = @import("./s3_dump.zig"); +const S3Dump = s3_dump_mod.S3Dump; + +const testing = std.testing; + +/// Minimal S3 stand-in: accepts connections, records every request (method, +/// target, raw head bytes, body), and answers each with the next scripted +/// status. Runs concurrently with the flush under test via io.concurrent. +const StubS3Server = struct { + const Received = struct { + method: std.http.Method, + target: []u8, + head: []u8, + body: []u8, + }; + + allocator: std.mem.Allocator, + statuses: []const std.http.Status, + listener: std.Io.net.Server, + port: u16, + received: std.ArrayList(Received) = .empty, + + fn start( + allocator: std.mem.Allocator, + io: std.Io, + statuses: []const std.http.Status, + ) !StubS3Server { + // std.Io exposes no getsockname, so probe a small fixed range + // instead of binding port 0. + var attempt: u16 = 0; + while (attempt < 32) : (attempt += 1) { + const port: u16 = 42741 + attempt; + const addr = try std.Io.net.IpAddress.parse("127.0.0.1", port); + const listener = addr.listen(io, .{}) catch |err| switch (err) { + error.AddressInUse => continue, + else => return err, + }; + return .{ + .allocator = allocator, + .statuses = statuses, + .listener = listener, + .port = port, + }; + } + return error.AddressInUse; + } + + fn deinit(self: *StubS3Server, io: std.Io) void { + defer self.* = undefined; + self.listener.deinit(io); + for (self.received.items) |r| { + self.allocator.free(r.target); + self.allocator.free(r.head); + self.allocator.free(r.body); + } + self.received.deinit(self.allocator); + } + + /// Serve until every scripted status has been sent. Handles both + /// keep-alive (several requests on one connection) and reconnects + /// (a new client per flush). + fn serve(self: *StubS3Server, io: std.Io) void { + var served: usize = 0; + outer: while (served < self.statuses.len) { + var stream = self.listener.accept(io) catch return; + defer stream.close(io); + var recv_buf: [64 * 1024]u8 = undefined; + var send_buf: [4 * 1024]u8 = undefined; + var conn_reader = stream.reader(io, &recv_buf); + var conn_writer = stream.writer(io, &send_buf); + var http_server = std.http.Server.init(&conn_reader.interface, &conn_writer.interface); + while (served < self.statuses.len) { + var request = http_server.receiveHead() catch continue :outer; + self.handleOne(&request, self.statuses[served]) catch continue :outer; + served += 1; + } + } + } + + fn handleOne( + self: *StubS3Server, + request: *std.http.Server.Request, + status: std.http.Status, + ) !void { + // Head pointers are invalidated once the body stream starts — copy first. + const method = request.head.method; + const target = try self.allocator.dupe(u8, request.head.target); + errdefer self.allocator.free(target); + const head = try self.allocator.dupe(u8, request.head_buffer); + errdefer self.allocator.free(head); + + var transfer_buf: [1024]u8 = undefined; + const body_reader = try request.readerExpectContinue(&transfer_buf); + const body = try body_reader.allocRemaining(self.allocator, .limited(1 << 20)); + errdefer self.allocator.free(body); + + try self.received.append(self.allocator, .{ + .method = method, + .target = target, + .head = head, + .body = body, + }); + try request.respond("", .{ .status = status }); + } + + fn headContains(head: []const u8, needle: []const u8) bool { + return std.ascii.indexOfIgnoreCase(head, needle) != null; + } +}; + +fn encodeTargetRef(allocator: std.mem.Allocator, kind: []const u8, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = kind, .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +fn targetJson(allocator: std.mem.Allocator, port: u16) ![]u8 { + return std.fmt.allocPrint( + allocator, + \\{{"endpoint": "http://127.0.0.1:{d}", "region": "us-east-1", "bucket": "waste", "prefix": "dumps/"}} + , + .{port}, + ); +} + +fn encodeRecord(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + const msg: *const []const u8 = @ptrCast(@alignCast(record)); + try writer.writeAll(msg.*); +} + +const test_credentials: S3Dump.Credentials = .{ + .access_key_id = "test-access-key", + .secret_access_key = "test-secret-key", +}; + +test "e2e stub: flush PUTs the signed ndjson batch under the sealed key" { + const allocator = testing.allocator; + // The shared test-runner io caps concurrency; the stub server needs a + // real thread, so build our own Threaded io like a consumer binary would. + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var stub = try StubS3Server.start(allocator, io, &.{.ok}); + defer stub.deinit(io); + var server_future = io.concurrent(StubS3Server.serve, .{ &stub, io }) catch + return error.SkipZigTest; + // If an assertion fires before the server saw its scripted requests, + // cancel instead of hanging in accept(); on the happy path (already + // awaited) this is a no-op. + defer server_future.cancel(io); + + var dump = S3Dump.init(allocator, .{ .max_attempts = 1 }, test_credentials); + defer dump.deinit(); + const tj = try targetJson(allocator, stub.port); + defer allocator.free(tj); + try dump.addTarget(io, "stub", tj); + + const ref = try encodeTargetRef(allocator, "s3", "stub"); + defer allocator.free(ref); + const slot = dump.resolve(io, .log, "dump-policy", ref).?; + + const rec1: []const u8 = "{\"body\":\"one\"}"; + const rec2: []const u8 = "{\"body\":\"two\"}"; + dump.deliver(io, slot, @ptrCast(&rec1), encodeRecord); + dump.deliver(io, slot, @ptrCast(&rec2), encodeRecord); + + const result = dump.flush(io, .{ .force = true }); + server_future.await(io); + + try testing.expectEqual(@as(u32, 1), result.objects_uploaded); + try testing.expectEqual(@as(u64, 2), result.records_uploaded); + try testing.expectEqual(@as(u32, 0), result.objects_failed); + // bytes_uploaded matches the exact body size; the successfully uploaded + // batch leaves nothing behind in the backlog gauge. + try testing.expectEqual(@as(u64, "{\"body\":\"one\"}\n{\"body\":\"two\"}\n".len), result.bytes_uploaded); + try testing.expectEqual(@as(usize, 0), result.backlog_bytes); + + try testing.expectEqual(@as(usize, 1), stub.received.items.len); + const req = stub.received.items[0]; + try testing.expectEqual(std.http.Method.PUT, req.method); + // Path-style URL: /{bucket}/{key} with the documented key layout. + try testing.expect(std.mem.startsWith(u8, req.target, "/waste/dumps/log/")); + try testing.expect(std.mem.endsWith(u8, req.target, ".ndjson")); + try testing.expect(std.mem.indexOf(u8, req.target, "dump-policy-") != null); + // The exact batch bytes arrived, one record per line. + try testing.expectEqualStrings("{\"body\":\"one\"}\n{\"body\":\"two\"}\n", req.body); + // SigV4-signed, with the payload hash S3 verifies end-to-end. + try testing.expect(StubS3Server.headContains(req.head, "AWS4-HMAC-SHA256")); + var hash: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(req.body, &hash, .{}); + const hex = std.fmt.bytesToHex(hash, .lower); + try testing.expect(StubS3Server.headContains(req.head, &hex)); + try testing.expect(StubS3Server.headContains(req.head, "application/x-ndjson")); +} + +test "e2e stub: 5xx requeues; the retry succeeds under the same key with the same bytes" { + const allocator = testing.allocator; + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var stub = try StubS3Server.start(allocator, io, &.{ .internal_server_error, .ok }); + defer stub.deinit(io); + var server_future = io.concurrent(StubS3Server.serve, .{ &stub, io }) catch + return error.SkipZigTest; + // If an assertion fires before the server saw its scripted requests, + // cancel instead of hanging in accept(); on the happy path (already + // awaited) this is a no-op. + defer server_future.cancel(io); + + var dump = S3Dump.init(allocator, .{ .max_attempts = 1 }, test_credentials); + defer dump.deinit(); + const tj = try targetJson(allocator, stub.port); + defer allocator.free(tj); + try dump.addTarget(io, "stub", tj); + + const ref = try encodeTargetRef(allocator, "s3", "stub"); + defer allocator.free(ref); + const slot = dump.resolve(io, .log, "dump-policy", ref).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(io, slot, @ptrCast(&rec), encodeRecord); + + // First flush: server answers 500 → batch requeued, nothing lost. The + // backlog gauge goes nonzero here — this is the metric an edge alerts on + // to catch a struggling destination before records_dropped climbs. + const r1 = dump.flush(io, .{ .force = true }); + try testing.expectEqual(@as(u32, 0), r1.objects_uploaded); + try testing.expectEqual(@as(u32, 1), r1.objects_failed); + try testing.expectEqual(@as(u32, 1), r1.objects_requeued); + try testing.expectEqual(@as(u64, 0), r1.records_dropped); + try testing.expectEqual(@as(u64, rec.len + 1), r1.bytes_failed); + try testing.expect(r1.backlog_bytes > 0); + + // Second flush: retry succeeds, and the gauge falls back to zero. + const r2 = dump.flush(io, .{ .force = true }); + server_future.await(io); + try testing.expectEqual(@as(u32, 1), r2.objects_uploaded); + try testing.expectEqual(@as(u64, 1), r2.records_uploaded); + try testing.expectEqual(@as(u64, rec.len + 1), r2.bytes_uploaded); + try testing.expectEqual(@as(usize, 0), r2.backlog_bytes); + + // Both attempts used the SAME key and carried identical bytes — the + // retry is an idempotent overwrite, never a duplicate object. + try testing.expectEqual(@as(usize, 2), stub.received.items.len); + try testing.expectEqualStrings(stub.received.items[0].target, stub.received.items[1].target); + try testing.expectEqualStrings(stub.received.items[0].body, stub.received.items[1].body); +} diff --git a/src/policy/matcher_index.zig b/src/policy/matcher_index.zig index 0f7e82c..b1658c7 100644 --- a/src/policy/matcher_index.zig +++ b/src/policy/matcher_index.zig @@ -878,6 +878,9 @@ fn IndexBuilder(comptime T: TelemetryType) type { /// policies are compiled without validation (used by direct build() /// callers such as unit tests). errors: ?*CompilationErrors, + /// Optional resolver for policy extensions (v1.6.0). When null, any + /// declared extension is skipped and reported (fail-open). + extension_resolver: ?policy_types.ExtensionResolver, patterns_by_key: std.HashMap( MatcherKeyT, PatternsPerKey, @@ -886,6 +889,7 @@ fn IndexBuilder(comptime T: TelemetryType) type { ), policy_info_list: std.ArrayList(PolicyInfo), typed_checks_list: std.ArrayList(TypedCheckT), + extension_bindings_list: std.ArrayList(policy_types.ExtensionBinding), path_storage: std.ArrayList([]const []const u8), policy_id_storage: std.ArrayList([]const u8), policy_index: PolicyIndex, @@ -903,12 +907,14 @@ fn IndexBuilder(comptime T: TelemetryType) type { temp_allocator: std.mem.Allocator, bus: *EventBus, errors: ?*CompilationErrors, + extension_resolver: ?policy_types.ExtensionResolver, ) Self { return .{ .allocator = allocator, .temp_allocator = temp_allocator, .bus = bus, .errors = errors, + .extension_resolver = extension_resolver, .patterns_by_key = std.HashMap( MatcherKeyT, PatternsPerKey, @@ -917,6 +923,7 @@ fn IndexBuilder(comptime T: TelemetryType) type { ).init(temp_allocator), .policy_info_list = .empty, .typed_checks_list = .empty, + .extension_bindings_list = .empty, .path_storage = .empty, .policy_id_storage = .empty, .policy_index = 0, @@ -997,6 +1004,47 @@ fn IndexBuilder(comptime T: TelemetryType) type { const keep_value = parseKeepValue(target); try self.storePolicyInfo(policy, target, keep_value, global_index); + try self.compileExtensions(policy); + } + + /// Compile the policy's extension declarations into bindings (v1.6.0). + /// Fail-open per spec rule 5: an extension that cannot be resolved (no + /// resolver wired, unsupported type/version, invalid config) is skipped + /// and reported WITHOUT invalidating the policy — its core + /// match/keep/transform still applies. Must run after storePolicyInfo + /// so the binding can reference the stored policy info. + fn compileExtensions(self: *Self, policy: *const Policy) !void { + if (policy.extensions.items.len == 0) return; + const info = &self.policy_info_list.items[self.policy_info_list.items.len - 1]; + for (policy.extensions.items) |*ext| { + const resolution: ?policy_types.ExtensionResolution = blk: { + const r = self.extension_resolver orelse break :blk null; + break :blk r.resolve(self.bus.io, r.ctx, T, policy.id, ext); + }; + const resolved = resolution orelse { + if (self.errors) |sink| { + try sink.addFmt( + @tagName(T) ++ ": extension \"{s}\" unsupported or invalid; skipped", + policy.id, + .{ext.type}, + ); + } + continue; + }; + // Config bytes go into the index's owned-string storage so the + // binding stays valid for the snapshot's lifetime regardless of + // registry-side policy memory churn (same treatment as ids). + const config_copy = try self.allocator.dupe(u8, ext.config); + try self.policy_id_storage.append(self.allocator, config_copy); + try self.extension_bindings_list.append(self.temp_allocator, .{ + .policy_index = info.index, + .policy_id = info.id, + .handler = resolved.handler, + .slot = resolved.slot, + .slices = policy_types.extensionSliceSet(ext.mode), + .config = config_copy, + }); + } } /// Record a compilation error for the policy being compiled: marks it @@ -1589,6 +1637,10 @@ fn IndexBuilder(comptime T: TelemetryType) type { const typed_checks = try self.allocator.dupe(TypedCheckT, self.typed_checks_list.items); // Free the builder's backing storage; the items are now in `typed_checks`. self.typed_checks_list.deinit(self.allocator); + const extension_bindings = try self.allocator.dupe( + policy_types.ExtensionBinding, + self.extension_bindings_list.items, + ); return IndexT{ .allocator = self.allocator, @@ -1597,6 +1649,7 @@ fn IndexBuilder(comptime T: TelemetryType) type { .policies_with_negation = policies_with_negation, .matcher_keys = matcher_keys, .typed_checks = typed_checks, + .extension_bindings = extension_bindings, .path_storage = self.path_storage, .policy_id_storage = self.policy_id_storage, .bus = self.bus, @@ -1621,6 +1674,7 @@ pub const LogMatcherIndex = struct { policies_with_negation: []PolicyIndex, matcher_keys: []LogMatcherKey, typed_checks: []TypedCheckType(.log), + extension_bindings: []policy_types.ExtensionBinding, path_storage: std.ArrayList([]const []const u8), policy_id_storage: std.ArrayList([]const u8), bus: *EventBus, @@ -1636,6 +1690,7 @@ pub const LogMatcherIndex = struct { bus: *EventBus, policies_slice: []const Policy, errors: ?*CompilationErrors, + extension_resolver: ?policy_types.ExtensionResolver, ) !LogMatcherIndex { const started_event: MatcherIndexBuildStarted = .{ .policy_count = policies_slice.len, @@ -1650,7 +1705,7 @@ pub const LogMatcherIndex = struct { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - var builder = IndexBuilder(.log).init(allocator, arena.allocator(), bus, errors); + var builder = IndexBuilder(.log).init(allocator, arena.allocator(), bus, errors, extension_resolver); for (policies_slice, 0..) |*policy, i| { try builder.processPolicy(policy, @intCast(i)); @@ -1700,8 +1755,13 @@ pub const LogMatcherIndex = struct { return self.typed_checks; } + pub fn getExtensionBindings(self: *const LogMatcherIndex) []const policy_types.ExtensionBinding { + return self.extension_bindings; + } + pub fn isEmpty(self: *const LogMatcherIndex) bool { - return self.matcher_keys.len == 0 and self.typed_checks.len == 0; + return self.matcher_keys.len == 0 and self.typed_checks.len == 0 and + self.extension_bindings.len == 0; } pub fn getDatabaseCount(self: *const LogMatcherIndex) usize { @@ -1747,6 +1807,7 @@ pub const LogMatcherIndex = struct { } } self.allocator.free(self.typed_checks); + self.allocator.free(self.extension_bindings); for (self.path_storage.items) |path| { for (path) |segment| { @@ -1779,6 +1840,7 @@ pub const MetricMatcherIndex = struct { policies_with_negation: []PolicyIndex, matcher_keys: []MetricMatcherKey, typed_checks: []TypedCheckType(.metric), + extension_bindings: []policy_types.ExtensionBinding, path_storage: std.ArrayList([]const []const u8), policy_id_storage: std.ArrayList([]const u8), bus: *EventBus, @@ -1790,6 +1852,7 @@ pub const MetricMatcherIndex = struct { bus: *EventBus, policies_slice: []const Policy, errors: ?*CompilationErrors, + extension_resolver: ?policy_types.ExtensionResolver, ) !MetricMatcherIndex { const started_event: MatcherIndexBuildStarted = .{ .policy_count = policies_slice.len, @@ -1804,7 +1867,7 @@ pub const MetricMatcherIndex = struct { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - var builder = IndexBuilder(.metric).init(allocator, arena.allocator(), bus, errors); + var builder = IndexBuilder(.metric).init(allocator, arena.allocator(), bus, errors, extension_resolver); for (policies_slice, 0..) |*policy, i| { try builder.processPolicy(policy, @intCast(i)); @@ -1854,8 +1917,13 @@ pub const MetricMatcherIndex = struct { return self.typed_checks; } + pub fn getExtensionBindings(self: *const MetricMatcherIndex) []const policy_types.ExtensionBinding { + return self.extension_bindings; + } + pub fn isEmpty(self: *const MetricMatcherIndex) bool { - return self.matcher_keys.len == 0 and self.typed_checks.len == 0; + return self.matcher_keys.len == 0 and self.typed_checks.len == 0 and + self.extension_bindings.len == 0; } pub fn getDatabaseCount(self: *const MetricMatcherIndex) usize { @@ -1896,6 +1964,7 @@ pub const MetricMatcherIndex = struct { } } self.allocator.free(self.typed_checks); + self.allocator.free(self.extension_bindings); for (self.path_storage.items) |path| { for (path) |segment| { @@ -1928,6 +1997,7 @@ pub const TraceMatcherIndex = struct { policies_with_negation: []PolicyIndex, matcher_keys: []TraceMatcherKey, typed_checks: []TypedCheckType(.trace), + extension_bindings: []policy_types.ExtensionBinding, path_storage: std.ArrayList([]const []const u8), policy_id_storage: std.ArrayList([]const u8), bus: *EventBus, @@ -1939,6 +2009,7 @@ pub const TraceMatcherIndex = struct { bus: *EventBus, policies_slice: []const Policy, errors: ?*CompilationErrors, + extension_resolver: ?policy_types.ExtensionResolver, ) !TraceMatcherIndex { const started_event: MatcherIndexBuildStarted = .{ .policy_count = policies_slice.len, @@ -1953,7 +2024,7 @@ pub const TraceMatcherIndex = struct { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - var builder = IndexBuilder(.trace).init(allocator, arena.allocator(), bus, errors); + var builder = IndexBuilder(.trace).init(allocator, arena.allocator(), bus, errors, extension_resolver); for (policies_slice, 0..) |*policy, i| { try builder.processPolicy(policy, @intCast(i)); @@ -2003,8 +2074,13 @@ pub const TraceMatcherIndex = struct { return self.typed_checks; } + pub fn getExtensionBindings(self: *const TraceMatcherIndex) []const policy_types.ExtensionBinding { + return self.extension_bindings; + } + pub fn isEmpty(self: *const TraceMatcherIndex) bool { - return self.matcher_keys.len == 0 and self.typed_checks.len == 0; + return self.matcher_keys.len == 0 and self.typed_checks.len == 0 and + self.extension_bindings.len == 0; } pub fn getDatabaseCount(self: *const TraceMatcherIndex) usize { @@ -2045,6 +2121,7 @@ pub const TraceMatcherIndex = struct { } } self.allocator.free(self.typed_checks); + self.allocator.free(self.extension_bindings); for (self.path_storage.items) |path| { for (path) |segment| { @@ -2387,7 +2464,7 @@ test "LogMatcherIndex: build empty" { var noop_bus: NoopEventBus = undefined; noop_bus.init(std.Options.debug_io); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{}, null, null); defer index.deinit(); try testing.expect(index.isEmpty()); @@ -2399,7 +2476,7 @@ test "MetricMatcherIndex: build empty" { var noop_bus: NoopEventBus = undefined; noop_bus.init(std.Options.debug_io); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{}, null, null); defer index.deinit(); try testing.expect(index.isEmpty()); @@ -2425,7 +2502,7 @@ test "LogMatcherIndex: build with single policy" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2462,7 +2539,7 @@ test "LogMatcherIndex: invalid regex policy is skipped (inert) and reported" { var errors = CompilationErrors.init(allocator); defer errors.deinit(); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, &errors); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, &errors, null); defer index.deinit(); // Inert: not compiled into the index at all. @@ -2501,7 +2578,7 @@ test "LogMatcherIndex: empty attribute path is skipped (inert) and reported" { var errors = CompilationErrors.init(allocator); defer errors.deinit(); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, &errors); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, &errors, null); defer index.deinit(); try testing.expectEqual(@as(usize, 0), index.getPolicyCount()); @@ -2548,7 +2625,7 @@ test "LogMatcherIndex: valid and invalid policies in one batch — valid still c var errors = CompilationErrors.init(allocator); defer errors.deinit(); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{ valid, bad_keep }, &errors); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{ valid, bad_keep }, &errors, null); defer index.deinit(); // Valid policy compiled; malformed-keep policy is inert and reported. @@ -2577,7 +2654,7 @@ test "LogMatcherIndex: invalid policy is inert even without an error sink" { }; defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expectEqual(@as(usize, 0), index.getPolicyCount()); @@ -2602,7 +2679,7 @@ test "MetricMatcherIndex: build with single policy" { }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2636,7 +2713,7 @@ test "LogMatcherIndex: build with keyed matchers" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2671,7 +2748,7 @@ test "LogMatcherIndex: negated matcher creates negated database" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2702,7 +2779,7 @@ test "LogMatcherIndex: scan database" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }); @@ -2739,7 +2816,7 @@ test "LogMatcherIndex: exists=true matcher is bucketed into exists_entries" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); // exists matchers no longer compile to Hyperscan patterns; they live in @@ -2780,7 +2857,7 @@ test "LogMatcherIndex: exists=false matcher creates negated pattern" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); // exists=false lives in exists_entries with negate=true; no Hyperscan DB. @@ -2825,7 +2902,7 @@ test "MetricMatcherIndex: metric_type with null match (implicit exists)" { }, }; - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); // The index should have registered one policy with one matcher @@ -2871,7 +2948,7 @@ test "TraceMatcherIndex: span_kind null match with second resource_attribute mat }, }; - var index = try TraceMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try TraceMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2914,7 +2991,7 @@ test "TraceMatcherIndex: span_kind with null match (implicit exists)" { }, }; - var index = try TraceMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try TraceMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2949,7 +3026,7 @@ test "MetricMatcherIndex: exists=true matcher is bucketed into exists_entries" { }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); // exists=true is dispatched via accessor.callExists, not Hyperscan. @@ -2988,7 +3065,7 @@ test "MetricMatcherIndex: metric_type exists is bucketed into exists_entries" { }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -3027,7 +3104,7 @@ test "MetricMatcherIndex: metric_type with regex pattern" { }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -3070,7 +3147,7 @@ test "MetricMatcherIndex: aggregation_temporality field creates Hyperscan databa }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -3130,7 +3207,7 @@ test "Mixed log and metric policies: each index only gets its type" { const policies = &[_]Policy{ log_policy, metric_policy }; // Log index should only have log policy - var log_index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), policies, null); + var log_index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), policies, null, null); defer log_index.deinit(); try testing.expectEqual(@as(usize, 1), log_index.getPolicyCount()); try testing.expectEqual(@as(usize, 1), log_index.getDatabaseCount()); @@ -3142,7 +3219,7 @@ test "Mixed log and metric policies: each index only gets its type" { try testing.expect(metric_in_log == null); // Metric index should only have metric policy - var metric_index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), policies, null); + var metric_index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), policies, null, null); defer metric_index.deinit(); try testing.expectEqual(@as(usize, 1), metric_index.getPolicyCount()); try testing.expectEqual(@as(usize, 1), metric_index.getDatabaseCount()); @@ -3235,7 +3312,7 @@ test "Log matcher with starts_with" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3273,7 +3350,7 @@ test "Log matcher with ends_with" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3311,7 +3388,7 @@ test "Log matcher with contains" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3352,7 +3429,7 @@ test "Log matcher with exact" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3391,7 +3468,7 @@ test "Log matcher with case_insensitive" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3433,7 +3510,7 @@ test "Log matcher with starts_with case_insensitive" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; diff --git a/src/policy/policy_engine.zig b/src/policy/policy_engine.zig index 9bd7a8d..8e33845 100644 --- a/src/policy/policy_engine.zig +++ b/src/policy/policy_engine.zig @@ -220,6 +220,12 @@ pub const EvaluateOptions = struct { /// Required — without it, rate limiting and unkeyed percentage sampling /// cannot make a real decision, so there is no safe default to degrade to. io: std.Io, + /// Extension dispatch sink (v1.6.0). When set and the snapshot compiled + /// extension bindings, each record is classified per binding + /// (kept/dropped/unmatched against the final keep outcome) and selected + /// records are handed to the sink after keep resolution, before + /// transforms. Null costs a single branch. + extension_sink: ?policy_types.ExtensionSink = null, }; // ============================================================================= @@ -373,6 +379,30 @@ pub const PolicyEngine = struct { // Record hit/miss stats using lock-free atomics self.recordMatchedPolicyStats(snapshot, &match_state); + // Extension dispatch (v1.6.0): classify this record for every compiled + // (policy, extension) binding against the final keep outcome and hand + // selected records to the sink. Runs after keep resolution and BEFORE + // transforms (and before the drop return below — `mode: dropped` is + // the flagship use), so sinks observe pre-transform records. The + // record is borrowed only for the duration of each deliver call. + if (options.extension_sink) |sink| { + const bindings = index.getExtensionBindings(); + const kept = match_state.decision != .drop; + for (bindings) |*binding| { + const info = index.getPolicyByIndex(binding.policy_index) orelse continue; + // A disabled policy's extensions are inactive with it. + if (!info.enabled) continue; + const matched = scan_state.match_counts[binding.policy_index] == info.required_match_count; + const slice: policy_types.ExtensionSlice = if (matched) + (if (kept) .kept else .dropped) + else + .unmatched; + if (binding.slices.contains(slice)) { + sink.deliver(sink.ctx, options.io, T, ctx, binding, slice); + } + } + } + if (match_state.decision == .drop) { return .dropped; } @@ -5290,3 +5320,859 @@ test "PolicyEngine: mixed signal policies scope stats to own signal type" { const trace_stats = snapshot.getStats(2).?; try testing.expectEqual(@as(i64, 1), trace_stats.hits.load(.monotonic)); } + +// ============================================================================= +// Extension dispatch tests (v1.6.0) +// ============================================================================= + +/// Test resolver: accepts extensions of type "test/record", assigns handler 7 +/// and sequential slots; anything else is unsupported (returns null). +const TestExtensionResolver = struct { + next_slot: u32 = 0, + + fn resolveFn( + io: std.Io, + ctx_ptr: *anyopaque, + signal: TelemetryType, + policy_id: []const u8, + extension: *const proto.policy.Extension, + ) ?policy_types.ExtensionResolution { + _ = io; + _ = signal; + _ = policy_id; + const self: *TestExtensionResolver = @ptrCast(@alignCast(ctx_ptr)); + if (!std.mem.eql(u8, extension.type, "test/record")) return null; + defer self.next_slot += 1; + return .{ .handler = 7, .slot = self.next_slot }; + } + + fn resolver(self: *TestExtensionResolver) policy_types.ExtensionResolver { + return .{ .ctx = self, .resolve = resolveFn }; + } +}; + +/// Test sink: records every delivery. Policy ids are duped because they are +/// only guaranteed valid for the snapshot's lifetime. +const RecordingExtensionSink = struct { + const Delivery = struct { + policy_id: []const u8, + handler: u8, + slot: u32, + slice: policy_types.ExtensionSlice, + }; + + allocator: std.mem.Allocator, + deliveries: std.ArrayList(Delivery) = .empty, + + fn deinit(self: *RecordingExtensionSink) void { + defer self.* = undefined; + for (self.deliveries.items) |d| self.allocator.free(d.policy_id); + self.deliveries.deinit(self.allocator); + } + + fn deliverFn( + ctx_ptr: *anyopaque, + io: ?std.Io, + signal: TelemetryType, + record: *const anyopaque, + binding: *const policy_types.ExtensionBinding, + slice: policy_types.ExtensionSlice, + ) void { + _ = io; + _ = signal; + _ = record; + const self: *RecordingExtensionSink = @ptrCast(@alignCast(ctx_ptr)); + const id = self.allocator.dupe(u8, binding.policy_id) catch return; + self.deliveries.append(self.allocator, .{ + .policy_id = id, + .handler = binding.handler, + .slot = binding.slot, + .slice = slice, + }) catch self.allocator.free(id); + } + + fn sink(self: *RecordingExtensionSink) policy_types.ExtensionSink { + return .{ .ctx = self, .deliver = deliverFn }; + } +}; + +fn makeExtensionLogPolicy( + allocator: std.mem.Allocator, + id: []const u8, + pattern: []const u8, + keep: []const u8, + ext_type: []const u8, + mode: proto.policy.ExtensionMode, +) !Policy { + var policy: Policy = .{ + .id = try allocator.dupe(u8, id), + .name = try allocator.dupe(u8, id), + .enabled = true, + .target = .{ .log = .{ + .keep = try allocator.dupe(u8, keep), + } }, + }; + try policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, pattern) }, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, ext_type), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "cfg-bytes"), + .mode = mode, + }); + return policy; +} + +fn evalTestLogWithSink( + engine: *const PolicyEngine, + ctx: *TestLogContext, + policy_id_buf: [][]const u8, + sink: policy_types.ExtensionSink, +) PolicyResult { + return engine.evaluate(.log, &TestLogContext.accessor, ctx, policy_id_buf, .{ + .io = std.Options.debug_io, + .extension_sink = sink, + }); +} + +test "extensions: mode dropped delivers matched-and-dropped records only" { + const allocator = testing.allocator; + + var policy = try makeExtensionLogPolicy( + allocator, + "dump-waste", + "^drop", + "none", + "test/record", + .EXTENSION_MODE_DROPPED, + ); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + // Matched and dropped: delivered with slice .dropped. + var log1: TestLogContext = .{ .message = "drop me" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(FilterDecision.drop, r1.decision); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); + try testing.expectEqual(policy_types.ExtensionSlice.dropped, rec.deliveries.items[0].slice); + try testing.expectEqualStrings("dump-waste", rec.deliveries.items[0].policy_id); + try testing.expectEqual(@as(u8, 7), rec.deliveries.items[0].handler); + + // Unmatched: mode dropped does not select unmatched records. + var log2: TestLogContext = .{ .message = "keep me" }; + _ = evalTestLogWithSink(&engine, &log2, &policy_id_buf, rec.sink()); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); +} + +test "extensions: default mode (unspecified) selects matched, kept and dropped" { + const allocator = testing.allocator; + + var policy = try makeExtensionLogPolicy( + allocator, + "observe", + "payment", + "all", + "test/record", + .EXTENSION_MODE_UNSPECIFIED, + ); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + // Matched and kept → delivered as .kept. + var log1: TestLogContext = .{ .message = "payment ok" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(FilterDecision.keep, r1.decision); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); + try testing.expectEqual(policy_types.ExtensionSlice.kept, rec.deliveries.items[0].slice); + + // Unmatched → not selected by MATCHED. + var log2: TestLogContext = .{ .message = "healthcheck" }; + _ = evalTestLogWithSink(&engine, &log2, &policy_id_buf, rec.sink()); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); +} + +test "extensions: mode all delivers unmatched records" { + const allocator = testing.allocator; + + var policy = try makeExtensionLogPolicy(allocator, "mirror", "payment", "all", "test/record", .EXTENSION_MODE_ALL); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + var log1: TestLogContext = .{ .message = "healthcheck" }; + _ = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); + try testing.expectEqual(policy_types.ExtensionSlice.unmatched, rec.deliveries.items[0].slice); +} + +test "extensions: dropped slice reflects final outcome across policies" { + const allocator = testing.allocator; + + // Policy A drops payment logs; policy B keeps them but wants the waste. + // B's extension must see the record as .dropped because the FINAL pipeline + // outcome (most restrictive across all policies) is drop. + var drop_policy: Policy = .{ + .id = try allocator.dupe(u8, "a-drop"), + .name = try allocator.dupe(u8, "a-drop"), + .enabled = true, + .target = .{ .log = .{ .keep = try allocator.dupe(u8, "none") } }, + }; + try drop_policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "payment") }, + }); + defer drop_policy.deinit(allocator); + + var dump_policy = try makeExtensionLogPolicy( + allocator, + "b-dump", + "payment", + "all", + "test/record", + .EXTENSION_MODE_DROPPED, + ); + defer dump_policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{ drop_policy, dump_policy }, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + var log1: TestLogContext = .{ .message = "payment failed" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(FilterDecision.drop, r1.decision); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); + try testing.expectEqualStrings("b-dump", rec.deliveries.items[0].policy_id); + try testing.expectEqual(policy_types.ExtensionSlice.dropped, rec.deliveries.items[0].slice); +} + +test "extensions: unsupported type is skipped fail-open and reported" { + const allocator = testing.allocator; + + var policy = try makeExtensionLogPolicy( + allocator, + "unknown-ext", + "^drop", + "none", + "vendor/unknown", + .EXTENSION_MODE_DROPPED, + ); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + // The unsupported extension is reported via the compile-error channel... + try testing.expect(registry.policy_errors.count() == 1); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + // ...but the policy's core keep semantics still apply (fail-open), and no + // delivery happens. + var log1: TestLogContext = .{ .message = "drop me" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(FilterDecision.drop, r1.decision); + try testing.expectEqual(@as(usize, 0), rec.deliveries.items.len); +} + +test "extensions: sink observes pre-transform record values" { + const allocator = testing.allocator; + + var transform: proto.policy.LogTransform = .{}; + try transform.redact.append(allocator, .{ + .field = .{ .log_attribute = try testMakeAttrPath(allocator, "service") }, + .replacement = try allocator.dupe(u8, "[REDACTED]"), + }); + + var policy: Policy = .{ + .id = try allocator.dupe(u8, "redact-and-dump"), + .name = try allocator.dupe(u8, "redact-and-dump"), + .enabled = true, + .target = .{ .log = .{ + .keep = try allocator.dupe(u8, "all"), + .transform = transform, + } }, + }; + try policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "sensitive") }, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, "test/record"), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, ""), + .mode = .EXTENSION_MODE_KEPT, + }); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + + // Sink that snapshots the record's `service` value at delivery time. + const ObservingSink = struct { + observed: ?[]const u8 = null, + buf: [64]u8 = undefined, + + fn deliverFn( + ctx_ptr: *anyopaque, + io: ?std.Io, + signal: TelemetryType, + record: *const anyopaque, + binding: *const policy_types.ExtensionBinding, + slice: policy_types.ExtensionSlice, + ) void { + _ = io; + _ = signal; + _ = binding; + _ = slice; + const Self = @This(); + const self: *Self = @ptrCast(@alignCast(ctx_ptr)); + const rec_ctx: *const MutableTestLogContext = @ptrCast(@alignCast(record)); + const service = rec_ctx.service orelse return; + const n = @min(service.len, self.buf.len); + @memcpy(self.buf[0..n], service[0..n]); + self.observed = self.buf[0..n]; + } + }; + var obs: ObservingSink = .{}; + + var ctx = MutableTestLogContext.init(allocator); + defer ctx.deinit(); + ctx.message = "sensitive data here"; + ctx.service = "secret-service"; + + var policy_id_buf: [16][]const u8 = undefined; + const result = engine.evaluate(.log, &MutableTestLogContext.accessor, &ctx, &policy_id_buf, .{ + .io = std.Options.debug_io, + .extension_sink = .{ .ctx = &obs, .deliver = ObservingSink.deliverFn }, + }); + + try testing.expectEqual(FilterDecision.keep, result.decision); + // The sink saw the pre-transform value... + try testing.expectEqualStrings("secret-service", obs.observed.?); + // ...and the transform still applied afterwards. + try testing.expectEqualStrings("[REDACTED]", ctx.service.?); +} + +// ============================================================================= +// Mock extension: spec-conformance tests (v1.6.0 §Extensions) +// ============================================================================= + +/// A full mock extension implementation, acting as both resolver and sink the +/// way a real handler module would: it declares a type id, gates on semver +/// major, validates config at resolve time, and records every delivery. Used +/// to verify the engine's dispatch behavior matches the spec. +const MockExtension = struct { + const type_id = "test/mock"; + const handler_id: u8 = 42; + + const Delivery = struct { + policy_id: []const u8, + signal: TelemetryType, + slice: policy_types.ExtensionSlice, + config: []const u8, + slot: u32, + }; + + allocator: std.mem.Allocator, + deliveries: std.ArrayList(Delivery) = .empty, + resolve_rejections: u32 = 0, + next_slot: u32 = 0, + + fn deinit(self: *MockExtension) void { + defer self.* = undefined; + for (self.deliveries.items) |d| { + self.allocator.free(d.policy_id); + self.allocator.free(d.config); + } + self.deliveries.deinit(self.allocator); + } + + fn resolver(self: *MockExtension) policy_types.ExtensionResolver { + return .{ .ctx = self, .resolve = resolveFn }; + } + + fn sink(self: *MockExtension) policy_types.ExtensionSink { + return .{ .ctx = self, .deliver = deliverFn }; + } + + fn resolveFn( + io: std.Io, + ctx_ptr: *anyopaque, + signal: TelemetryType, + policy_id: []const u8, + extension: *const proto.policy.Extension, + ) ?policy_types.ExtensionResolution { + _ = io; + _ = signal; + _ = policy_id; + const self: *MockExtension = @ptrCast(@alignCast(ctx_ptr)); + // Spec rule 4: implementations declare which types they support. + if (!std.mem.eql(u8, extension.type, type_id)) { + self.resolve_rejections += 1; + return null; + } + // Semver gate: only major 1 of this mock's schema is supported. + const version = std.SemanticVersion.parse(extension.version) catch { + self.resolve_rejections += 1; + return null; + }; + if (version.major != 1) { + self.resolve_rejections += 1; + return null; + } + // Config validation at compile time (spec: reject unsatisfiable + // config; the engine must skip the extension fail-open). + if (std.mem.eql(u8, extension.config, "invalid")) { + self.resolve_rejections += 1; + return null; + } + defer self.next_slot += 1; + return .{ .handler = handler_id, .slot = self.next_slot }; + } + + fn deliverFn( + ctx_ptr: *anyopaque, + io: ?std.Io, + signal: TelemetryType, + record: *const anyopaque, + binding: *const policy_types.ExtensionBinding, + slice: policy_types.ExtensionSlice, + ) void { + _ = io; + _ = record; + const self: *MockExtension = @ptrCast(@alignCast(ctx_ptr)); + std.debug.assert(binding.handler == handler_id); + const id = self.allocator.dupe(u8, binding.policy_id) catch return; + const config = self.allocator.dupe(u8, binding.config) catch { + self.allocator.free(id); + return; + }; + self.deliveries.append(self.allocator, .{ + .policy_id = id, + .signal = signal, + .slice = slice, + .config = config, + .slot = binding.slot, + }) catch { + self.allocator.free(id); + self.allocator.free(config); + }; + } + + fn count(self: *const MockExtension, policy_id: []const u8, slice: policy_types.ExtensionSlice) usize { + var n: usize = 0; + for (self.deliveries.items) |d| { + if (d.slice == slice and std.mem.eql(u8, d.policy_id, policy_id)) n += 1; + } + return n; + } + + fn total(self: *const MockExtension, policy_id: []const u8) usize { + var n: usize = 0; + for (self.deliveries.items) |d| { + if (std.mem.eql(u8, d.policy_id, policy_id)) n += 1; + } + return n; + } +}; + +test "mock extension: full mode x record-class dispatch matrix (spec Extension Input table)" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + // One policy per mode, all matching "payment" with keep "all", plus a + // separate drop policy so a record can be removed by a *different* + // policy's keep (the spec's final-pipeline-outcome semantics). + var policies: [6]Policy = undefined; + var initialized: usize = 0; + defer for (policies[0..initialized]) |*p| p.deinit(allocator); + + policies[0] = .{ + .id = try allocator.dupe(u8, "0-drop-declined"), + .name = try allocator.dupe(u8, "0-drop-declined"), + .enabled = true, + .target = .{ .log = .{ .keep = try allocator.dupe(u8, "none") } }, + }; + try policies[0].target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "declined") }, + }); + initialized = 1; + + const mode_policies = [_]struct { id: []const u8, mode: proto.policy.ExtensionMode }{ + .{ .id = "mode-kept", .mode = .EXTENSION_MODE_KEPT }, + .{ .id = "mode-dropped", .mode = .EXTENSION_MODE_DROPPED }, + .{ .id = "mode-unmatched", .mode = .EXTENSION_MODE_UNMATCHED }, + .{ .id = "mode-matched", .mode = .EXTENSION_MODE_MATCHED }, + .{ .id = "mode-all", .mode = .EXTENSION_MODE_ALL }, + }; + for (mode_policies, 1..) |mp, i| { + policies[i] = try makeExtensionLogPolicy( + allocator, + mp.id, + "payment", + "all", + MockExtension.type_id, + mp.mode, + ); + initialized = i + 1; + } + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&policies, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + // Record class 1: matches the mode policies, kept. + var kept_log: TestLogContext = .{ .message = "payment ok" }; + const r1 = evalTestLogWithSink(&engine, &kept_log, &policy_id_buf, mock.sink()); + try testing.expectEqual(FilterDecision.keep, r1.decision); + + // Record class 2: matches the mode policies AND the drop policy — the + // final outcome is drop, caused by a different policy. + var dropped_log: TestLogContext = .{ .message = "payment declined" }; + const r2 = evalTestLogWithSink(&engine, &dropped_log, &policy_id_buf, mock.sink()); + try testing.expectEqual(FilterDecision.drop, r2.decision); + + // Record class 3: matches nothing. + var unmatched_log: TestLogContext = .{ .message = "healthcheck" }; + const r3 = evalTestLogWithSink(&engine, &unmatched_log, &policy_id_buf, mock.sink()); + try testing.expectEqual(FilterDecision.unset, r3.decision); + + // The spec's Extension Input table, row by row: + // kept: matched and survived keep. + try testing.expectEqual(@as(usize, 1), mock.count("mode-kept", .kept)); + try testing.expectEqual(@as(usize, 1), mock.total("mode-kept")); + // dropped: matched but removed by the final keep outcome. + try testing.expectEqual(@as(usize, 1), mock.count("mode-dropped", .dropped)); + try testing.expectEqual(@as(usize, 1), mock.total("mode-dropped")); + // unmatched: did not match the policy's matchers. + try testing.expectEqual(@as(usize, 1), mock.count("mode-unmatched", .unmatched)); + try testing.expectEqual(@as(usize, 1), mock.total("mode-unmatched")); + // matched = kept + dropped, never unmatched. + try testing.expectEqual(@as(usize, 1), mock.count("mode-matched", .kept)); + try testing.expectEqual(@as(usize, 1), mock.count("mode-matched", .dropped)); + try testing.expectEqual(@as(usize, 2), mock.total("mode-matched")); + // all = every record of the signal. + try testing.expectEqual(@as(usize, 1), mock.count("mode-all", .kept)); + try testing.expectEqual(@as(usize, 1), mock.count("mode-all", .dropped)); + try testing.expectEqual(@as(usize, 1), mock.count("mode-all", .unmatched)); + try testing.expectEqual(@as(usize, 8), mock.deliveries.items.len); +} + +test "mock extension: multiple extensions per policy, opaque config routed per declaration" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + // Spec rule 1: a policy MAY declare multiple extensions. Each has its own + // mode and its own opaque config. + var policy = try makeExtensionLogPolicy( + allocator, + "multi-ext", + "payment", + "all", + MockExtension.type_id, + .EXTENSION_MODE_KEPT, + ); + defer policy.deinit(allocator); + // Overwrite first extension's config, then add a second extension. + allocator.free(@constCast(policy.extensions.items[0].config)); + policy.extensions.items[0].config = try allocator.dupe(u8, "cfg-A"); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.2.3"), + .config = try allocator.dupe(u8, "cfg-B"), + .mode = .EXTENSION_MODE_DROPPED, + }); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + // Kept record: only the mode-kept extension (cfg-A) fires. + var log1: TestLogContext = .{ .message = "payment ok" }; + _ = evalTestLogWithSink(&engine, &log1, &policy_id_buf, mock.sink()); + try testing.expectEqual(@as(usize, 1), mock.deliveries.items.len); + try testing.expectEqualStrings("cfg-A", mock.deliveries.items[0].config); + try testing.expectEqual(policy_types.ExtensionSlice.kept, mock.deliveries.items[0].slice); + + // Re-evaluating delivers through the same binding with a stable slot. + const slot_a = mock.deliveries.items[0].slot; + var log2: TestLogContext = .{ .message = "payment ok" }; + _ = evalTestLogWithSink(&engine, &log2, &policy_id_buf, mock.sink()); + try testing.expectEqual(@as(usize, 2), mock.deliveries.items.len); + try testing.expectEqual(slot_a, mock.deliveries.items[1].slot); +} + +test "mock extension: version and config validation reject fail-open at compile time" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + // Three extensions: unsupported major, invalid config, and a good one. + var policy = try makeExtensionLogPolicy( + allocator, + "gate-ext", + "^drop", + "none", + MockExtension.type_id, + .EXTENSION_MODE_DROPPED, + ); + defer policy.deinit(allocator); + allocator.free(@constCast(policy.extensions.items[0].version)); + policy.extensions.items[0].version = try allocator.dupe(u8, "2.0.0"); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "invalid"), + .mode = .EXTENSION_MODE_DROPPED, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "good"), + .mode = .EXTENSION_MODE_DROPPED, + }); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + // Both rejects were reported under the policy's id... + try testing.expectEqual(@as(u32, 2), mock.resolve_rejections); + try testing.expectEqual(@as(usize, 2), registry.policy_errors.get("gate-ext").?.items.len); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + // ...the policy's core keep still applies (fail-open), and only the valid + // extension fires. + var log1: TestLogContext = .{ .message = "drop me" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, mock.sink()); + try testing.expectEqual(FilterDecision.drop, r1.decision); + try testing.expectEqual(@as(usize, 1), mock.deliveries.items.len); + try testing.expectEqualStrings("good", mock.deliveries.items[0].config); +} + +test "mock extension: dispatch MUST NOT change the evaluation outcome" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + // keep-all policy with a transform AND a mode-all extension: the result + // with a sink wired must be identical to the result without one, and the + // transform must still apply. + var transform: proto.policy.LogTransform = .{}; + try transform.redact.append(allocator, .{ + .field = .{ .log_attribute = try testMakeAttrPath(allocator, "service") }, + .replacement = try allocator.dupe(u8, "[REDACTED]"), + }); + var policy: Policy = .{ + .id = try allocator.dupe(u8, "redact-observed"), + .name = try allocator.dupe(u8, "redact-observed"), + .enabled = true, + .target = .{ .log = .{ + .keep = try allocator.dupe(u8, "all"), + .transform = transform, + } }, + }; + try policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "sensitive") }, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "cfg"), + .mode = .EXTENSION_MODE_ALL, + }); + defer policy.deinit(allocator); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var with_sink = MutableTestLogContext.init(allocator); + defer with_sink.deinit(); + with_sink.message = "sensitive data"; + with_sink.service = "secret-service"; + const r_with = engine.evaluate(.log, &MutableTestLogContext.accessor, &with_sink, &policy_id_buf, .{ + .io = std.Options.debug_io, + .extension_sink = mock.sink(), + }); + + var without_sink = MutableTestLogContext.init(allocator); + defer without_sink.deinit(); + without_sink.message = "sensitive data"; + without_sink.service = "secret-service"; + var policy_id_buf2: [16][]const u8 = undefined; + const r_without = engine.evaluate(.log, &MutableTestLogContext.accessor, &without_sink, &policy_id_buf2, .{ + .io = std.Options.debug_io, + }); + + // Identical decision, matches, and transform effects either way. + try testing.expectEqual(r_without.decision, r_with.decision); + try testing.expectEqual(r_without.was_transformed, r_with.was_transformed); + try testing.expectEqual(r_without.matched_policy_ids.len, r_with.matched_policy_ids.len); + try testing.expectEqualStrings("[REDACTED]", with_sink.service.?); + try testing.expectEqualStrings("[REDACTED]", without_sink.service.?); + try testing.expectEqual(@as(usize, 1), mock.deliveries.items.len); +} + +test "mock extension: disabled policy's extensions are inactive" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + var policy = try makeExtensionLogPolicy( + allocator, + "disabled-ext", + "payment", + "all", + MockExtension.type_id, + .EXTENSION_MODE_ALL, + ); + policy.enabled = false; + defer policy.deinit(allocator); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var log1: TestLogContext = .{ .message = "payment ok" }; + _ = evalTestLogWithSink(&engine, &log1, &policy_id_buf, mock.sink()); + try testing.expectEqual(@as(usize, 0), mock.deliveries.items.len); +} + +test "mock extension: metric policies dispatch with the metric signal" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + var policy: Policy = .{ + .id = try allocator.dupe(u8, "metric-ext"), + .name = try allocator.dupe(u8, "metric-ext"), + .enabled = true, + .target = .{ .metric = .{ .keep = true } }, + }; + try policy.target.?.metric.match.append(allocator, .{ + .field = .{ .metric_field = .METRIC_FIELD_NAME }, + .match = .{ .regex = try allocator.dupe(u8, "^cpu") }, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "cfg"), + .mode = .EXTENSION_MODE_MATCHED, + }); + defer policy.deinit(allocator); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var metric: TestMetricContext = .{ .name = "cpu.usage" }; + const result = engine.evaluate(.metric, &TestMetricContext.accessor, &metric, &policy_id_buf, .{ + .io = std.Options.debug_io, + .extension_sink = mock.sink(), + }); + try testing.expectEqual(FilterDecision.keep, result.decision); + try testing.expectEqual(@as(usize, 1), mock.deliveries.items.len); + try testing.expectEqual(TelemetryType.metric, mock.deliveries.items[0].signal); + try testing.expectEqual(policy_types.ExtensionSlice.kept, mock.deliveries.items[0].slice); +} diff --git a/src/policy/provider.zig b/src/policy/provider.zig index 45b5966..c0552a0 100644 --- a/src/policy/provider.zig +++ b/src/policy/provider.zig @@ -34,6 +34,33 @@ pub const StatsCollector = struct { } }; +/// Extension sync plumbing (spec v1.6.0), implemented outside policy_zig by +/// the extensions module: capability advertisement for sync requests and +/// routing of broadcast extension configs from responses. A fn-pointer seam +/// like `StatsCollector` — not a vtable of behaviors, just the two crossings. +/// +/// Handed to a provider by the registry at subscribe time, the same way +/// `StatsCollector` is. Providers with no control plane (file, testing) accept +/// the call and ignore it: there is nobody to advertise capabilities to and no +/// broadcast channel to receive from. +pub const ExtensionSyncHooks = struct { + ctx: *anyopaque, + /// Build ClientMetadata.supported_extensions. Allocate from `arena`. + capabilities: *const fn ( + io: std.Io, + ctx: *anyopaque, + arena: std.mem.Allocator, + ) anyerror![]proto.policy.ExtensionCapability, + /// Receive SyncResponse.extension_configs. Called before policies are + /// handed to the registry, so broadcast targets exist by the time the + /// snapshot compiles extension bindings against them. + apply_configs: *const fn ( + io: std.Io, + ctx: *anyopaque, + configs: []const proto.policy.ExtensionConfig, + ) void, +}; + /// Update notification sent by providers to subscribers pub const PolicyUpdate = struct { policies: []const Policy, diff --git a/src/policy/provider_http.zig b/src/policy/provider_http.zig index 4bdaf2f..a335c68 100644 --- a/src/policy/provider_http.zig +++ b/src/policy/provider_http.zig @@ -42,6 +42,12 @@ const HttpSyncStatusReported = struct { id: []const u8, match_hits: i64, match_m const HttpFetchStarted = struct {}; const HttpFetchCompleted = struct {}; +/// Extension sync plumbing lives in the generic provider layer now (it's +/// handed to every provider variant by the registry, like `StatsCollector`). +/// Re-exported here so existing `HttpProvider.ExtensionSyncHooks` references +/// keep resolving. +pub const ExtensionSyncHooks = policy_provider.ExtensionSyncHooks; + /// HTTP-based policy provider that polls a remote endpoint pub const HttpProvider = struct { allocator: std.mem.Allocator, @@ -73,6 +79,9 @@ pub const HttpProvider = struct { // each sync to obtain the per-policy hit/miss/error rows to report. stats_collector: ?StatsCollector, + // Extension sync plumbing (v1.6.0), set via setExtensionSyncHooks. + extension_hooks: ?ExtensionSyncHooks, + pub const Config = struct { id: []const u8, url: []const u8, @@ -133,6 +142,7 @@ pub const HttpProvider = struct { .sync_state_mutex = .init, .bus = bus, .stats_collector = null, + .extension_hooks = null, }; return self; @@ -143,6 +153,11 @@ pub const HttpProvider = struct { return self.id; } + /// Wire extension sync plumbing (v1.6.0). Call before start(). + pub fn setExtensionSyncHooks(self: *HttpProvider, hooks: ExtensionSyncHooks) void { + self.extension_hooks = hooks; + } + /// Record the hash from a successful sync. /// This hash will be sent in subsequent sync requests. pub fn recordSyncedHash(self: *HttpProvider, hash: []const u8) !void { @@ -275,6 +290,12 @@ pub const HttpProvider = struct { self.bus.info(event); } + // Route broadcast extension configs first, so targets they carry are + // known before the registry compiles the policies that reference them. + if (self.extension_hooks) |hooks| { + hooks.apply_configs(self.bus.io, hooks.ctx, response.extension_configs.items); + } + // Notify callback with policies from response if (self.callback) |cb| { try cb.call(.{ @@ -354,9 +375,20 @@ pub const HttpProvider = struct { last_hash = self.last_successful_hash orelse &.{}; } + // Extension capability advertisement (v1.6.0). Failure here must not + // block a sync: advertise nothing instead. + const supported_extensions: []proto.policy.ExtensionCapability = if (self.extension_hooks) |hooks| + hooks.capabilities(self.bus.io, hooks.ctx, temp_allocator) catch &.{} + else + &.{}; + // Create SyncRequest with the new structure const sync_request: SyncRequest = .{ .client_metadata = .{ + .supported_extensions = .{ + .items = supported_extensions, + .capacity = supported_extensions.len, + }, .supported_policy_stages = .{ .items = @constCast(supported_policy_stages), .capacity = supported_policy_stages.len, diff --git a/src/policy/registry.zig b/src/policy/registry.zig index a5f9c7b..e59d8be 100644 --- a/src/policy/registry.zig +++ b/src/policy/registry.zig @@ -250,6 +250,14 @@ pub const PolicyRegistry = struct { // Event bus for observability bus: *EventBus, + // Optional extension resolver (v1.6.0), consulted at snapshot compile + // time to resolve/validate each policy's extension declarations. + extension_resolver: ?policy_types.ExtensionResolver, + + // Optional extension sync hooks (v1.6.0), pushed to each provider on + // subscribe for capability advertisement + broadcast-config routing. + extension_sync_hooks: ?policy_provider.ExtensionSyncHooks, + /// Subscription context for provider callbacks. /// Allocated with stable address so the callback pointer remains valid. const Subscription = struct { @@ -284,13 +292,32 @@ pub const PolicyRegistry = struct { .policy_errors = .empty, .subscriptions = .empty, .bus = bus, + .extension_resolver = null, + .extension_sync_hooks = null, }; } + /// Wire an extension resolver (v1.6.0), consulted at snapshot compile + /// time. Call before providers deliver policies; takes effect on the next + /// recompile. Without one, declared extensions are skipped (fail-open) + /// and reported via PolicySyncStatus.errors. + pub fn setExtensionResolver(self: *PolicyRegistry, resolver: policy_types.ExtensionResolver) void { + self.extension_resolver = resolver; + } + + /// Wire extension sync hooks (v1.6.0). Stored here and pushed to each + /// provider on `subscribe`, so the consumer wires them once instead of + /// per-provider. Call before `subscribe`; providers subscribed earlier + /// won't retroactively receive them (matches the stats-collector contract). + pub fn setExtensionSyncHooks(self: *PolicyRegistry, hooks: policy_provider.ExtensionSyncHooks) void { + self.extension_sync_hooks = hooks; + } + /// Subscribe a provider to the registry in one step: hand it our stats - /// collector (so it can pull per-policy hit/miss/error rows before each sync) - /// and subscribe to its policy updates with an internal callback that feeds - /// updates back into the registry. + /// collector (so it can pull per-policy hit/miss/error rows before each sync), + /// hand it the extension sync hooks if any are wired, and subscribe to its + /// policy updates with an internal callback that feeds updates back into the + /// registry. pub fn subscribe(self: *PolicyRegistry, prov: Provider) !void { const sub = try self.allocator.create(Subscription); errdefer self.allocator.destroy(sub); @@ -304,6 +331,10 @@ pub const PolicyRegistry = struct { .collect = collectStatsThunk, }); + if (self.extension_sync_hooks) |hooks| { + prov.setExtensionSyncHooks(hooks); + } + try prov.subscribe(.{ .context = @ptrCast(sub), .onUpdate = Subscription.handleUpdate, @@ -663,6 +694,7 @@ pub const PolicyRegistry = struct { self.bus, policies_slice, &comp_errors, + self.extension_resolver, ); errdefer log_idx.deinit(); @@ -671,6 +703,7 @@ pub const PolicyRegistry = struct { self.bus, policies_slice, &comp_errors, + self.extension_resolver, ); errdefer metric_idx.deinit(); @@ -679,6 +712,7 @@ pub const PolicyRegistry = struct { self.bus, policies_slice, &comp_errors, + self.extension_resolver, ); errdefer trace_idx.deinit(); diff --git a/src/policy/root.zig b/src/policy/root.zig index c8cece5..0bc1cb7 100644 --- a/src/policy/root.zig +++ b/src/policy/root.zig @@ -58,6 +58,7 @@ pub const PolicyMetadata = source.PolicyMetadata; pub const provider = @import("./provider.zig"); pub const PolicyCallback = provider.PolicyCallback; pub const PolicyUpdate = provider.PolicyUpdate; +pub const ExtensionSyncHooks = provider.ExtensionSyncHooks; /// Re-export registry pub const registry = @import("./registry.zig"); @@ -101,6 +102,14 @@ pub const TraceAccessor = types.TraceAccessor; pub const TypedValue = types.TypedValue; pub const TelemetryType = types.TelemetryType; +// Extensions (spec v1.6.0) +pub const ExtensionSlice = types.ExtensionSlice; +pub const ExtensionSliceSet = types.ExtensionSliceSet; +pub const ExtensionBinding = types.ExtensionBinding; +pub const ExtensionResolution = types.ExtensionResolution; +pub const ExtensionResolver = types.ExtensionResolver; +pub const ExtensionSink = types.ExtensionSink; + // ============================================================================= // Matcher Index (Hyperscan-based pattern matching) // ============================================================================= diff --git a/src/policy/types.zig b/src/policy/types.zig index d8c9ef4..ecee376 100644 --- a/src/policy/types.zig +++ b/src/policy/types.zig @@ -22,6 +22,95 @@ pub const TelemetryType = enum { trace, }; +// ============================================================================= +// Extensions (spec v1.6.0) — dispatch types +// ============================================================================= + +/// Which disjoint slice of the telemetry stream a record fell into, relative +/// to one policy: matched and kept, matched but removed by the final keep +/// outcome (which another policy may have caused), or unmatched. The proto +/// ExtensionMode values MATCHED and ALL are unions over these, resolved to an +/// ExtensionSliceSet at snapshot compile time. +pub const ExtensionSlice = enum(u2) { kept, dropped, unmatched }; + +pub const ExtensionSliceSet = std.EnumSet(ExtensionSlice); + +/// Resolve a proto ExtensionMode to the slice set it selects. UNSPECIFIED +/// and unknown future modes resolve to MATCHED, the spec default. +pub fn extensionSliceSet(mode: proto.policy.ExtensionMode) ExtensionSliceSet { + return switch (mode) { + .EXTENSION_MODE_KEPT => .init(.{ .kept = true }), + .EXTENSION_MODE_DROPPED => .init(.{ .dropped = true }), + .EXTENSION_MODE_UNMATCHED => .init(.{ .unmatched = true }), + .EXTENSION_MODE_ALL => .init(.{ .kept = true, .dropped = true, .unmatched = true }), + .EXTENSION_MODE_MATCHED, .EXTENSION_MODE_UNSPECIFIED, _ => .init(.{ .kept = true, .dropped = true }), + }; +} + +/// One (policy, extension) pair compiled into a signal's matcher index. +/// `handler` and `slot` are assigned by the resolver at snapshot compile time +/// and are opaque to policy_zig — the sink that receives the binding +/// interprets them. Dispatch never re-parses the extension declaration or +/// compares type strings. +pub const ExtensionBinding = struct { + /// Index into the signal index's policy list (matcher_index.PolicyIndex). + policy_index: u16, + /// Policy id, owned by the index (valid for the snapshot's lifetime). + policy_id: []const u8, + /// Opaque handler identifier from the resolver (e.g. an enum tag value). + handler: u8, + /// Opaque per-binding slot from the resolver (e.g. a batch index). + slot: u32, + /// Slices of the stream this binding's mode selects. + slices: ExtensionSliceSet, + /// Extension config bytes, owned by the index. + config: []const u8, +}; + +/// Result of resolving one extension declaration to a handler. +pub const ExtensionResolution = struct { + handler: u8, + slot: u32, +}; + +/// Resolves and validates one extension declaration at snapshot compile time. +/// Implemented outside policy_zig (the extensions module); policy_zig only +/// stores the opaque resolution in the binding. Return null when the +/// extension is unsupported or its config is invalid: per spec rule 5 the +/// extension is skipped (fail-open) and reported via PolicySyncStatus.errors, +/// while the policy's core match/keep/transform still applies. +pub const ExtensionResolver = struct { + ctx: *anyopaque, + resolve: *const fn ( + io: std.Io, + ctx: *anyopaque, + signal: TelemetryType, + policy_id: []const u8, + extension: *const proto.policy.Extension, + ) ?ExtensionResolution, +}; + +/// Receives records selected by an extension's mode. One function pointer + +/// ctx (the shape of the accessor primitives, not a vtable): it exists only +/// to break the module cycle — policy_zig cannot import the extensions +/// module that implements it. +/// +/// Called synchronously inside `evaluate`, after the final keep outcome is +/// known and before transforms run, so implementations observe pre-transform +/// records. `record` is borrowed and valid ONLY for the duration of the +/// call; implementations copy (encode) what they keep and MUST NOT block. +pub const ExtensionSink = struct { + ctx: *anyopaque, + deliver: *const fn ( + ctx: *anyopaque, + io: ?std.Io, + signal: TelemetryType, + record: *const anyopaque, + binding: *const ExtensionBinding, + slice: ExtensionSlice, + ) void, +}; + // ============================================================================= // Service and Provider Configuration // ============================================================================= @@ -601,6 +690,18 @@ pub const Provider = union(enum) { } } + /// Hand the provider extension sync hooks (v1.6.0): capability + /// advertisement in sync requests and routing of broadcast extension + /// configs from responses. Only the HTTP provider has a control plane; + /// file/testing accept the call and ignore it (nowhere to advertise to, no + /// broadcast to receive). Wired uniformly by `registry.subscribe`. + pub fn setExtensionSyncHooks(self: Provider, hooks: policy_provider.ExtensionSyncHooks) void { + switch (self) { + .http => |p| p.setExtensionSyncHooks(hooks), + .file, .testing => {}, + } + } + pub fn sourceType(self: Provider) SourceType { return switch (self) { .file => .file, diff --git a/src/proto/tero/policy/v1.pb.zig b/src/proto/tero/policy/v1.pb.zig index 3f05abd..981994e 100644 --- a/src/proto/tero/policy/v1.pb.zig +++ b/src/proto/tero/policy/v1.pb.zig @@ -9,6 +9,276 @@ const google_api = @import("../../google/api.pb.zig"); /// import package opentelemetry.proto.common.v1 const opentelemetry_proto_common_v1 = @import("../../opentelemetry/proto/common/v1.pb.zig"); +/// ExtensionMode selects which slice of the telemetry stream the engine delivers +/// to an extension, relative to its policy. {KEPT, DROPPED, UNMATCHED} are +/// disjoint and partition the stream; MATCHED and ALL are convenience unions. +pub const ExtensionMode = enum(i32) { + EXTENSION_MODE_UNSPECIFIED = 0, + EXTENSION_MODE_KEPT = 1, + EXTENSION_MODE_DROPPED = 2, + EXTENSION_MODE_UNMATCHED = 3, + EXTENSION_MODE_MATCHED = 4, + EXTENSION_MODE_ALL = 5, + _, +}; + +/// Extension attaches implementation-specific behavior to a policy. +/// +/// Implementations MUST declare which extension types they support. An extension +/// with an unrecognized type is handled per the spec (fail-open, or reject policy +/// load when the extension is required for the policy's behavior). +pub const Extension = struct { + type: []const u8 = &.{}, + version: []const u8 = &.{}, + config: []const u8 = &.{}, + mode: ExtensionMode = @enumFromInt(0), + + pub const _desc_table = .{ + .type = fd(1, .{ .scalar = .string }), + .version = fd(2, .{ .scalar = .string }), + .config = fd(3, .{ .scalar = .bytes }), + .mode = fd(4, .@"enum"), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +}; + +/// ExtensionCapability declares a client's support for one extension type. The +/// repeated `config` entries are opaque, type-defined capability descriptors — +/// for example, the destinations the client can route to. The policy engine does +/// not interpret them; the extension handler does. Providers use this to avoid +/// sending policies whose extensions the client cannot satisfy. +pub const ExtensionCapability = struct { + type: []const u8 = &.{}, + min_version: []const u8 = &.{}, + config: std.ArrayList([]const u8) = .empty, + + pub const _desc_table = .{ + .type = fd(1, .{ .scalar = .string }), + .min_version = fd(2, .{ .scalar = .string }), + .config = fd(3, .{ .repeated = .{ .scalar = .bytes } }), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +}; + +/// ExtensionConfig broadcasts opaque, type-defined configuration for one +/// extension type to clients (via SyncResponse.extension_configs). Each `config` +/// entry is parsed by the handler registered for `type`. For tero's s3-dump +/// these are serialized ExtensionTargets (see tero_extensions.proto). +pub const ExtensionConfig = struct { + type: []const u8 = &.{}, + config: std.ArrayList([]const u8) = .empty, + + pub const _desc_table = .{ + .type = fd(1, .{ .scalar = .string }), + .config = fd(2, .{ .repeated = .{ .scalar = .bytes } }), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +}; + /// AttributePath specifies how to access an attribute value. /// /// The path is represented as an array of string segments. Each segment represents @@ -1873,6 +2143,7 @@ pub const Policy = struct { created_at_unix_nano: u64 = 0, modified_at_unix_nano: u64 = 0, labels: std.ArrayList(opentelemetry_proto_common_v1.KeyValue) = .empty, + extensions: std.ArrayList(Extension) = .empty, target: ?target_union = null, pub const _target_case = enum { @@ -1899,6 +2170,7 @@ pub const Policy = struct { .created_at_unix_nano = fd(5, .{ .scalar = .fixed64 }), .modified_at_unix_nano = fd(6, .{ .scalar = .fixed64 }), .labels = fd(7, .{ .repeated = .submessage }), + .extensions = fd(20, .{ .repeated = .submessage }), .target = fd(null, .{ .oneof = target_union }), }; @@ -1977,11 +2249,13 @@ pub const ClientMetadata = struct { supported_policy_stages: std.ArrayList(PolicyStage) = .empty, labels: std.ArrayList(opentelemetry_proto_common_v1.KeyValue) = .empty, resource_attributes: std.ArrayList(opentelemetry_proto_common_v1.KeyValue) = .empty, + supported_extensions: std.ArrayList(ExtensionCapability) = .empty, pub const _desc_table = .{ .supported_policy_stages = fd(1, .{ .packed_repeated = .@"enum" }), .labels = fd(2, .{ .repeated = .submessage }), .resource_attributes = fd(3, .{ .repeated = .submessage }), + .supported_extensions = fd(4, .{ .repeated = .submessage }), }; /// Encodes the message to the writer @@ -2321,6 +2595,7 @@ pub const SyncResponse = struct { recommended_sync_interval_seconds: u32 = 0, sync_type: SyncType = @enumFromInt(0), error_message: []const u8 = &.{}, + extension_configs: std.ArrayList(ExtensionConfig) = .empty, pub const _desc_table = .{ .policies = fd(1, .{ .repeated = .submessage }), @@ -2329,6 +2604,7 @@ pub const SyncResponse = struct { .recommended_sync_interval_seconds = fd(4, .{ .scalar = .uint32 }), .sync_type = fd(5, .@"enum"), .error_message = fd(6, .{ .scalar = .string }), + .extension_configs = fd(7, .{ .repeated = .submessage }), }; /// Encodes the message to the writer @@ -2411,3 +2687,175 @@ pub fn PolicyService(comptime UserDataType: type, comptime ErrorSet: type) type Sync: *const fn (userdata: *UserDataType, request: SyncRequest) ErrorSet!SyncResponse, }; } + +/// ExtensionTarget is a pre-configured, named destination an extension can route +/// telemetry to (for example, an S3 bucket or a downstream OTLP endpoint). +/// +/// Targets are configured out of band on the implementation, or broadcast to +/// clients via SyncResponse.extension_configs (serialized into the s3-dump +/// extension's config). Policies reference a target by (kind, name) via +/// ExtensionTargetRef and never carry its configuration inline. +pub const ExtensionTarget = struct { + kind: []const u8 = &.{}, + name: []const u8 = &.{}, + config: []const u8 = &.{}, + + pub const _desc_table = .{ + .kind = fd(1, .{ .scalar = .string }), + .name = fd(2, .{ .scalar = .string }), + .config = fd(3, .{ .scalar = .bytes }), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +}; + +/// ExtensionTargetRef references a pre-configured ExtensionTarget by identity. +/// Carried inside an Extension's config (to name where it routes) and inside an +/// ExtensionCapability's config (to advertise reachable destinations). The +/// referenced target MUST be known to the implementation — locally configured or +/// received via SyncResponse.extension_configs. +pub const ExtensionTargetRef = struct { + kind: []const u8 = &.{}, + name: []const u8 = &.{}, + + pub const _desc_table = .{ + .kind = fd(1, .{ .scalar = .string }), + .name = fd(2, .{ .scalar = .string }), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +};