feat: update to add S3 extension

README.md

@@ -146,6 +146,7 @@ src/
     trace_sampler.zig   # Trace sampling
     rate_limiter.zig    # Rate limiting
     hyperscan.zig       # Vectorscan/Hyperscan C bindings
+  extensions/           # Policy extensions (spec v1.6.0): s3-dump handler
   observability/        # Event bus, spans, formatters
   proto/                # Generated protobuf definitions
 ```
@@ -161,23 +162,83 @@ src/
 This gives O(k \* n) performance where k = unique matcher keys and n = input
 text length, independent of policy count.
 
+## Supported Extensions
+
+Per policy-spec conformance item 9, this implementation supports the following
+extension `type`/`version` pairs (see `design/extensions.md` and the
+`extensions` module):
+
+| type                  | versions | notes                                                                                      |
+| --------------------- | -------- | ------------------------------------------------------------------------------------------ |
+| `com.usetero/s3-dump` | 1.0.x    | batched ndjson objects via z3; failed uploads retried in-memory (bounded), idempotent keys |
+
+Extensions declared by a policy but not supported (or with an unresolvable
+target) are skipped fail-open and reported via `PolicySyncStatus.errors`; the
+policy's core match/keep/transform still applies.
+
+Enabling extensions in a consumer (the `extensions` module owns the handlers
+and adapts them to policy_zig's seams):
+
+```zig
+const ext = @import("extensions");
+
+// One Extensions object owns every enabled handler. Encoders are per-signal
+// because the engine hands the sink an opaque record it can't serialize.
+var exts = ext.Extensions.init(allocator, .{ .log = encodeLog });
+defer exts.deinit();
+
+// Enable s3-dump and configure targets directly on the returned handle.
+// Credentials come from the consumer (env/secrets) — never from policies.
+const s3 = exts.enableS3Dump(allocator, .{}, .{
+    .access_key_id = key,
+    .secret_access_key = secret,
+});
+try s3.addTarget(io, "eu-bucket",
+    \\{"bucket": "tero-waste", "region": "eu-west-1", "prefix": "dumps/"}
+);
+
+// One call wires the resolver (snapshot compile) + sync hooks (advertised on
+// each provider). subscribe() then pushes the hooks to every provider — the
+// HTTP provider advertises capabilities and receives broadcast targets; a
+// file provider accepts and ignores them.
+exts.register(&registry);
+try registry.subscribe(.{ .http = http_provider });
+
+// Per evaluate: pass the sink so selected records are delivered.
+_ = engine.evaluate(.log, &accessor, &record, &buf, .{
+    .io = io,
+    .extension_sink = exts.sink(),
+});
+
+// From a background loop: flush uploads and read the result as metrics.
+const r = exts.flush(io, .{});
+// r.objects_uploaded / .records_dropped / .backlog_bytes → your metrics client
+```
+
+`com.usetero/s3-dump`'s upload path is verified two ways: `zig build test`
+(covered by `task test`) includes a hermetic test against an in-process HTTP
+stub — no network or docker needed — and `task test:s3-e2e` spins up a real
+MinIO container via docker and round-trips an upload through it.
+
 ## Task Commands
 
 This project uses [Task](https://taskfile.dev/) for common operations:
 
-| Command              | Description                                      |
-| -------------------- | ------------------------------------------------ |
-| `task`               | Build (debug)                                    |
-| `task test`          | Run all tests                                    |
-| `task build:release` | Build with ReleaseFast                           |
-| `task build:safe`    | Build with ReleaseSafe                           |
-| `task format`        | Format source files                              |
-| `task format:check`  | Check formatting                                 |
-| `task lint`          | Run linting checks                               |
-| `task clean`         | Clean build artifacts                            |
-| `task do`            | Run all pre-commit checks (format + lint + test) |
-| `task signoff`       | Full signoff (do + build:safe + gh signoff)      |
-| `task ci:setup`      | Install CI/dev dependencies                      |
+| Command              | Description                                                     |
+| -------------------- | --------------------------------------------------------------- |
+| `task`               | Build (debug)                                                   |
+| `task test`          | Run all tests                                                   |
+| `task test:s3-e2e`   | Real-storage s3-dump smoke test against MinIO (requires docker) |
+| `task bench:s3`      | s3-dump scale benchmarks against MinIO (requires docker)        |
+| `task build:release` | Build with ReleaseFast                                          |
+| `task build:safe`    | Build with ReleaseSafe                                          |
+| `task format`        | Format source files                                             |
+| `task format:check`  | Check formatting                                                |
+| `task lint`          | Run linting checks                                              |
+| `task clean`         | Clean build artifacts                                           |
+| `task do`            | Run all pre-commit checks (format + lint + test)                |
+| `task signoff`       | Full signoff (do + build:safe + gh signoff)                     |
+| `task ci:setup`      | Install CI/dev dependencies                                     |
 
 ## License
 

Taskfile.yml

@@ -71,6 +71,78 @@ tasks:
     cmds:
       - zig build test --summary all
 
+  test:s3-e2e:
+    desc: "Real-storage smoke test for the s3-dump extension against a local MinIO container (requires docker)"
+    cmds:
+      - |
+        if ! command -v docker >/dev/null 2>&1; then
+          echo "docker is required for task test:s3-e2e"
+          exit 1
+        fi
+        CONTAINER=policy-zig-minio-e2e
+        docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
+        docker run -d --rm --name "$CONTAINER" \
+          -p 9000:9000 \
+          -e MINIO_ROOT_USER=minioadmin \
+          -e MINIO_ROOT_PASSWORD=minioadmin \
+          minio/minio server /data >/dev/null
+        trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1' EXIT
+
+        echo "Waiting for MinIO..."
+        for i in $(seq 1 30); do
+          if curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; then
+            break
+          fi
+          if [ "$i" = 30 ]; then
+            echo "MinIO did not become healthy in time"
+            docker logs "$CONTAINER" || true
+            exit 1
+          fi
+          sleep 1
+        done
+
+        AWS_ACCESS_KEY_ID=minioadmin \
+        AWS_SECRET_ACCESS_KEY=minioadmin \
+        S3_ENDPOINT=http://127.0.0.1:9000 \
+        S3_BUCKET=policy-zig-e2e \
+          zig build test-s3-e2e --summary all
+
+  bench:s3:
+    desc: "s3-dump scale benchmarks against a local MinIO container (requires docker)"
+    cmds:
+      - |
+        if ! command -v docker >/dev/null 2>&1; then
+          echo "docker is required for task bench:s3"
+          exit 1
+        fi
+        CONTAINER=policy-zig-minio-bench
+        docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
+        docker run -d --rm --name "$CONTAINER" \
+          -p 9000:9000 \
+          -e MINIO_ROOT_USER=minioadmin \
+          -e MINIO_ROOT_PASSWORD=minioadmin \
+          minio/minio server /data >/dev/null
+        trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1' EXIT
+
+        echo "Waiting for MinIO..."
+        for i in $(seq 1 30); do
+          if curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; then
+            break
+          fi
+          if [ "$i" = 30 ]; then
+            echo "MinIO did not become healthy in time"
+            docker logs "$CONTAINER" || true
+            exit 1
+          fi
+          sleep 1
+        done
+
+        AWS_ACCESS_KEY_ID=minioadmin \
+        AWS_SECRET_ACCESS_KEY=minioadmin \
+        S3_ENDPOINT=http://127.0.0.1:9000 \
+        S3_BUCKET=policy-zig-bench \
+          zig build bench-s3
+
   format:
     desc: "Format all Zig source files"
     aliases: [fmt, f]

build.zig

@@ -44,6 +44,23 @@ pub fn build(b: *std.Build) void {
     mod.link_libc = true;
     mod.linkSystemLibrary("hs", .{});
 
+    // Extensions module (spec v1.6.0): concrete extension handlers (s3-dump).
+    // Separate module so policy_zig stays free of I/O and of the z3
+    // dependency; only consumers that enable extensions link it.
+    const z3_dep = b.dependency("z3", .{
+        .target = target,
+        .optimize = optimize,
+    });
+    const ext_mod = b.addModule("extensions", .{
+        .root_source_file = b.path("src/extensions/root.zig"),
+        .target = target,
+        .imports = &.{
+            .{ .name = "proto", .module = proto_mod },
+            .{ .name = "policy_zig", .module = mod },
+            .{ .name = "s3", .module = z3_dep.module("s3") },
+        },
+    });
+
     // Tests
     const mod_tests = b.addTest(.{
         .root_module = mod,
@@ -59,9 +76,60 @@ pub fn build(b: *std.Build) void {
     });
     const run_o11y_tests = b.addRunArtifact(o11y_tests);
 
+    // Extensions module tests (need libc/hyperscan transitively via policy_zig)
+    const ext_tests = b.addTest(.{
+        .root_module = ext_mod,
+    });
+    ext_tests.root_module.link_libc = true;
+    ext_tests.root_module.linkSystemLibrary("hs", .{});
+    const run_ext_tests = b.addRunArtifact(ext_tests);
+
     const test_step = b.step("test", "Run tests");
     test_step.dependOn(&run_mod_tests.step);
     test_step.dependOn(&run_o11y_tests.step);
+    test_step.dependOn(&run_ext_tests.step);
+
+    // Real-storage s3-dump smoke test (needs a running S3-compatible
+    // server — see `task test:s3-e2e`). Deliberately NOT part of `test`:
+    // it requires network I/O and external state, unlike every other test.
+    const s3_e2e_tests = b.addTest(.{
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("src/extensions/s3_minio_test.zig"),
+            .target = target,
+            .imports = &.{
+                .{ .name = "proto", .module = proto_mod },
+                .{ .name = "policy_zig", .module = mod },
+                .{ .name = "s3", .module = z3_dep.module("s3") },
+            },
+        }),
+    });
+    s3_e2e_tests.root_module.link_libc = true;
+    s3_e2e_tests.root_module.linkSystemLibrary("hs", .{});
+    const run_s3_e2e_tests = b.addRunArtifact(s3_e2e_tests);
+    const s3_e2e_step = b.step("test-s3-e2e", "Run the s3-dump smoke test against a real S3-compatible server");
+    s3_e2e_step.dependOn(&run_s3_e2e_tests.step);
+
+    // s3-dump scale benchmarks (needs MinIO — see `task bench:s3`).
+    // ReleaseFast like the other benchmarks; excluded from `test`. A plain
+    // executable, not a test: it reports numbers, not pass/fail.
+    const s3_bench = b.addExecutable(.{
+        .name = "bench-s3",
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("src/extensions/s3_bench.zig"),
+            .target = target,
+            .optimize = .ReleaseFast,
+            .imports = &.{
+                .{ .name = "proto", .module = proto_mod },
+                .{ .name = "policy_zig", .module = mod },
+                .{ .name = "s3", .module = z3_dep.module("s3") },
+            },
+        }),
+    });
+    s3_bench.root_module.link_libc = true;
+    s3_bench.root_module.linkSystemLibrary("hs", .{});
+    const run_s3_bench = b.addRunArtifact(s3_bench);
+    const s3_bench_step = b.step("bench-s3", "Run s3-dump scale benchmarks against a real S3-compatible server");
+    s3_bench_step.dependOn(&run_s3_bench.step);
 
     // Benchmarks
     const zbench_dep = b.dependency("zbench", .{ .target = target, .optimize = .ReleaseFast });
@@ -96,6 +164,8 @@ pub fn build(b: *std.Build) void {
             .destination_directory = b.path("src/proto"),
             .source_files = &.{
                 b.path("proto/tero/policy/v1/policy.proto"),
+                b.path("proto/tero/policy/v1/extension.proto"),
+                b.path("proto/tero/policy/v1/tero_extensions.proto"),
                 b.path("proto/tero/policy/v1/log.proto"),
                 b.path("proto/tero/policy/v1/metric.proto"),
                 b.path("proto/tero/policy/v1/trace.proto"),

build.zig.zon

@@ -50,6 +50,10 @@
             .url = "git+https://github.com/hendriknielaender/zBench.git#b2b89c475e3ef1bb2bd71255c80478a82d3e0ca8",
             .hash = "zbench-0.13.0-YTdc7xVBAQCCMC-IdLLuotBeiNNNm8k9Pi2V4VYqrwfI",
         },
+        .z3 = .{
+            .url = "git+https://codeberg.org/fellowtraveler/z3#83b1f8d6222188f39c09966b8572bc9168813d04",
+            .hash = "z3-0.4.1-N25-cH-fAQAbj6ABBY0Bq85Kv-d7sdIvANgLCE9NCaEF",
+        },
     },
     // Specifies the set of files and directories that are included in this package.
     // Only files and directories listed here are included in the `hash` that