From 41e6affbb7eaeed5eda61500433de0b93db8a5e7 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Tue, 7 Jul 2026 14:27:47 -0400 Subject: [PATCH 1/8] support log sampling --- proto/tero/policy/v1/log.proto | 31 ++++--- proto/tero/policy/v1/metric.proto | 20 +++-- proto/tero/policy/v1/shared.proto | 11 +-- proto/tero/policy/v1/trace.proto | 26 +++--- src/policy/matcher_index.zig | 54 ++++++++---- src/policy/parser.zig | 58 +++++++++++-- src/policy/policy_engine.zig | 124 ++++++++++++++++++++++++++- src/policy/probabilistic_sampler.zig | 38 ++++++++ src/proto/tero/policy/v1.pb.zig | 13 +-- 9 files changed, 313 insertions(+), 62 deletions(-) diff --git a/proto/tero/policy/v1/log.proto b/proto/tero/policy/v1/log.proto index f7d9c4e..b464f88 100644 --- a/proto/tero/policy/v1/log.proto +++ b/proto/tero/policy/v1/log.proto @@ -23,7 +23,7 @@ message LogTarget { // Valid values: // "all" - Keep everything (default, can be omitted) // "none" - Drop everything - // "N%" - Keep N percent (0-100), e.g. "50%" + // "N%" - Keep approximately N percent (0-100), e.g. "50%" // "N/s" - Keep at most N per second (shorthand for "N/1s"), e.g. "100/s" // "N/m" - Keep at most N per minute (shorthand for "N/1m"), e.g. "1000/m" // "N/Ds" - Keep at most N per D-second window, e.g. "1/5s" @@ -41,6 +41,11 @@ message LogTarget { // keep/drop decision. Use for lifecycle events (request_id, trace_id, job_id) // to avoid sampling individual log lines independently. // + // For percentage sampling (N%), omitting sample_key means each matching log is + // sampled independently using a random value generated at evaluation time. + // That default gives the expected approximate distribution, but is not + // idempotent across repeated evaluation of the same log sequence. + // // Only applies when keep is a sampling value (N%, N/s, N/m, N/Ds, N/Dm). // Example: sample_key = log_attribute["request_id"] with keep = "10%" means // 10% of requests are kept, with all logs from each kept request preserved. @@ -79,8 +84,8 @@ enum LogField { LOG_FIELD_BODY = 1; LOG_FIELD_SEVERITY_TEXT = 2; // trace_id and span_id are bytes. They are authored as lowercase hex and - // matched as raw bytes (exact/equals), or as their hex rendering for string - // match types. See the Value message and the spec's + // matched as raw bytes with equals.hex_value, or as their hex rendering for + // string pattern match types. See the Value message and the spec's // "Bytes and Identifier Fields" section. LOG_FIELD_TRACE_ID = 3; LOG_FIELD_SPAN_ID = 4; @@ -124,12 +129,17 @@ message LogMatcher { // Match type. Exactly one must be set. // - // The string match types (exact, regex, starts_with, ends_with, contains) - // operate only on string field values. To match non-string values, use the - // typed equals matcher or the numeric comparison matchers (gt, gte, lt, lte). + // Use the typed equals matcher for equality. The exact field is deprecated and + // will move to reserved in a future version after wire-format users migrate to + // equals.string_value. + // + // The string pattern match types (regex, starts_with, ends_with, contains) + // operate only on string field values. Use the numeric comparison matchers + // (gt, gte, lt, lte) for ranges. oneof match { - // Exact string match (string field values only) - string exact = 10; + // Deprecated: use equals.string_value instead. This field will move to + // reserved in a future version. + string exact = 10 [deprecated = true]; // Regular expression match (string field values only) string regex = 11; @@ -146,7 +156,7 @@ message LogMatcher { // Literal substring match (string field values only) string contains = 15; - // Typed equality for non-string field values (bool, int, double, bytes) + // Typed equality for string, bool, int, double, or bytes field values. Value equals = 22; // Numeric greater-than comparison (int/double field values) @@ -165,7 +175,8 @@ message LogMatcher { // If true, inverts the match result bool negate = 20; - // If true, applies case-insensitive matching to all match types + // If true, applies case-insensitive matching to equals.string_value and the + // string pattern match types. bool case_insensitive = 21; } diff --git a/proto/tero/policy/v1/metric.proto b/proto/tero/policy/v1/metric.proto index 84998b7..35a8db4 100644 --- a/proto/tero/policy/v1/metric.proto +++ b/proto/tero/policy/v1/metric.proto @@ -100,12 +100,17 @@ message MetricMatcher { // Match type. Exactly one must be set. // Note: For metric_type field, only exists is valid (type equality is implicit). // - // The string match types (exact, regex, starts_with, ends_with, contains) - // operate only on string field values. To match non-string values, use the - // typed equals matcher or the numeric comparison matchers (gt, gte, lt, lte). + // Use the typed equals matcher for equality. The exact field is deprecated and + // will move to reserved in a future version after wire-format users migrate to + // equals.string_value. + // + // The string pattern match types (regex, starts_with, ends_with, contains) + // operate only on string field values. Use the numeric comparison matchers + // (gt, gte, lt, lte) for ranges. oneof match { - // Exact string match (string field values only) - string exact = 10; + // Deprecated: use equals.string_value instead. This field will move to + // reserved in a future version. + string exact = 10 [deprecated = true]; // Regular expression match (string field values only) string regex = 11; @@ -122,7 +127,7 @@ message MetricMatcher { // Literal substring match (string field values only) string contains = 15; - // Typed equality for non-string field values (bool, int, double, bytes) + // Typed equality for string, bool, int, double, or bytes field values. Value equals = 22; // Numeric greater-than comparison (int/double field values) @@ -141,6 +146,7 @@ message MetricMatcher { // If true, inverts the match result bool negate = 20; - // If true, applies case-insensitive matching to all match types + // If true, applies case-insensitive matching to equals.string_value and the + // string pattern match types. bool case_insensitive = 21; } diff --git a/proto/tero/policy/v1/shared.proto b/proto/tero/policy/v1/shared.proto index 47f43af..260cead 100644 --- a/proto/tero/policy/v1/shared.proto +++ b/proto/tero/policy/v1/shared.proto @@ -51,9 +51,7 @@ message AttributePath { repeated string path = 1; } -// Value carries a typed, non-string scalar for the `equals` matcher. String -// equality is expressed with the `exact` match type, so this message -// intentionally has no string variant. +// Value carries a typed scalar for the `equals` matcher. // // `equals` matches when the field value has the same type and value as the set // variant. Integer and floating-point values are compared in a single numeric @@ -68,14 +66,16 @@ message AttributePath { // equals: true -> bool_value // equals: 200 -> int_value // equals: 0.5 -> double_value +// equals: "foo" -> string_value // // Bytes are authored either as proto-native base64 (`bytes_value`) or, more // readably, as a lowercase-hex string (`hex_value`). The two are equivalent — // hex_value is decoded to bytes at policy-compile time and yields the same bytes // as the corresponding bytes_value — but hex_value keeps identifiers // (trace/span ids) readable in the canonical proto/JSON form instead of base64. -// A bare string literal (e.g. `equals: "foo"`) MUST be rejected — use `exact` -// for strings. +// +// String equality SHOULD be authored with string_value. The older matcher-level +// `exact` field is deprecated and will move to reserved in a future version. message Value { // Exactly one must be set. oneof value { @@ -88,6 +88,7 @@ message Value { // input, canonically lowercase). Decoded to bytes at policy-compile time; // an invalid hex string MUST be rejected. string hex_value = 5; + string string_value = 6; } } diff --git a/proto/tero/policy/v1/trace.proto b/proto/tero/policy/v1/trace.proto index 0d65ba3..792751b 100644 --- a/proto/tero/policy/v1/trace.proto +++ b/proto/tero/policy/v1/trace.proto @@ -32,9 +32,9 @@ enum TraceField { // Span fields TRACE_FIELD_NAME = 1; // trace_id, span_id, and parent_span_id are bytes. They are authored as - // lowercase hex and matched as raw bytes (exact/equals), or as their hex - // rendering for string match types. See the Value message and the spec's - // "Bytes and Identifier Fields" section. + // lowercase hex and matched as raw bytes with equals.hex_value, or as their + // hex rendering for string pattern match types. See the Value message and the + // spec's "Bytes and Identifier Fields" section. TRACE_FIELD_TRACE_ID = 2; TRACE_FIELD_SPAN_ID = 3; TRACE_FIELD_PARENT_SPAN_ID = 4; @@ -119,12 +119,17 @@ message TraceMatcher { // Match type. Exactly one must be set. // Note: For span_kind and span_status fields, only exists is valid (equality is implicit). // - // The string match types (exact, regex, starts_with, ends_with, contains) - // operate only on string field values. To match non-string values, use the - // typed equals matcher or the numeric comparison matchers (gt, gte, lt, lte). + // Use the typed equals matcher for equality. The exact field is deprecated and + // will move to reserved in a future version after wire-format users migrate to + // equals.string_value. + // + // The string pattern match types (regex, starts_with, ends_with, contains) + // operate only on string field values. Use the numeric comparison matchers + // (gt, gte, lt, lte) for ranges. oneof match { - // Exact string match (string field values only) - string exact = 10; + // Deprecated: use equals.string_value instead. This field will move to + // reserved in a future version. + string exact = 10 [deprecated = true]; // Regular expression match (string field values only) string regex = 11; @@ -141,7 +146,7 @@ message TraceMatcher { // Literal substring match (string field values only) string contains = 15; - // Typed equality for non-string field values (bool, int, double, bytes) + // Typed equality for string, bool, int, double, or bytes field values. Value equals = 22; // Numeric greater-than comparison (int/double field values) @@ -160,7 +165,8 @@ message TraceMatcher { // If true, inverts the match result bool negate = 20; - // If true, applies case-insensitive matching to all match types + // If true, applies case-insensitive matching to equals.string_value and the + // string pattern match types. bool case_insensitive = 21; } diff --git a/src/policy/matcher_index.zig b/src/policy/matcher_index.zig index 36bfaa9..1c6c897 100644 --- a/src/policy/matcher_index.zig +++ b/src/policy/matcher_index.zig @@ -579,6 +579,10 @@ fn compileValue(allocator: std.mem.Allocator, v: proto.policy.Value) !?CompiledV } break :blk .{ .bytes = bytes }; }, + // string_value is string equality (≡ the deprecated `exact`) and is + // routed to the Hyperscan pattern path by addMatcher; it never + // compiles to a typed check. Null keeps an unrouted one fail-open. + .string_value => null, }; } @@ -1061,6 +1065,15 @@ fn IndexBuilder(comptime T: TelemetryType) type { try self.recordError(signal ++ ": match[{d}]: invalid pattern \"{s}\"", policy_id, .{ idx, p }); return null; }, + // equals.string_value compiles to a Hyperscan pattern like + // `exact`; other equals variants are typed checks. + .equals => |v| if (v.value != null and v.value.? == .string_value) { + const p = v.value.?.string_value; + if (!self.patternCompiles(p, .exact, matcher.case_insensitive)) { + try self.recordError(signal ++ ": match[{d}]: invalid pattern \"{s}\"", policy_id, .{ idx, p }); + return null; + } + }, else => {}, } return field_ref; @@ -1192,24 +1205,30 @@ fn IndexBuilder(comptime T: TelemetryType) type { // primitive. An invalid/unset value is silently dropped (fail-open). switch (m) { .equals => |v| { - const compiled = (try compileValue(self.allocator, v)) orelse { - self.bus.debug(TypedMatcherSkipped{ .matcher_idx = matcher_idx }); + // equals.string_value is string equality (≡ the deprecated + // `exact`, v1.6.0) and falls through to the pattern path + // below, so it gets Hyperscan and case_insensitive support. + const is_string = v.value != null and v.value.? == .string_value; + if (!is_string) { + const compiled = (try compileValue(self.allocator, v)) orelse { + self.bus.debug(TypedMatcherSkipped{ .matcher_idx = matcher_idx }); + return; + }; + // typed checks count toward required_match_count so the + // policy's match threshold is still enforced. + if (matcher.negate) { + self.current_negated_count += 1; + } else { + self.current_positive_count += 1; + } + try self.typed_checks_list.append(self.allocator, .{ + .policy_index = self.policy_index, + .field_ref = field_ref, + .matcher = .{ .equals = compiled }, + .negate = matcher.negate, + }); return; - }; - // typed checks count toward required_match_count so the - // policy's match threshold is still enforced. - if (matcher.negate) { - self.current_negated_count += 1; - } else { - self.current_positive_count += 1; } - try self.typed_checks_list.append(self.allocator, .{ - .policy_index = self.policy_index, - .field_ref = field_ref, - .matcher = .{ .equals = compiled }, - .negate = matcher.negate, - }); - return; }, .gt => |v| { const compiled = compileNumericValue(v) orelse { @@ -1273,11 +1292,12 @@ fn IndexBuilder(comptime T: TelemetryType) type { const pattern, const pattern_match_type, const negate = switch (m) { .regex => |r| .{ r, match_type.regex, matcher.negate }, .exact => |e| .{ e, match_type.exact, matcher.negate }, + .equals => |v| .{ v.value.?.string_value, match_type.exact, matcher.negate }, .exists => unreachable, .starts_with => |s| .{ s, match_type.starts_with, matcher.negate }, .ends_with => |s| .{ s, match_type.ends_with, matcher.negate }, .contains => |s| .{ s, match_type.contains, matcher.negate }, - .equals, .gt, .gte, .lt, .lte => unreachable, + .gt, .gte, .lt, .lte => unreachable, }; if (pattern.len == 0) { diff --git a/src/policy/parser.zig b/src/policy/parser.zig index a7a63cf..68ba01e 100644 --- a/src/policy/parser.zig +++ b/src/policy/parser.zig @@ -124,6 +124,7 @@ const LogMatcherJson = struct { // equals: true → bool_value // equals: 200 → int_value // equals: 0.5 → double_value + // equals: "foo" → string_value (v1.6.0) // equals: { hex_value: "deadbeef" } → canonical // gt/gte/lt/lte accept integer or float literals only. equals: ?std.json.Value = null, @@ -523,13 +524,14 @@ fn parseLogMatcher(allocator: std.mem.Allocator, jm: LogMatcherJson) !LogMatcher /// bool → bool_value /// int → int_value /// float → double_value -/// object with hex_value/bytes_value/bool_value/int_value/double_value → canonical -/// A bare string literal is rejected per spec (use `exact` for strings). +/// string → string_value (v1.6.0) +/// object with hex_value/bytes_value/bool_value/int_value/double_value/string_value → canonical fn parseValue(allocator: std.mem.Allocator, json_val: std.json.Value) !Value { switch (json_val) { .bool => |b| return Value{ .value = .{ .bool_value = b } }, .integer => |i| return Value{ .value = .{ .int_value = i } }, .float => |f| return Value{ .value = .{ .double_value = f } }, + .string => |s| return Value{ .value = .{ .string_value = try allocator.dupe(u8, s) } }, .object => |obj| { if (obj.get("bool_value")) |v| { return Value{ .value = .{ .bool_value = v.bool } }; @@ -549,9 +551,11 @@ fn parseValue(allocator: std.mem.Allocator, json_val: std.json.Value) !Value { const bytes = try allocator.dupe(u8, v.string); return Value{ .value = .{ .bytes_value = bytes } }; } + if (obj.get("string_value")) |v| { + return Value{ .value = .{ .string_value = try allocator.dupe(u8, v.string) } }; + } return error.InvalidValue; }, - .string => return error.InvalidValue, // use `exact` for strings else => return error.InvalidValue, } } @@ -2670,9 +2674,23 @@ test "parseValue: hex_value canonical" { try std.testing.expectEqualSlices(u8, &.{ 0xde, 0xad, 0xbe, 0xef }, v.value.?.bytes_value); } -test "parseValue: string rejected" { +test "parseValue: string shorthand (v1.6.0)" { + const allocator = std.testing.allocator; + const v = try parseValue(allocator, .{ .string = "foo" }); + defer allocator.free(v.value.?.string_value); + try std.testing.expect(v.value.? == .string_value); + try std.testing.expectEqualStrings("foo", v.value.?.string_value); +} + +test "parseValue: string_value canonical (v1.6.0)" { const allocator = std.testing.allocator; - try std.testing.expectError(error.InvalidValue, parseValue(allocator, .{ .string = "foo" })); + var obj = std.json.ObjectMap.empty; + defer obj.deinit(allocator); + try obj.put(allocator, "string_value", .{ .string = "checkout-api" }); + const v = try parseValue(allocator, .{ .object = obj }); + defer allocator.free(v.value.?.string_value); + try std.testing.expect(v.value.? == .string_value); + try std.testing.expectEqualStrings("checkout-api", v.value.?.string_value); } test "parseNumericValue: int" { @@ -2738,6 +2756,36 @@ test "parsePoliciesBytes: log policy with equals bool" { try std.testing.expect(matcher.match.?.equals.value.?.bool_value == true); } +test "parsePoliciesBytes: log policy with equals string (v1.6.0)" { + const allocator = std.testing.allocator; + + const json = + \\{ + \\ "policies": [{ + \\ "id": "drop-checkout", + \\ "name": "Drop checkout logs", + \\ "log": { + \\ "match": [ + \\ { "resource_attribute": ["service.name"], "equals": "checkout-api" } + \\ ], + \\ "keep": "none" + \\ } + \\ }] + \\} + ; + + const policies = try parsePoliciesBytes(allocator, json); + defer { + for (policies) |*p| p.deinit(allocator); + allocator.free(policies); + } + + const matcher = policies[0].target.?.log.match.items[0]; + try std.testing.expect(matcher.match.? == .equals); + try std.testing.expect(matcher.match.?.equals.value.? == .string_value); + try std.testing.expectEqualStrings("checkout-api", matcher.match.?.equals.value.?.string_value); +} + test "parsePoliciesBytes: log policy with gte int" { const allocator = std.testing.allocator; diff --git a/src/policy/policy_engine.zig b/src/policy/policy_engine.zig index 7dd93d4..1df9a3c 100644 --- a/src/policy/policy_engine.zig +++ b/src/policy/policy_engine.zig @@ -730,7 +730,13 @@ pub const PolicyEngine = struct { break :blk if (sr.keep) FilterDecision.keep else FilterDecision.drop; } - // Non-trace: simple keep/drop from shouldKeep + // Non-trace (log): with no sample_key (or a missing/empty + // key value) the spec requires an independent random + // decision per record, so empty input takes the random + // path instead of failing closed. + if (input.len == 0) { + break :blk if (s.shouldKeepRandom(io)) FilterDecision.keep else FilterDecision.drop; + } break :blk if (s.shouldKeep(input)) FilterDecision.keep else FilterDecision.drop; } break :blk applyKeepValue(io, policy_info); @@ -2248,6 +2254,76 @@ test "typed: negated equals" { try testing.expectEqual(FilterDecision.drop, evalTypedLog(&engine, &ctx500, &policy_id_buf).decision); } +test "typed: equals string_value matches like exact (v1.6.0)" { + const allocator = testing.allocator; + + const sv: proto.policy.Value = .{ .value = .{ .string_value = try allocator.dupe(u8, "checkout-api") } }; + var policy = try makeLogTypedPolicy(allocator, "drop-checkout", "service.name", .{ .equals = sv }, "none"); + defer policy.deinit(allocator); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var ctx = TypedLogContext.init(allocator); + defer ctx.deinit(); + _ = try ctx.withString("service.name", "checkout-api"); + try testing.expectEqual(FilterDecision.drop, evalTypedLog(&engine, &ctx, &policy_id_buf).decision); + + // Prefix is not equality + var ctx2 = TypedLogContext.init(allocator); + defer ctx2.deinit(); + _ = try ctx2.withString("service.name", "checkout-api-v2"); + try testing.expectEqual(FilterDecision.unset, evalTypedLog(&engine, &ctx2, &policy_id_buf).decision); + + // Different case does not match without case_insensitive + var ctx3 = TypedLogContext.init(allocator); + defer ctx3.deinit(); + _ = try ctx3.withString("service.name", "CHECKOUT-API"); + try testing.expectEqual(FilterDecision.unset, evalTypedLog(&engine, &ctx3, &policy_id_buf).decision); +} + +test "typed: equals string_value with case_insensitive (v1.6.0)" { + const allocator = testing.allocator; + + var attr_path: proto.policy.AttributePath = .{}; + try attr_path.path.append(allocator, try allocator.dupe(u8, "env")); + + var policy: Policy = .{ + .id = try allocator.dupe(u8, "drop-prod"), + .name = try allocator.dupe(u8, "drop-prod"), + .enabled = true, + .target = .{ .log = .{ + .keep = try allocator.dupe(u8, "none"), + } }, + }; + try policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_attribute = attr_path }, + .match = .{ .equals = .{ .value = .{ .string_value = try allocator.dupe(u8, "production") } } }, + .case_insensitive = true, + }); + defer policy.deinit(allocator); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var ctx = TypedLogContext.init(allocator); + defer ctx.deinit(); + _ = try ctx.withString("env", "PRODUCTION"); + try testing.expectEqual(FilterDecision.drop, evalTypedLog(&engine, &ctx, &policy_id_buf).decision); +} + test "evaluate: policy with keep=all and no transform" { const allocator = testing.allocator; @@ -4479,14 +4555,56 @@ test "PolicyEngine: sample_key missing field falls back to default" { const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); var policy_id_buf: [16][]const u8 = undefined; - // Should still work (falls back to context pointer hash) + // Missing key value falls back to the independent random default var log1: TestLogContext = .{ .message = "test message" }; const result = evalTestLog(&engine, &log1, &policy_id_buf); - // Should get a decision (either keep or drop based on hash) + // Should get a decision (either keep or drop, randomly) try testing.expect(result.decision == .keep or result.decision == .drop); } +test "PolicyEngine: percentage sampling without sample_key is independent random (v1.6.0)" { + const allocator = testing.allocator; + + // 50% log sampling with no sample_key: each record must get an + // independent random decision — previously this failed closed and + // dropped everything. + var policy: Policy = .{ + .id = try allocator.dupe(u8, "sample-unkeyed"), + .name = try allocator.dupe(u8, "sample-unkeyed"), + .enabled = true, + .target = .{ .log = .{ + .keep = try allocator.dupe(u8, "50%"), + } }, + }; + try policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "^.*") }, + }); + defer policy.deinit(allocator); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + try registry.updatePolicies(&.{policy}, "file-provider", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var kept: u32 = 0; + const total: u32 = 2000; + for (0..total) |_| { + var log: TestLogContext = .{ .message = "same message every time" }; + if (evalTestLog(&engine, &log, &policy_id_buf).decision == .keep) kept += 1; + } + + // Identical records must not all get the same decision, and the split + // should be roughly 50% (wide bounds keep this test deterministic enough). + const ratio = @as(f64, @floatFromInt(kept)) / @as(f64, @floatFromInt(total)); + try testing.expect(ratio > 0.4 and ratio < 0.6); +} + test "PolicyEngine: log resource_schema_url matching" { const allocator = testing.allocator; diff --git a/src/policy/probabilistic_sampler.zig b/src/policy/probabilistic_sampler.zig index 15e5a31..66d1094 100644 --- a/src/policy/probabilistic_sampler.zig +++ b/src/policy/probabilistic_sampler.zig @@ -123,6 +123,21 @@ pub const ProbabilisticSampler = struct { return self.sample(input, "").keep; } + /// Independent random keep/drop for unkeyed log percentage sampling. + /// Per spec v1.6.0, without a sample_key each record MUST be sampled with a + /// fresh high-quality random value — never derived from timestamps or + /// record content — so this is intentionally not idempotent. + /// A null io fails open to keep (matching the rate limiter), except 0%. + pub fn shouldKeepRandom(self: ProbabilisticSampler, io: ?std.Io) bool { + if (self.percentage >= 100.0) return true; + if (self.percentage <= 0.0) return false; + const real_io = io orelse return true; + var buf: [8]u8 = undefined; + real_io.random(&buf); + const r = std.mem.readInt(u64, &buf, .little) & (max_56bit - 1); + return r >= self.threshold; + } + /// Calculate threshold from percentage /// T = floor((1 - percentage/100) * 2^56) pub fn calculateThreshold(percentage: f32) u64 { @@ -673,6 +688,29 @@ test "ProbabilisticSampler: initFromPercentage" { try testing.expect(full.shouldKeep("anything")); } +test "ProbabilisticSampler: shouldKeepRandom edge cases and distribution" { + const io = std.Options.debug_io; + + // 100% keeps all, 0% rejects all — with or without io + try testing.expect(ProbabilisticSampler.initFromPercentage(100).shouldKeepRandom(io)); + try testing.expect(ProbabilisticSampler.initFromPercentage(100).shouldKeepRandom(null)); + try testing.expect(!ProbabilisticSampler.initFromPercentage(0).shouldKeepRandom(io)); + try testing.expect(!ProbabilisticSampler.initFromPercentage(0).shouldKeepRandom(null)); + + // Null io fails open to keep for a sampling percentage + try testing.expect(ProbabilisticSampler.initFromPercentage(50).shouldKeepRandom(null)); + + // Independent random decisions approximate the configured percentage + const sampler = ProbabilisticSampler.initFromPercentage(50); + var kept: u32 = 0; + const total: u32 = 10000; + for (0..total) |_| { + if (sampler.shouldKeepRandom(io)) kept += 1; + } + const ratio = @as(f64, @floatFromInt(kept)) / @as(f64, @floatFromInt(total)); + try testing.expect(ratio > 0.45 and ratio < 0.55); +} + test "ProbabilisticSampler: shouldKeep matches sample" { const sampler = ProbabilisticSampler.initFromPercentage(50); const input = "test-input-bytes"; diff --git a/src/proto/tero/policy/v1.pb.zig b/src/proto/tero/policy/v1.pb.zig index 63b6899..3f05abd 100644 --- a/src/proto/tero/policy/v1.pb.zig +++ b/src/proto/tero/policy/v1.pb.zig @@ -122,9 +122,7 @@ pub const AttributePath = struct { } }; -/// Value carries a typed, non-string scalar for the `equals` matcher. String -/// equality is expressed with the `exact` match type, so this message -/// intentionally has no string variant. +/// Value carries a typed scalar for the `equals` matcher. /// /// `equals` matches when the field value has the same type and value as the set /// variant. Integer and floating-point values are compared in a single numeric @@ -139,14 +137,16 @@ pub const AttributePath = struct { /// equals: true -> bool_value /// equals: 200 -> int_value /// equals: 0.5 -> double_value +/// equals: "foo" -> string_value /// /// Bytes are authored either as proto-native base64 (`bytes_value`) or, more /// readably, as a lowercase-hex string (`hex_value`). The two are equivalent — /// hex_value is decoded to bytes at policy-compile time and yields the same bytes /// as the corresponding bytes_value — but hex_value keeps identifiers /// (trace/span ids) readable in the canonical proto/JSON form instead of base64. -/// A bare string literal (e.g. `equals: "foo"`) MUST be rejected — use `exact` -/// for strings. +/// +/// String equality SHOULD be authored with string_value. The older matcher-level +/// `exact` field is deprecated and will move to reserved in a future version. pub const Value = struct { value: ?value_union = null, @@ -156,6 +156,7 @@ pub const Value = struct { double_value, bytes_value, hex_value, + string_value, }; pub const value_union = union(_value_case) { bool_value: bool, @@ -163,12 +164,14 @@ pub const Value = struct { double_value: f64, bytes_value: []const u8, hex_value: []const u8, + string_value: []const u8, pub const _desc_table = .{ .bool_value = fd(1, .{ .scalar = .bool }), .int_value = fd(2, .{ .scalar = .int64 }), .double_value = fd(3, .{ .scalar = .double }), .bytes_value = fd(4, .{ .scalar = .bytes }), .hex_value = fd(5, .{ .scalar = .string }), + .string_value = fd(6, .{ .scalar = .string }), }; }; From e439ea787d59469f373bfcb1804a8b277a16672e Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Tue, 7 Jul 2026 14:58:31 -0400 Subject: [PATCH 2/8] remove old accessor --- src/bench/main.zig | 19 ++-- src/policy/log_transform.zig | 25 +++-- src/policy/policy_engine.zig | 174 +++++++++++++---------------------- src/policy/types.zig | 86 +++++++---------- 4 files changed, 123 insertions(+), 181 deletions(-) diff --git a/src/bench/main.zig b/src/bench/main.zig index c569c6d..4744e70 100644 --- a/src/bench/main.zig +++ b/src/bench/main.zig @@ -16,6 +16,7 @@ const TraceFieldRef = policy_zig.TraceFieldRef; const LogAccessor = policy_zig.LogAccessor; const MetricAccessor = policy_zig.MetricAccessor; const TraceAccessor = policy_zig.TraceAccessor; +const TypedValue = policy_zig.TypedValue; const NoopEventBus = o11y.NoopEventBus; const COUNTS = [_]usize{ 1, 10, 100, 1000 }; @@ -24,38 +25,38 @@ const COUNTS = [_]usize{ 1, 10, 100, 1000 }; const BenchLog = struct { body: []const u8, - fn access(ctx: *const anyopaque, field: FieldRef) ?[]const u8 { + fn access(ctx: *const anyopaque, field: FieldRef) ?TypedValue { const self: *const BenchLog = @ptrCast(@alignCast(ctx)); return switch (field) { - .log_field => |lf| if (lf == .LOG_FIELD_BODY) self.body else null, + .log_field => |lf| if (lf == .LOG_FIELD_BODY) .{ .string = self.body } else null, else => null, }; } - const accessor: LogAccessor = .{ .value = access }; + const accessor: LogAccessor = .{ .typed_value = access }; }; const BenchMetric = struct { name: []const u8, - fn access(ctx: *const anyopaque, field: MetricFieldRef) ?[]const u8 { + fn access(ctx: *const anyopaque, field: MetricFieldRef) ?TypedValue { const self: *const BenchMetric = @ptrCast(@alignCast(ctx)); return switch (field) { - .metric_field => |mf| if (mf == .METRIC_FIELD_NAME) self.name else null, + .metric_field => |mf| if (mf == .METRIC_FIELD_NAME) .{ .string = self.name } else null, else => null, }; } - const accessor: MetricAccessor = .{ .value = access }; + const accessor: MetricAccessor = .{ .typed_value = access }; }; const BenchTrace = struct { name: []const u8, - fn access(ctx: *const anyopaque, field: TraceFieldRef) ?[]const u8 { + fn access(ctx: *const anyopaque, field: TraceFieldRef) ?TypedValue { const self: *const BenchTrace = @ptrCast(@alignCast(ctx)); return switch (field) { - .trace_field => |tf| if (tf == .TRACE_FIELD_NAME) self.name else null, + .trace_field => |tf| if (tf == .TRACE_FIELD_NAME) .{ .string = self.name } else null, else => null, }; } - const accessor: TraceAccessor = .{ .value = access }; + const accessor: TraceAccessor = .{ .typed_value = access }; }; // --- JSON builders (allocator passed per-call, Zig 0.16 style) --- diff --git a/src/policy/log_transform.zig b/src/policy/log_transform.zig index bc792dd..5443fee 100644 --- a/src/policy/log_transform.zig +++ b/src/policy/log_transform.zig @@ -202,10 +202,9 @@ pub fn applyRemove( /// /// When `options.compiled` is null, replaces the entire field value with /// `rule.replacement`. When non-null, runs the pre-compiled regex over the -/// textual representation returned by `accessor.value` and substitutes each -/// match using the pre-parsed replacement template. If the accessor returns -/// null (field missing or non-string), or the regex finds no match, the -/// operation is a no-op. +/// `.string` value returned by `accessor.typed_value` and substitutes each +/// match using the pre-parsed replacement template. If the field is missing +/// or non-string, or the regex finds no match, the operation is a no-op. /// /// Returns true if the field was redacted. pub fn applyRedact( @@ -220,7 +219,9 @@ pub fn applyRedact( if (options.compiled) |c| { const allocator = options.scratch orelse return false; const io = options.io orelse return false; - const value = accessor.value(ctx, field_ref) orelse return false; + const tv = accessor.typed_value(ctx, field_ref) orelse return false; + if (tv != .string) return false; // regex redact no-ops on non-string values + const value = tv.string; var out: std.ArrayList(u8) = .empty; // No deinit: `out.items` is handed to the accessor.set by reference @@ -243,7 +244,7 @@ pub fn applyRedact( } // Whole-value redact: only fires if the field exists. - if (accessor.value(ctx, field_ref) == null) return false; + if (!accessor.callExists(ctx, field_ref)) return false; accessor.set.?(ctx, field_ref, rule.replacement); return true; @@ -414,8 +415,12 @@ const TestContext = struct { self.allocator.free(removed.value); } + fn accessorTypedValue(ctx: *const anyopaque, field: FieldRef) ?types.TypedValue { + return .{ .string = accessorValue(ctx, field) orelse return null }; + } + const accessor: LogAccessor = .{ - .value = accessorValue, + .typed_value = accessorTypedValue, .set = accessorSet, .delete = accessorDelete, .move = accessorMove, @@ -568,10 +573,10 @@ test "applyRedact: regex output outlives applyRedact return" { stored: ?[]const u8 = null, - fn valueFn(ctx: *const anyopaque, field: FieldRef) ?[]const u8 { + fn valueFn(ctx: *const anyopaque, field: FieldRef) ?types.TypedValue { _ = field; const self: *const Self = @ptrCast(@alignCast(ctx)); - return self.stored; + return .{ .string = self.stored orelse return null }; } fn setFn(ctx: *anyopaque, field: FieldRef, value: []const u8) void { @@ -585,7 +590,7 @@ test "applyRedact: regex output outlives applyRedact return" { } fn moveFn(_: *anyopaque, _: FieldRef, _: []const u8) void {} const accessor: LogAccessor = .{ - .value = valueFn, + .typed_value = valueFn, .set = setFn, .delete = deleteFn, .move = moveFn, diff --git a/src/policy/policy_engine.zig b/src/policy/policy_engine.zig index 1df9a3c..6b04477 100644 --- a/src/policy/policy_engine.zig +++ b/src/policy/policy_engine.zig @@ -487,7 +487,17 @@ pub const PolicyEngine = struct { // they have no Hyperscan DB and don't read the field value. if (!matcher_key.has_value_db) continue; - const raw_value = accessor.value(ctx, field_ref) orelse { + // String matchers consume the `.string` variant. Identifier + // fields arrive as `.bytes` and are hex-rendered before matching + // (policies write them as hex literals). Other variants are not + // string-matchable and skip the scan like an absent field. + var id_hex_buf: [max_hex_id_bytes * 2]u8 = undefined; + const value = blk: { + if (accessor.typed_value(ctx, field_ref)) |tv| switch (tv) { + .string => |s| break :blk s, + .bytes => |b| if (isHexBytesField(T, field_ref)) break :blk hexRenderId(b, &id_hex_buf), + else => {}, + }; if (self.bus.isEnabled(.debug)) { const event: MatcherKeyFieldNotPresent = .{ .telemetry_type = T, @@ -502,14 +512,6 @@ pub const PolicyEngine = struct { continue; }; - // Identifier fields are stored as raw bytes but written as hex - // literals in policies — render before string matching. - var id_hex_buf: [max_hex_id_bytes * 2]u8 = undefined; - const value = if (isHexBytesField(T, field_ref)) - hexRenderId(raw_value, &id_hex_buf) - else - raw_value; - if (self.bus.isEnabled(.debug)) { const event: MatcherKeyFieldValue = .{ .telemetry_type = T, @@ -577,12 +579,7 @@ pub const PolicyEngine = struct { // comparison fires (negation failed), leave alone when it doesn't // (negation succeeded). for (index.getTypedChecks()) |check| { - const typed_val: ?policy_types.TypedValue = if (comptime accessor.typed_value != null) - accessor.typed_value.?(ctx, check.field_ref) - else if (accessor.value(ctx, check.field_ref)) |s| - policy_types.TypedValue{ .string = s } - else - null; + const typed_val = accessor.typed_value(ctx, check.field_ref); const fired = check.matcher.evaluate(typed_val); @@ -612,10 +609,9 @@ pub const PolicyEngine = struct { /// Get raw bytes for probabilistic sampling. /// - /// Prefers `accessor.typed_value` to get `TypedValue.bytes` directly — - /// the sampler expects raw bytes and no longer accepts hex-encoded strings. - /// Falls back to `accessor.value` (treating the result as raw bytes) when - /// `typed_value` is not wired or returns a non-bytes variant. + /// The sampler expects raw bytes: `.bytes` values are used directly + /// (identifiers, no hex-encoded strings), `.string` values as their byte + /// content. Other variants cannot seed randomness and return null. /// /// - Traces: `TRACE_FIELD_TRACE_ID` raw bytes (16 bytes per OTel spec). /// - Logs with sample_key: sample key field value as raw bytes. @@ -626,27 +622,20 @@ pub const PolicyEngine = struct { ctx: *anyopaque, policy_info: *const PolicyInfo, ) ?[]const u8 { - if (T == .trace) { - const trace_id_ref: FieldRefType(T) = .{ .trace_field = .TRACE_FIELD_TRACE_ID }; - if (comptime accessor.typed_value != null) { - if (accessor.typed_value.?(ctx, trace_id_ref)) |tv| { - if (tv == .bytes) return tv.bytes; - } - } - return accessor.value(ctx, trace_id_ref); - } else if (T == .log) { - if (policy_info.sample_key) |sample_key| { - if (FieldRef.fromSampleKeyField(sample_key.field)) |field_ref| { - if (comptime accessor.typed_value != null) { - if (accessor.typed_value.?(ctx, field_ref)) |tv| { - if (tv == .bytes) return tv.bytes; - } - } - return accessor.value(ctx, field_ref); - } - } - } - return null; + const field_ref: FieldRefType(T) = switch (T) { + .trace => .{ .trace_field = .TRACE_FIELD_TRACE_ID }, + .log => blk: { + const sample_key = policy_info.sample_key orelse return null; + break :blk FieldRef.fromSampleKeyField(sample_key.field) orelse return null; + }, + .metric => return null, + }; + const tv = accessor.typed_value(ctx, field_ref) orelse return null; + return switch (tv) { + .bytes => |b| b, + .string => |s| s, + else => null, + }; } /// Find all matching policies, apply sampling/rate limiting, and determine final decision. @@ -709,7 +698,10 @@ pub const PolicyEngine = struct { if (T == .trace) { // Trace sampling: read tracestate, run full sample(), write threshold back const ts_ref: FieldRefType(T) = .{ .trace_field = .TRACE_FIELD_TRACE_STATE }; - const tracestate = accessor.value(ctx, ts_ref) orelse ""; + const tracestate = ts: { + const tv = accessor.typed_value(ctx, ts_ref) orelse break :ts ""; + break :ts if (tv == .string) tv.string else ""; + }; const sr = s.sample(input, tracestate); if (sr.keep) { @@ -930,7 +922,11 @@ const TestLogContext = struct { }; } - pub const accessor: LogAccessor = .{ .value = fieldAccessor }; + fn typedAccessor(ctx_ptr: *const anyopaque, field: FieldRef) ?TypedValue { + return .{ .string = fieldAccessor(ctx_ptr, field) orelse return null }; + } + + pub const accessor: LogAccessor = .{ .typed_value = typedAccessor }; }; fn evalTestLog( @@ -1841,8 +1837,12 @@ const MutableTestLogContext = struct { if (value) |v| self.setAttribute(to, v) catch return; } + fn typedAccessor(ctx_ptr: *const anyopaque, field: FieldRef) ?TypedValue { + return .{ .string = fieldAccessor(ctx_ptr, field) orelse return null }; + } + pub const accessor: LogAccessor = .{ - .value = fieldAccessor, + .typed_value = typedAccessor, .set = accessorSet, .delete = accessorDelete, .move = accessorMove, @@ -1927,15 +1927,6 @@ const TypedLogContext = struct { }; } - fn fieldValue(ctx_ptr: *const anyopaque, field: FieldRef) ?[]const u8 { - const self: *const TypedLogContext = @ptrCast(@alignCast(ctx_ptr)); - const key = attrKey(field) orelse return null; - return switch (self.attrs.get(key) orelse return null) { - .string => |s| s, - else => null, // non-string → null for string-match path - }; - } - fn fieldTypedValue(ctx_ptr: *const anyopaque, field: FieldRef) ?TypedValue { const self: *const TypedLogContext = @ptrCast(@alignCast(ctx_ptr)); const key = attrKey(field) orelse return null; @@ -1948,7 +1939,6 @@ const TypedLogContext = struct { } pub const accessor: LogAccessor = .{ - .value = fieldValue, .typed_value = fieldTypedValue, }; }; @@ -3462,7 +3452,11 @@ const TestMetricContext = struct { }; } - pub const accessor: MetricAccessor = .{ .value = fieldAccessor }; + fn typedAccessor(ctx_ptr: *const anyopaque, field: MetricFieldRef) ?TypedValue { + return .{ .string = fieldAccessor(ctx_ptr, field) orelse return null }; + } + + pub const accessor: MetricAccessor = .{ .typed_value = typedAccessor }; }; fn evalMetric( @@ -4139,38 +4133,10 @@ test "PolicyEngine: percentage sampling - 100% keeps all" { try testing.expectEqual(FilterDecision.keep, result.decision); } -test "PolicyEngine: percentage sampling - deterministic per context" { - const allocator = testing.allocator; - - var policy: Policy = .{ - .id = try allocator.dupe(u8, "policy-1"), - .name = try allocator.dupe(u8, "sample-50-percent"), - .enabled = true, - .target = .{ .log = .{ - .keep = try allocator.dupe(u8, "50%"), - } }, - }; - try policy.target.?.log.match.append(allocator, .{ - .field = .{ .log_field = .LOG_FIELD_BODY }, - .match = .{ .regex = try allocator.dupe(u8, "test") }, - }); - defer policy.deinit(allocator); - - var noop_bus: NoopEventBus = undefined; - noop_bus.init(std.Options.debug_io); - var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); - defer registry.deinit(); - try registry.updatePolicies(&.{policy}, "file-provider", .file); - - const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); - var policy_id_buf: [16][]const u8 = undefined; - - // Same context should produce same decision (deterministic) - var test_log: TestLogContext = .{ .message = "test message" }; - const result1 = evalTestLog(&engine, &test_log, &policy_id_buf); - const result2 = evalTestLog(&engine, &test_log, &policy_id_buf); - try testing.expectEqual(result1.decision, result2.decision); -} +// Note: deterministic percentage sampling requires a sample_key (see the +// "sample_key provides deterministic sampling" test). Without one, the spec +// (v1.6.0) requires an independent random decision per record — covered by +// "percentage sampling without sample_key is independent random". test "PolicyEngine: rate limiting - respects limit" { const allocator = testing.allocator; @@ -4798,27 +4764,16 @@ const TestTraceContext = struct { last_mutate_value: ?[]const u8 = null, mutate_count: usize = 0, - pub fn fieldAccessor(ctx_ptr: *const anyopaque, field: TraceFieldRef) ?[]const u8 { - const self: *const TestTraceContext = @ptrCast(@alignCast(ctx_ptr)); - return switch (field) { - .trace_field => |tf| switch (tf) { - .TRACE_FIELD_NAME => self.name, - .TRACE_FIELD_SPAN_ID => self.span_id, - .TRACE_FIELD_TRACE_STATE => self.trace_state, - // trace_id is bytes — return null from the string path so - // string matchers don't fire on it; sampling uses typed_value. - .TRACE_FIELD_TRACE_ID => null, - else => null, - }, - else => null, - }; - } - pub fn typedFieldAccessor(ctx_ptr: *const anyopaque, field: TraceFieldRef) ?TypedValue { const self: *const TestTraceContext = @ptrCast(@alignCast(ctx_ptr)); return switch (field) { .trace_field => |tf| switch (tf) { - .TRACE_FIELD_TRACE_ID => if (self.trace_id) |id| TypedValue{ .bytes = id } else null, + .TRACE_FIELD_NAME => .{ .string = self.name orelse return null }, + .TRACE_FIELD_SPAN_ID => .{ .string = self.span_id orelse return null }, + .TRACE_FIELD_TRACE_STATE => .{ .string = self.trace_state orelse return null }, + // trace_id is raw bytes: the engine hex-renders it for string + // matchers and hands the raw bytes to the sampler. + .TRACE_FIELD_TRACE_ID => .{ .bytes = self.trace_id orelse return null }, else => null, }, else => null, @@ -4838,7 +4793,6 @@ const TestTraceContext = struct { } pub const accessor: TraceAccessor = .{ - .value = fieldAccessor, .typed_value = typedFieldAccessor, .set = accessorSet, }; @@ -4938,17 +4892,17 @@ test "PolicyEngine: trace sampling drop does not write threshold" { try testing.expectEqual(@as(usize, 0), ctx.mutate_count); } -// Context whose `value(trace_id)` returns the raw 16 bytes (the canonical -// in-memory form for both wire paths). Exercises the engine's hex rendering of -// identifier bytes before string matching. +// Context whose `typed_value(trace_id)` returns the raw 16 bytes (the +// canonical in-memory form for both wire paths). Exercises the engine's hex +// rendering of identifier bytes before string matching. const TestHexTraceContext = struct { trace_id: ?[]const u8 = null, - pub fn fieldAccessor(ctx_ptr: *const anyopaque, field: TraceFieldRef) ?[]const u8 { + pub fn typedFieldAccessor(ctx_ptr: *const anyopaque, field: TraceFieldRef) ?TypedValue { const self: *const TestHexTraceContext = @ptrCast(@alignCast(ctx_ptr)); return switch (field) { .trace_field => |tf| switch (tf) { - .TRACE_FIELD_TRACE_ID => self.trace_id, + .TRACE_FIELD_TRACE_ID => .{ .bytes = self.trace_id orelse return null }, else => null, }, else => null, @@ -4957,7 +4911,7 @@ const TestHexTraceContext = struct { pub fn noopSet(_: *anyopaque, _: TraceFieldRef, _: []const u8) void {} - pub const accessor: TraceAccessor = .{ .value = fieldAccessor, .set = noopSet }; + pub const accessor: TraceAccessor = .{ .typed_value = typedFieldAccessor, .set = noopSet }; }; test "PolicyEngine: string matcher on trace_id renders raw bytes as hex" { diff --git a/src/policy/types.zig b/src/policy/types.zig index 84c7c82..909ccd0 100644 --- a/src/policy/types.zig +++ b/src/policy/types.zig @@ -287,12 +287,9 @@ pub const MetricFieldRef = union(enum) { /// A typed field value returned by the `typed_value` accessor primitive. /// -/// Used by the `equals`, `gt`, `gte`, `lt`, and `lte` matchers, which need -/// to compare against non-string values (booleans, integers, floats, bytes). -/// -/// The default string path (`value` accessor) is still used for string-typed -/// matchers and for `typed_value`-less accessors that fall back to -/// `TypedValue.string` wrapping. +/// String matchers (exact/regex/starts_with/ends_with/contains) consume the +/// `.string` variant; the typed matchers (`equals`, `gt`, `gte`, `lt`, `lte`) +/// compare all variants. /// /// `string` and `bytes` slices are borrowed from the consumer record and must /// remain valid for the duration of the `engine.evaluate` call. @@ -313,27 +310,29 @@ pub const TypedValue = union(enum) { /// the engine dispatches every read and write through these function /// pointers, with ctx being the consumer-owned record the pointers operate on. /// -/// `value` is the only required field. Optional primitives advertise consumer -/// capability: when an `apply*` transform needs a primitive that's null, the -/// transform no-ops (does not count toward `*_applied`) — so a consumer that -/// doesn't wire `set` simply won't apply redact/add even if a matching policy -/// fires. +/// `typed_value` is the only required field. Optional primitives advertise +/// consumer capability: when an `apply*` transform needs a primitive that's +/// null, the transform no-ops (does not count toward `*_applied`) — so a +/// consumer that doesn't wire `set` simply won't apply redact/add even if a +/// matching policy fires. pub const LogAccessor = struct { - /// Read field as bytes for pattern matching. - /// Returns null when the field is absent OR its underlying value is not a - /// string. Consumers that want exists-matchers to fire on non-string - /// fields should wire `exists` to report presence independently. + /// Read a field as a typed value. This is the single read primitive: + /// string matchers consume the `.string` variant, the typed matchers + /// (`equals`/`gt`/`gte`/`lt`/`lte`) compare all variants, sampling reads + /// `.bytes`/`.string`, and regex redact reads `.string`. + /// + /// Identifier fields (`LOG_FIELD_TRACE_ID`, `LOG_FIELD_SPAN_ID`): return + /// `.bytes` with the RAW id bytes. The engine hex-renders them for string + /// matchers (so hex literals in policies match) and uses the raw bytes + /// for sampling. Returning null hides the id from matching. /// - /// Exception — identifier fields (`LOG_FIELD_TRACE_ID`, `LOG_FIELD_SPAN_ID`): - /// return the RAW id bytes here, not null. The engine hex-renders them before - /// matching (so hex literals in policies match), and consults this primitive, - /// not `typed_value`, for string matchers. Returning null hides the id from - /// exact/contains/regex matching. - value: *const fn (ctx: *const anyopaque, field: FieldRef) ?[]const u8, + /// Return null when the field is absent. A present-but-wrong-type value + /// causes a non-match (never a runtime error) consistent with fail-open. + typed_value: *const fn (ctx: *const anyopaque, field: FieldRef) ?TypedValue, /// Returns true if the field is present, regardless of underlying type. - /// When null, the engine falls back to `value(...) != null`, preserving - /// the old "non-null means present" semantics. + /// When null, the engine falls back to `typed_value(...) != null`, + /// preserving the "non-null means present" semantics. exists: ?*const fn (ctx: *const anyopaque, field: FieldRef) bool = null, /// Upsert a field. The engine pre-checks `upsert=false` conflicts via @@ -364,37 +363,24 @@ pub const LogAccessor = struct { /// Wiring this enables: log.rename. move: ?*const fn (ctx: *anyopaque, from: FieldRef, to: []const u8) void = null, - /// Read a field as a typed value for `equals`/`gt`/`gte`/`lt`/`lte` matchers. - /// - /// When null, the engine falls back to `value()` and treats the result as - /// `TypedValue.string` — typed matchers will fire only against string fields. - /// Override to expose int/double/bool/bytes attribute values so typed - /// matchers can compare against them correctly. - /// - /// Return null when the field is absent. A present-but-wrong-type value - /// causes a non-match (never a runtime error) consistent with fail-open. - typed_value: ?*const fn (ctx: *const anyopaque, field: FieldRef) ?TypedValue = null, - /// Returns true if the field is present. Uses the wired `exists` primitive - /// when available, otherwise falls back to `value != null`. + /// when available, otherwise falls back to `typed_value != null`. pub fn callExists(self: *const LogAccessor, ctx: *const anyopaque, field: FieldRef) bool { if (self.exists) |f| return f(ctx, field); - return self.value(ctx, field) != null; + return self.typed_value(ctx, field) != null; } }; /// Read+write interface to a metric record. pub const MetricAccessor = struct { - value: *const fn (ctx: *const anyopaque, field: MetricFieldRef) ?[]const u8, + /// Read a field as a typed value. See `LogAccessor.typed_value`. + typed_value: *const fn (ctx: *const anyopaque, field: MetricFieldRef) ?TypedValue, exists: ?*const fn (ctx: *const anyopaque, field: MetricFieldRef) bool = null, - /// Typed read for `equals`/`gt`/`gte`/`lt`/`lte` matchers. See `LogAccessor.typed_value`. - typed_value: ?*const fn (ctx: *const anyopaque, field: MetricFieldRef) ?TypedValue = null, - pub fn callExists(self: *const MetricAccessor, ctx: *const anyopaque, field: MetricFieldRef) bool { if (self.exists) |f| return f(ctx, field); - return self.value(ctx, field) != null; + return self.typed_value(ctx, field) != null; } }; @@ -403,26 +389,22 @@ pub const MetricAccessor = struct { /// writes when probabilistic sampling produces a threshold; consumers wire /// `set` to merge that threshold into their tracestate representation. pub const TraceAccessor = struct { - /// Read field as bytes for pattern matching; null when absent/non-string. + /// Read a field as a typed value. See `LogAccessor.typed_value`. /// - /// Exception — identifier fields (`TRACE_FIELD_TRACE_ID`, - /// `TRACE_FIELD_SPAN_ID`, `TRACE_FIELD_PARENT_SPAN_ID`): return the RAW id - /// bytes here, not null. The engine hex-renders them before matching and - /// consults this primitive, not `typed_value`, for string matchers. - /// Returning null hides the id from exact/contains/regex matching. - value: *const fn (ctx: *const anyopaque, field: TraceFieldRef) ?[]const u8, + /// Identifier fields (`TRACE_FIELD_TRACE_ID`, `TRACE_FIELD_SPAN_ID`, + /// `TRACE_FIELD_PARENT_SPAN_ID`): return `.bytes` with the RAW id bytes; + /// the engine hex-renders them for string matchers and uses the raw bytes + /// for sampling. + typed_value: *const fn (ctx: *const anyopaque, field: TraceFieldRef) ?TypedValue, exists: ?*const fn (ctx: *const anyopaque, field: TraceFieldRef) bool = null, /// Wiring this enables: trace sampling threshold writeback. set: ?*const fn (ctx: *anyopaque, field: TraceFieldRef, value: []const u8) void = null, - /// Typed read for `equals`/`gt`/`gte`/`lt`/`lte` matchers. See `LogAccessor.typed_value`. - typed_value: ?*const fn (ctx: *const anyopaque, field: TraceFieldRef) ?TypedValue = null, - pub fn callExists(self: *const TraceAccessor, ctx: *const anyopaque, field: TraceFieldRef) bool { if (self.exists) |f| return f(ctx, field); - return self.value(ctx, field) != null; + return self.typed_value(ctx, field) != null; } }; From 0a2f753e3bae264015b888083b11107abae3e1b7 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Tue, 7 Jul 2026 17:28:55 -0400 Subject: [PATCH 3/8] design --- Claude.md | 1144 ++++++++++++++++++++++++++++++++++++++++++ design/extensions.md | 479 ++++++++++++++++++ 2 files changed, 1623 insertions(+) create mode 100644 Claude.md create mode 100644 design/extensions.md diff --git a/Claude.md b/Claude.md new file mode 100644 index 0000000..9343c4b --- /dev/null +++ b/Claude.md @@ -0,0 +1,1144 @@ +# Policy-zig - Project Overview + +## Architecture + +This is a Zig project implementing a computing runtime with a hybrid +architectural approach: + +- **Data-Oriented Design (DoD)**: Primary focus for performance-critical paths +- **Functional Programming**: For composability and testability +- **Object-Oriented Programming**: Where it provides clarity without sacrificing + performance + +## Data-Oriented Design Principles + +### Core Philosophy + +**The CPU is fast, but memory is slow.** All design decisions must optimize for +memory access patterns and cache coherency. **Predictable memory usage is the +goal.** Each binary should have a predictable memory footprint based on +throughput. + +### Key Strategies + +#### 1. Identify and Optimize Uniform Data + +- Group similar data together in memory +- Minimize the size of each individual item +- Process data in bulk rather than one item at a time + +#### 2. Use Indexes Instead of Pointers + +```zig +// BAD: Pointer-heavy approach (64 bits per reference) +const Node = struct { + data: Data, + next: ?*Node, + parent: ?*Node, +}; + +// GOOD: Index-based approach (typically 32 bits or less) +const NodeId = u32; +const Node = struct { + data: Data, + next: ?NodeId, + parent: ?NodeId, +}; +const NodeStorage = std.ArrayList(Node); +``` + +#### 3. Manage Type Safety with Compact Representations + +```zig +// Use strongly-typed handles instead of raw integers +const HandleType = enum { request, response, connection }; +fn Handle(comptime T: HandleType) type { + return enum(u32) { _ }; +} + +const RequestHandle = Handle(.request); +const ResponseHandle = Handle(.response); +``` + +#### 4. Store Booleans Separately + +SKIP THIS FOR NOW. + +#### 5. Eliminate Padding with Struct-of-Arrays + +```zig +// BAD: Array-of-Structs (AoS) - poor cache utilization +const Connection = struct { + id: u64, // 8 bytes + status: u8, // 1 byte + 3 bytes padding + port: u16, // 2 bytes + 2 bytes padding + address: [16]u8, // 16 bytes +}; +const connections = std.ArrayList(Connection); + +// GOOD: Struct-of-Arrays (SoA) - excellent cache utilization +const Connection = struct { + id: u64, // 8 bytes + status: u8, // 1 byte + 3 bytes padding + port: u16, // 2 bytes + 2 bytes padding + address: [16]u8, // 16 bytes +}; +const connections = std.MultiArrayList(Connection); + +``` + +#### 6. Use Hash Maps for Sparse Data + +```zig +// When only a small subset of entities has certain data +const SparseMetadata = std.AutoHashMap(EntityId, Metadata); +``` + +#### 7. Encoding Instead of Polymorphism + +```zig +// BAD: Runtime polymorphism (vtable overhead) +const Handler = struct { + vtable: *const VTable, + data: *anyopaque, +}; + +// GOOD: Tagged unions with explicit types +const HandlerType = enum { http, websocket, grpc }; +const Handler = union(HandlerType) { + http: HttpHandler, + websocket: WebSocketHandler, + grpc: GrpcHandler, + + fn process(self: *Handler, data: []const u8) void { + switch (self.*) { + .http => |*h| h.processHttp(data), + .websocket => |*w| w.processWs(data), + .grpc => |*g| g.processGrpc(data), + } + } +}; +``` + +## Module System + +### Philosophy + +The codebase is organized into **composable modules** that can be selectively +combined to create different distributions. Each module should: + +1. Be self-contained with minimal dependencies +2. Export a clear public API through its root file +3. Define its own types and data structures +4. Be independently testable + +### Structure + +``` +src/ +├── core/ # Core runtime primitives +│ ├── allocator.zig +│ ├── event_loop.zig +│ └── scheduler.zig +├── network/ # Network stack +│ ├── tcp.zig +│ ├── udp.zig +│ └── http.zig +├── storage/ # Storage layer +│ ├── cache.zig +│ └── persistence.zig +├── security/ # Security primitives +│ ├── tls.zig +│ └── auth.zig +└── distributions/ # Distribution-specific compositions + ├── minimal.zig + ├── full.zig + └── custom.zig +``` + +### Distribution Composition + +Each distribution manually selects and composes the modules it needs: + +```zig +// distributions/minimal.zig +const core = @import("../core/event_loop.zig"); +const network = @import("../network/tcp.zig"); + +pub fn main() !void { + var runtime = try core.Runtime.init(); + defer runtime.deinit(); + + var tcp_server = try network.TcpServer.init(&runtime); + defer tcp_server.deinit(); + + try runtime.run(); +} +``` + +## Code Organization Best Practices + +### 1. Separate Hot and Cold Data + +```zig +// Hot data: accessed frequently +const EntityHot = struct { + position: Vec3, + velocity: Vec3, +}; + +// Cold data: accessed rarely +const EntityCold = struct { + name: []const u8, + metadata: Metadata, +}; + +const EntityId = u32; +const hot_data = std.ArrayList(EntityHot); +const cold_data = std.AutoHashMap(EntityId, EntityCold); +``` + +### 2. Batch Processing + +```zig +// Process data in batches for better cache utilization +fn updatePositions(entities: []Entity, dt: f32) void { + for (entities) |*entity| { + entity.position.x += entity.velocity.x * dt; + entity.position.y += entity.velocity.y * dt; + entity.position.z += entity.velocity.z * dt; + } +} +``` + +### 3. Memory Alignment + +```zig +// Align data structures for optimal CPU access +const CacheLine = 64; // bytes +const alignas = CacheLine; + +const PerformanceCritical = struct { + // Hot data aligned to cache line + data: [16]u32 align(alignas), +}; +``` + +### 4. Arena Allocators for Temporary Data + +```zig +fn processRequest(arena: *std.heap.ArenaAllocator, request: Request) !Response { + // All temporary allocations use arena + const temp_buffer = try arena.allocator().alloc(u8, 1024); + // No need to free - arena will be reset + + // ... process request ... + + return response; +} +``` + +## Testing Strategy + +- **Unit tests**: Test individual functions with `test` blocks +- **Integration tests**: Test module interactions +- **Benchmark tests**: Verify performance characteristics using `std.time` +- **Memory tests**: Use `std.testing.allocator` to detect leaks + +## Performance Guidelines + +1. **Profile before optimizing** - Use `std.time.Timer` and platform profilers +2. **Minimize allocations** - Reuse buffers where possible +3. **Prefer stack over heap** - Use fixed-size buffers when bounds are known +4. **Avoid indirection** - Direct function calls over function pointers when + possible +5. **Think in terms of data transformations** - Input → Process → Output + +## Functional Programming Aspects + +- Use pure functions where possible (no side effects) +- Leverage comptime for zero-cost abstractions +- Prefer explicit error handling with `!` and `catch` +- Use const by default, mut when necessary + +## When to Use OOP + +Object-oriented patterns are acceptable when: + +- They improve code clarity without performance cost +- The domain naturally maps to objects (e.g., Connection, Server) +- Methods help organize related functionality +- State encapsulation provides safety + +**Always measure**: If OOP introduces indirection or hurts cache coherency, +refactor to DoD. + +## Build System + +The project uses Zig's build system (`build.zig`) to: + +- Define multiple build targets (distributions) +- Manage dependencies +- Configure compilation options +- Run tests + +## Getting Started + +When modifying or extending this project: + +1. Identify the module(s) you need to change +2. Consider the data layout and access patterns +3. Prefer transforming contiguous arrays over scattered object updates +4. Write tests to verify correctness +5. Benchmark to verify performance +6. Update relevant distributions if needed + +## Questions to Ask Before Coding + +1. How will this data be accessed? (Sequential, random, sparse?) +2. What is the typical working set size? +3. Can I process this in batches? +4. Are there hot/cold splits in this data? +5. Can I use indexes instead of pointers? +6. Does this need to be in the hot path? + +--- + +Remember: **Optimize for data locality first, algorithmic complexity second.** + +--- + +# Zig 0.15.x std.io Breaking Changes Reference + +> **HISTORICAL (2026-06):** the codebase is now fully Zig 0.16-native — std.Io +> is an injected interface (io parameter everywhere), the HTTP data plane rides +> std.Io.net + std.http.Server with structured cancellation (Io.Group), and +> request bodies stream through src/pipeline/. The notes below describe the +> 0.14→0.15 migration era and are kept for context only; verify any API against +> zigdoc before relying on it. + +**Zig 0.15.x introduced "Writergate"—a complete redesign of std.io Reader and +Writer interfaces that fundamentally changes how I/O operations work.** The new +interfaces replace generic `anytype`-based APIs with concrete types that +integrate buffering directly into the interface itself. This change delivers +45-49% performance improvements but requires explicit buffer management and +manual flushing. All existing code using std.io readers and writers will break +and must be migrated. + +## Why this matters for Tero policy-zig + +These changes are **extremely breaking** (Andrew Kelley's words) and affect +every file, network, and console I/O operation. The new architecture eliminates +the generic `std.io.GenericReader` and `std.io.GenericWriter` types, requiring +buffers to be provided at instantiation time. Code that worked in Zig 0.14.x +will fail to compile or produce silent failures (missing output due to unflushed +buffers) in 0.15.x. This redesign is permanent and foundational for Zig's future +async/await implementation—there is no going back to the old interfaces. + +Understanding these changes is critical for Tero policy-zig because the project +needs current, accurate I/O patterns. The old patterns are now deprecated or +completely removed. Future AI assistants must know: **you cannot copy interface +instances, you must always flush writers, and buffers are mandatory**. These +aren't optional optimizations—they're fundamental requirements of the new API. + +--- + +## The fundamental architectural shift + +Zig 0.15.x moves from generic, implementation-defined buffering to concrete, +interface-level buffering. The **buffer now lives in the interface itself** +(above the vtable), not in the implementation. This "buffer above vtable" design +enables the hot path—most I/O operations—to work directly on the buffer without +vtable calls, while only calling into the vtable when the buffer fills or +empties. + +**Old architecture (0.14.x):** Generic types like `std.io.GenericReader(T)` and +`std.io.GenericWriter(T)` wrapped underlying resources. Buffering was optional +via separate `bufferedReader()` and `bufferedWriter()` wrappers. Every function +accepting a reader or writer became generic with `anytype` parameters, poisoning +struct types. + +**New architecture (0.15.x):** Concrete `std.Io.Reader` and `std.Io.Writer` +types with integrated buffers. No more generics for basic I/O. Buffer is a +required field in the interface struct itself, making buffering the default +behavior rather than an opt-in wrapper. + +The performance gains are substantial: **45.7% faster wall time**, 49% fewer CPU +cycles, and 24.4% fewer instructions in I/O-heavy operations. Debug mode +compilation is roughly 5x faster using the self-hosted x86 backend. The +trade-off is explicit complexity—developers must manage buffers and remember to +flush. + +--- + +## Critical breaking change 1: Explicit buffer requirements + +Every reader and writer instantiation **must provide a buffer**. There is no +default allocation, no hidden memory management. This is Zig philosophy: no +hidden control flow, no surprise allocations. + +**Code that breaks:** + +```zig +// 0.14.x - worked fine +const stdout = std.io.getStdOut().writer(); +try stdout.print("Hello\n", .{}); +``` + +**Required pattern in 0.15.x:** + +```zig +// Must allocate buffer explicitly +var stdout_buffer: [4096]u8 = undefined; +var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); +const stdout = &stdout_writer.interface; +try stdout.print("Hello\n", .{}); +try stdout.flush(); // CRITICAL: Must flush! +``` + +**Buffer sizing guidelines:** + +- **4096 bytes (one page):** Standard for general I/O operations +- **8192 bytes:** Better for high-throughput scenarios +- **std.crypto.tls.max_ciphertext_record_len:** Required minimum for TLS + operations +- **Empty buffer `&.{}`:** Opt-out to unbuffered mode (not recommended) + +Buffers can be stack-allocated for short-lived operations or heap-allocated for +long-running resources. The official recommendation is to **consider making +frequently-used buffers (like stdout) global** to avoid repeated stack +allocation overhead. + +--- + +## Critical breaking change 2: The flush requirement + +Buffered data **does not automatically commit**. Forgetting `flush()` causes +silent failures where programs run successfully but produce no output. This is +the single most common migration bug. + +**Why flushing is mandatory:** The interface accumulates data in its buffer to +reduce system calls. Until you call `flush()`, the data remains in memory, never +reaching the file descriptor, socket, or console. + +**When you must flush:** + +- Before program exit +- After writing data you expect to see immediately +- Before switching between read and write operations on the same resource +- Before passing control to code that might read what you wrote +- After writing to stdout/stderr if you need immediate visibility + +**Example of silent failure:** + +```zig +var buffer: [4096]u8 = undefined; +var writer = std.fs.File.stdout().writer(&buffer); +const stdout = &writer.interface; +try stdout.print("Important message\n", .{}); +// Program ends - buffer never flushed, output never appears! +``` + +**Correct pattern:** + +```zig +var buffer: [4096]u8 = undefined; +var writer = std.fs.File.stdout().writer(&buffer); +const stdout = &writer.interface; +try stdout.print("Important message\n", .{}); +try stdout.flush(); // Now output appears +``` + +--- + +## Critical breaking change 3: The @fieldParentPtr footgun + +The new interfaces use `@fieldParentPtr` internally to recover the parent +wrapper struct from the interface field. This creates a **critical invariant: +you must never copy interface instances**. + +**What breaks:** + +```zig +var file_reader = file.reader(&buf); +const reader = file_reader.interface; // WRONG - copies the interface +while (reader.takeDelimiterExclusive('\n')) |line| { + // Crashes with "switch on corrupt value" or undefined behavior +} +``` + +**Correct pattern:** + +```zig +var file_reader = file.reader(&buf); +const reader = &file_reader.interface; // RIGHT - pointer to interface +while (reader.takeDelimiterExclusive('\n')) |line| { + // Works correctly +} +``` + +**Why this happens:** When you copy the interface, the vtable methods call +`@fieldParentPtr` expecting to find the parent struct at a specific memory +offset. But the copied interface is no longer at that offset relative to the +parent, causing memory corruption or crashes. + +**Rule of thumb:** Interface types are always passed as pointers +(`*std.Io.Reader`, `*std.Io.Writer`). Never use them as values. + +--- + +## New Reader interface and methods + +The `std.Io.Reader` type replaces `std.io.GenericReader` and all reader-related +generic types. It's a concrete struct with three fields: + +```zig +pub const Reader = struct { + vtable: *const VTable, + buffer: []u8, // Ring buffer + seek: usize = 0, // Bytes consumed from buffer + end: usize = 0, // End of buffered data +}; +``` + +**Key methods that replace old APIs:** + +**`takeDelimiterExclusive(delimiter: u8)`** replaces +`readUntilDelimiterOrEof()`: + +```zig +while (reader.takeDelimiterExclusive('\n')) |line| { + // Process line (delimiter not included) +} else |err| switch (err) { + error.EndOfStream => {}, // Normal end of stream + error.StreamTooLong => {}, // Line exceeds buffer size + error.ReadFailed => return err, // Actual I/O error +} +``` + +**`stream(writer: *Writer, limit: StreamLimit)`** for copying data: + +```zig +// Copy up to 1024 bytes from reader to writer +const n = try reader.stream(writer, .limited(1024)); + +// Copy until end of stream +const n = try reader.stream(writer, .until_end); +``` + +**`peek()` and `take()`** for zero-copy buffer access: + +```zig +const available = reader.peek(); // View buffered data without consuming +const slice = try reader.take(100); // Take exactly 100 bytes +``` + +**`toss(n: usize)`** to discard data: + +```zig +// Skip delimiter byte after streamDelimiter +try reader.toss(1); +``` + +**Removed methods from 0.14.x:** + +- `readUntilDelimiterOrEof()` → Use `takeDelimiterExclusive()` +- `readAll()` → Use `stream()` with appropriate limit +- `readNoEof()` → Use `take()` +- Generic variants → All replaced by concrete methods + +--- + +## New Writer interface and methods + +The `std.Io.Writer` type replaces `std.io.GenericWriter` with a concrete +implementation: + +```zig +pub const Writer = struct { + vtable: *const VTable, + buffer: []u8, + end: usize = 0, // Current buffer fill level +}; +``` + +**Essential methods:** + +**`writeAll(bytes: []const u8)`** for writing complete slices: + +```zig +try writer.writeAll("Complete message\n"); +``` + +**`print(comptime fmt: []const u8, args: anytype)`** for formatted output: + +```zig +try writer.print("Value: {d}\n", .{42}); +``` + +**`flush()`** to commit buffered data: + +```zig +try writer.flush(); // Calls drain() on implementation +``` + +**`drain()`** (vtable method) for custom implementations—writes buffered data to +underlying resource. + +**New specialized writers:** + +**`std.Io.Writer.Allocating`** for dynamic allocation: + +```zig +var writer = std.Io.Writer.Allocating.init(allocator); +defer writer.deinit(); +try writer.writer.writeAll("Data"); +const result = writer.written(); // Get accumulated slice +``` + +**`std.Io.Writer.Discarding`** for counting without storage: + +```zig +var counter = std.Io.Writer.Discarding{}; +try counter.writer.writeAll(data); +const bytes_written = counter.count; +``` + +**`std.Io.Writer.fixed(buffer: []u8)`** for fixed buffers: + +```zig +var buf: [1024]u8 = undefined; +var writer = std.Io.Writer.fixed(&buf); +try writer.writeAll("data"); +``` + +--- + +## Stream interfaces are gone + +The `std.io.SeekableStream` type and related abstractions **have been completely +removed**. There is no direct replacement—use specific concrete types instead. + +**Migration paths:** + +**For file operations:** + +- Use `*std.fs.File.Reader` with positional read support +- Use `*std.fs.File.Writer` with positional write support +- These types memoize file information (size, position) for efficient seeking + +**For memory operations:** + +- Use `std.ArrayListUnmanaged(u8)` as a growable buffer +- Use `std.Io.Writer.fixed()` for fixed-size memory buffers + +**For network operations:** + +- `std.net.Stream.reader()` and `std.net.Stream.writer()` now return + `std.fs.File.Reader` and `std.fs.File.Writer` wrappers +- Must provide buffers to these methods + +**Example migration:** + +```zig +// 0.14.x - had SeekableStream +var seekable = file.seekableStream(); +try seekable.seekTo(100); + +// 0.15.x - use File.Reader with positional reads +var buf: [4096]u8 = undefined; +var reader = file.reader(&buf); +// Positional reads are automatic when file supports them +``` + +--- + +## File operations now require explicit buffering + +The `std.fs.File` type's reader and writer methods **require buffer +parameters**. + +**Old pattern (0.14.x):** + +```zig +const file = try std.fs.cwd().openFile("data.txt", .{}); +const reader = file.reader(); // No buffer needed +``` + +**New pattern (0.15.x):** + +```zig +const file = try std.fs.cwd().openFile("data.txt", .{}); +var read_buffer: [4096]u8 = undefined; +var file_reader = file.reader(&read_buffer); +const reader = &file_reader.interface; +``` + +**File.Reader benefits:** + +- Memoizes file size and position +- Automatically uses positional reads when available (seeking becomes no-ops) +- Supports `sendFile()` for zero-copy file-to-file transfers +- Falls back gracefully to streaming when positional I/O unavailable + +**File.Writer benefits:** + +- Similar memoization for write operations +- Integrated buffering reduces system calls +- Positional writes when supported + +**Removed file APIs:** + +- `writeFileAll()` → Use `File.Writer` methods +- `writeFileAllUnseekable()` → Use `File.Writer` methods +- Direct `WriteFileOptions` → Pass buffer to `writer()` method + +**Changed APIs:** + +- `std.fs.Dir.atomicFile()` now requires `write_buffer` in options +- `std.fs.Dir.copyFile()` can no longer fail with `error.OutOfMemory` (uses + stack-allocated buffers) + +--- + +## Complete list of std.io namespace changes + +**Fully removed (no deprecation period):** + +- `std.io.SeekableStream` → Use concrete file/memory types +- `std.io.BitReader` → Removed entirely +- `std.io.BitWriter` → Removed entirely +- `std.io.LimitedReader` → Use `Reader.stream()` with `.limited()` +- `std.fifo.LinearFifo` → Replaced by Reader/Writer as ring buffers +- `std.RingBuffer` → Obsolete with new interfaces +- All ring buffer implementations (5 different types consolidated) + +**Deprecated (warnings, will be removed):** + +- `std.io.GenericReader` → Use `std.Io.Reader` +- `std.io.GenericWriter` → Use `std.Io.Writer` +- `std.io.AnyReader` → Use `std.Io.Reader` +- `std.io.AnyWriter` → Use `std.Io.Writer` +- `std.io.bufferedReader()` → Buffering now built-in +- `std.io.bufferedWriter()` → Buffering now built-in + +**New APIs replacing old patterns:** + +- `takeDelimiterExclusive()` replaces `readUntilDelimiterOrEof()` +- `stream()` replaces `readAll()` and related methods +- `Reader.pull()` (vtable) replaces custom read implementations +- `Writer.drain()` (vtable) replaces custom write implementations + +**Migration adapter (temporary):** + +```zig +// For gradual migration only - has known bugs +fn foo(old_writer: anytype) !void { + var adapter = old_writer.adaptToNewApi(&.{}); + const w: *std.Io.Writer = &adapter.new_interface; + try w.print("{s}", .{"example"}); +} +``` + +This adapter is **not recommended for production** and may not work in all cases +(GitHub issue #24483 documents limitations). + +--- + +## HTTP Client and Server changes + +Both `std.http.Client` and `std.http.Server` underwent major restructuring due +to the I/O changes. + +**HTTP Server migration:** + +```zig +// 0.14.x +var read_buffer: [8000]u8 = undefined; +var server = std.http.Server.init(connection, &read_buffer); + +// 0.15.x - separate buffers for read and write +var recv_buffer: [4000]u8 = undefined; +var send_buffer: [4000]u8 = undefined; +var conn_reader = connection.stream.reader(&recv_buffer); +var conn_writer = connection.stream.writer(&send_buffer); +var server = std.http.Server.init(conn_reader.interface(), &conn_writer.interface); +``` + +**HTTP Client migration:** + +```zig +// 0.14.x +var server_header_buffer: [1024]u8 = undefined; +var req = try client.open(.GET, uri, .{ + .server_header_buffer = &server_header_buffer, +}); +defer req.deinit(); +try req.send(); +try req.wait(); +const body_reader = try req.reader(); + +// 0.15.x - new request/response pattern +var req = try client.request(.GET, uri, .{}); +defer req.deinit(); +try req.sendBodiless(); +var response = try req.receiveHead(&.{}); + +// Process headers before reading body +var it = response.head.iterateHeaders(); +while (it.next()) |header| { + // Headers become invalid after reader() call +} + +var reader_buffer: [100]u8 = undefined; +const body_reader = response.reader(&reader_buffer); +``` + +**Key architectural improvement:** HTTP modules no longer depend on `std.net`. +They operate purely on Reader/Writer interfaces, making them more testable and +flexible. + +--- + +## TLS client requires four separate buffers + +The `std.crypto.tls.Client` type now requires **four distinct buffers** for +operation: + +```zig +// 1. Network stream read buffer +var stream_read_buf: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined; +var stream_reader = stream.reader(&stream_read_buf); + +// 2. Network stream write buffer +var stream_write_buf: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined; +var stream_writer = stream.writer(&stream_write_buf); + +// 3. TLS client read buffer +var tls_read_buf: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined; + +// 4. TLS client write buffer +var tls_write_buf: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined; + +var tls_client = try std.crypto.tls.Client.init( + stream_reader.interface(), + &stream_writer.interface, + .{ + .ca = .{.bundle = bundle}, + .host = .{ .explicit = "example.com" }, + .read_buffer = &tls_read_buf, + .write_buffer = &tls_write_buf, + }, +); +``` + +This complexity has been controversial in the community, but reflects the +layered nature of TLS: the outer layer buffers network I/O, the inner layer +buffers TLS protocol operations. + +**Benefit:** `std.crypto.tls.Client` no longer depends on `std.net` or `std.fs`, +operating purely on abstract I/O interfaces. + +--- + +## Compression API restructured + +The `std.compress.flate` module underwent major changes: + +**Decompression (still supported):** + +```zig +var decompress_buffer: [std.compress.flate.max_window_len]u8 = undefined; +var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &decompress_buffer); +const decompress_reader: *std.Io.Reader = &decompress.reader; + +// Or stream directly to writer +var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{}); +const n = try decompress.streamRemaining(writer); +``` + +**Compression (temporarily removed):** Deflate compression functionality was +removed in 0.15.x. The official release notes state this is temporary, +prioritizing language design over standard library completeness. Third-party +compression libraries are recommended for now. + +**Container formats:** The API now requires explicit container parameter +(`.zlib`, `.gzip`, `.raw`) instead of inferring from context. + +--- + +## Format string changes require migration + +Custom format implementations must be updated to the new signature: + +**Old format method (0.14.x):** + +```zig +pub fn format( + this: @This(), + comptime format_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, +) !void { + try writer.print("Custom: {}", .{this.value}); +} +``` + +**New format method (0.15.x):** + +```zig +pub fn format( + this: @This(), + writer: *std.Io.Writer +) std.Io.Writer.Error!void { + try writer.print("Custom: {}", .{this.value}); +} +``` + +**Format specifier changes:** + +- **`{}`** is now ambiguous if type has a `format()` method (compilation error) +- **`{f}`** explicitly calls the `format()` method +- **`{any}`** explicitly skips the `format()` method, using default + representation + +**Why this changed:** Prevents silent bugs where adding or removing a format +method changes output behavior without any code changes at call sites. + +**Migration strategy:** Use `zig build -freference-trace` to find all format +string issues. The compiler will show exact locations where format specifiers +need updating. + +--- + +## New best practices for Zig 0.15.x I/O + +**1. Default to buffering with appropriate sizes** + +Use 4096 bytes (one memory page) as the standard buffer size for general I/O. +Increase to 8192 or more for high-throughput operations. Use +`std.crypto.tls.max_ciphertext_record_len` for TLS operations. + +```zig +// Standard pattern +var buffer: [4096]u8 = undefined; +var writer = file.writer(&buffer); +``` + +**2. Always flush before visibility is needed** + +Flush writers before program exit, before reading back from the same resource, +and whenever output must be immediately visible. + +```zig +try writer.print("Status: {s}\n", .{status}); +try writer.flush(); // Make status visible now +``` + +**3. Use pointers for interfaces, never copy** + +Always reference interfaces via pointer to avoid the @fieldParentPtr footgun. + +```zig +const writer = &file_writer.interface; // RIGHT +``` + +**4. Consider global buffers for frequently-used I/O** + +For stdout/stderr used throughout a program, making the buffer global avoids +repeated stack allocation and simplifies code. + +```zig +var stdout_buffer: [8192]u8 = undefined; + +pub fn main() !void { + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + const stdout = &stdout_writer.interface; + // Use stdout throughout program +} +``` + +**5. Handle all error cases from new Reader methods** + +The new delimiter methods return distinct error types that should be handled +explicitly. + +```zig +while (reader.takeDelimiterExclusive('\n')) |line| { + // Process line +} else |err| switch (err) { + error.EndOfStream => {}, // Normal completion + error.StreamTooLong => return err, // Line exceeded buffer + error.ReadFailed => return err, // I/O failure +} +``` + +**6. Use Writer.Allocating for unbounded data** + +When reading lines that may exceed buffer size, use allocating writer: + +```zig +var line_writer = std.Io.Writer.Allocating.init(allocator); +defer line_writer.deinit(); + +while (reader.streamDelimiter(&line_writer.writer, '\n')) |_| { + const line = line_writer.written(); + // Process line + line_writer.clearRetainingCapacity(); + try reader.toss(1); // Skip delimiter +} else |err| if (err != error.EndOfStream) return err; +``` + +**7. Accept *std.Io.Reader and *std.Io.Writer in public APIs** + +Use concrete interface types in function signatures instead of `anytype`: + +```zig +pub fn processData(reader: *std.Io.Reader, writer: *std.Io.Writer) !void { + // No longer forces calling code to be generic +} +``` + +--- + +## Common migration patterns + +**Pattern: Stdout printing** + +```zig +// Allocate buffer once (can be global) +var stdout_buffer: [4096]u8 = undefined; +var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); +const stdout = &stdout_writer.interface; + +// Use throughout program +try stdout.print("Message: {s}\n", .{message}); +try stdout.flush(); // Don't forget! +``` + +**Pattern: Reading file line by line** + +```zig +const file = try std.fs.cwd().openFile("data.txt", .{}); +defer file.close(); + +var buffer: [8192]u8 = undefined; +var file_reader = file.reader(&buffer); +const reader = &file_reader.interface; + +while (reader.takeDelimiterExclusive('\n')) |line| { + // Process each line +} else |err| if (err != error.EndOfStream) return err; +``` + +**Pattern: Writing to file** + +```zig +const file = try std.fs.cwd().createFile("output.txt", .{}); +defer file.close(); + +var buffer: [4096]u8 = undefined; +var file_writer = file.writer(&buffer); +const writer = &file_writer.interface; + +try writer.writeAll("Content\n"); +try writer.flush(); // Ensure data written +``` + +**Pattern: Network socket I/O** + +```zig +const stream = try std.net.tcpConnectToHost(allocator, "example.com", 80); +defer stream.close(); + +var read_buf: [4096]u8 = undefined; +var reader = stream.reader(&read_buf); + +var write_buf: [4096]u8 = undefined; +var writer = stream.writer(&write_buf); + +try writer.interface.writeAll("GET / HTTP/1.0\r\n\r\n"); +try writer.interface.flush(); + +// Read response +while (reader.interface.takeDelimiterExclusive('\n')) |line| { + // Process response line +} else |err| if (err != error.EndOfStream) return err; +``` + +--- + +## What's coming in Zig 0.16.x + +The std.io changes in 0.15.x are **foundational for the return of async/await** +expected in 0.16.x or later releases. The roadmap includes: + +**Complete std.Io interface:** All blocking operations (file system, networking, +timers, synchronization) will require an `Io` instance, similar to how memory +allocation requires an `Allocator` instance. + +**Event loops as first-class citizens:** The new I/O abstraction enables +different concurrency models (async/await, thread pools, blocking I/O) to work +with the same library code. + +**Better testing and reliability:** I/O as an interface parameter enables +mocking, instrumentation, and resource leak detection. + +**Quote from Andrew Kelley:** "All code that performs I/O will need access to an +Io instance, similar to how all code that allocates memory needs access to an +Allocator instance." + +The 0.15.x changes, while painful, establish the architecture necessary for Zig +to support modern async I/O patterns without language-level async keywords. This +is a deliberate choice to keep async in the standard library rather than making +it a language feature. + +--- + +## Documentation resources for Tero policy-zig + +**Official sources:** + +- Zig 0.15.1 Release Notes: + https://ziglang.org/download/0.15.1/release-notes.html +- Standard Library Documentation: https://ziglang.org/documentation/0.15.2/std/ +- Andrew Kelley's "Don't Forget To Flush" talk (Systems Distributed 2025) + +**Community resources:** + +- "migrating to zig 0.15: the roadblocks nobody warned you about" (sngeth.com) +- "Zig 0.15.1 I/O Overhaul" (dev.to/bkataru) +- "I'm too dumb for Zig's new IO interface" (openmymind.net) - honest account of + migration difficulties +- "Inside Zig's New Writer" (joegm.github.io) - technical deep dive +- Ziggit.dev forum discussions on migration challenges + +**GitHub:** + +- Writergate PR #24329 with full technical rationale +- Active issue tracker for bugs and policy-zig cases + +--- + +## Migration checklist for Tero policy-zig + +For updating your claude.md documentation to prevent outdated patterns: + +- [ ] Document that all readers/writers require explicit buffer allocation +- [ ] Emphasize the mandatory flush requirement (most common bug) +- [ ] Warn about @fieldParentPtr footgun (never copy interfaces) +- [ ] Update all code examples from 0.14.x patterns to 0.15.x patterns +- [ ] Replace `std.io.GenericReader/Writer` with `std.Io.Reader/Writer` +- [ ] Replace `readUntilDelimiterOrEof()` with `takeDelimiterExclusive()` +- [ ] Document that std.io.SeekableStream is removed +- [ ] Update HTTP client/server examples with new multi-buffer patterns +- [ ] Note TLS client requires four buffers minimum +- [ ] Document format string changes ({} → {f} or {any}) +- [ ] Add examples showing proper error handling for StreamTooLong +- [ ] Include buffer sizing guidelines (4096 minimum, TLS requires + max_ciphertext_record_len) +- [ ] Show Writer.Allocating pattern for unbounded data +- [ ] Emphasize that compression functionality was temporarily removed + +This comprehensive reference should enable AI models to provide accurate, +current guidance for Zig 0.15.x I/O operations in the Tero policy-zig project. diff --git a/design/extensions.md b/design/extensions.md new file mode 100644 index 0000000..f2414c7 --- /dev/null +++ b/design/extensions.md @@ -0,0 +1,479 @@ +# Extensions package + `com.usetero/s3-dump` + +Design for implementing policy-spec v1.6.0 extensions in policy-zig, and the +first concrete extension: dumping telemetry to S3-compatible storage. + +Status: proposal (no code yet). + +## What the spec requires (v1.6.0) + +From `spec.md` §Extensions and `extension.proto` / `tero_extensions.proto`: + +- A policy MAY carry `extensions: [{type, version, config, mode}]`. `type` is + reverse-FQDN (`com.usetero/s3-dump`), `version` is semver, `config` is + **opaque bytes** the engine never interprets, `mode` selects a traffic slice. +- Pipeline order is `Match → Keep → Extension dispatch → Transform`. Extensions + receive **pre-transform** records and MUST NOT change the keep or transform + outcome. The keep verdict used for classification is the record's **final + pipeline outcome** across all policies (that's what makes `mode: dropped` on + a `keep: .01%` policy receive the ~99.99% waste). +- Modes, relative to the extension's policy: `kept`, `dropped`, `unmatched` + (disjoint partition), `matched` = kept+dropped (default), `all` = everything. +- Handshake: the client advertises `ClientMetadata.supported_extensions` + (`ExtensionCapability{type, min_version, repeated bytes config}`); the + provider MAY broadcast `SyncResponse.extension_configs` + (`ExtensionConfig{type, repeated bytes config}`). For s3-dump the capability + descriptors are serialized `ExtensionTargetRef`s and the broadcast entries + are serialized `ExtensionTarget`s. +- Rules: fail-open (an unsupported/unsatisfiable extension is skipped; core + match/keep/transform still applies; SHOULD reject the policy only when the + extension is *required* for its behavior — never true for a dump), execute + off the hot path with non-blocking operations, and document supported + `type`/`version` pairs (conformance item 9). +- `ExtensionTarget{kind, name, config}` is a pre-configured named destination; + policies reference it by `ExtensionTargetRef{kind, name}` and never carry + credentials or connection details inline. + +## Goals / non-goals + +**v1 goals** + +- Generic extension mechanism: handler tagged union, engine dispatch hook, + compile-stage validation, capability advertisement, config broadcast + handling. +- One extension: `com.usetero/s3-dump` v1.0.0, all three signals, ndjson + objects, S3-compatible endpoints (AWS, MinIO, R2) via the + [z3](https://codeberg.org/fellowtraveler/z3) client. + +**Non-goals (v1)** + +- Columnar object formats (OTel Arrow, Vortex, Parquet) — see + [Data format](#data-format-edge-native-rows-not-columnar); gzip; multipart + upload (z3 supports it if batches ever need it); upload retry queues. +- Post-transform delivery (spec leaves it type-defined; we don't need it). +- Any second extension type. The mechanism is generic but we build exactly one. + +## Prerequisite: proto update + +We deliberately skipped the extension protos when updating to v1.6.0. First +step is finishing that: + +1. Vendor `extension.proto`, `tero_extensions.proto`, and the v1.6.0 + `policy.proto` (adds `Policy.extensions = 20`, + `ClientMetadata.supported_extensions = 4`, + `SyncResponse.extension_configs = 7`) into `proto/tero/policy/v1/`. +2. `zig build -Dgen-proto=true gen-proto`. + +No engine behavior changes from this alone — unknown fields were already +ignored on the wire. + +## Package layout + +A new module `extensions`, mirroring how `observability` is wired: + +``` +src/extensions/ + root.zig // ExtensionHandler tagged union, dispatch fn, re-exports + s3_dump.zig // com.usetero/s3-dump handler +``` + +One new dependency in `build.zig.zon`: +[z3](https://codeberg.org/fellowtraveler/z3) — pure-Zig async S3 client (MIT, +min Zig 0.16, zero deps beyond std). It covers SigV4 signing, path- and +virtual-host-style requests, S3-compatible endpoints (tested against R2), and +multipart uploads if batches ever outgrow single PUTs. This replaces the +hand-rolled `sigv4.zig` + `std.http` plumbing from an earlier draft. Only the +`extensions` module imports it — `policy_zig` stays dependency-free. + +`build.zig`: + +```zig +const ext_mod = b.addModule("extensions", .{ + .root_source_file = b.path("src/extensions/root.zig"), + .imports = &.{ proto, policy_zig, observability }, +}); +``` + +Rationale for a separate module rather than folding into `policy_zig`: the +engine stays free of I/O and of any knowledge of concrete extension types +(spec: config is opaque to the engine), and consumers that don't use +extensions link nothing new. + +## Core mechanism + +### 1. Engine dispatch hook (`policy_zig` change — the only one) + +Dispatch must happen *inside* `evaluate`: after the final keep decision is +known but **before transforms run** (spec ordering). Post-`evaluate` dispatch +by the consumer would see redacted records, so that's out. + +Add an optional sink to the per-call evaluate options, alongside `io`: + +```zig +/// One function pointer + ctx — the same shape as the accessor primitives, +/// NOT a vtable (Claude.md: encoding over polymorphism; the fn pointer here +/// exists only to break the module cycle — policy_zig cannot import the +/// extensions module that implements it). All real dispatch happens behind +/// it via the ExtensionHandler tagged union. +pub const ExtensionSink = struct { + ctx: *anyopaque, + /// Called once per (record, policy-with-extensions) pair whose mode + /// selects this record, between keep resolution and transforms. + deliver: *const fn ( + ctx: *anyopaque, + io: ?std.Io, + telemetry: DeliveredRecord, // signal tag + consumer ctx + encode fn + binding: *const ExtensionBinding, // policy index + resolved handler tag + config + mode + slice: SliceTag, // kept | dropped | unmatched + ) void, +}; +``` + +- `evaluate(..., .{ .io = io, .extension_sink = null })` — a null sink is a + single branch; zero cost for existing consumers. +- The engine classifies per binding using state it already has: + `matched(policy)` (from `MatchState`) and the final decision. `kept` = + matched ∧ keep, `dropped` = matched ∧ drop, `unmatched` = ¬matched; + `matched`/`all` are unions resolved at compile time into "which slices fire". +- `UNMATCHED`/`ALL` bindings must be visited even when the record matched no + policy, so the binding list is iterated unconditionally — but only when the + snapshot has any bindings (one `bindings.len == 0` check keeps the common + case free). +- Delivery passes the consumer `ctx` + accessor-adjacent `encode` fn (below), + not bytes: the engine has no serialized form of the record. + +### 2. Compile-stage bindings and validation + +Per the "validate during compilation" convention, `IndexBuilder` grows an +extensions pass: + +- Collect `ExtensionBinding{policy_index, handler: ExtensionTag, mode, config}` + for every enabled policy into the snapshot (plain slice, iterated at + dispatch). The `type` string is resolved to the `ExtensionTag` enum **once, + here** — dispatch never compares strings (Claude.md: encode, don't point). +- Validation, fail-open per spec rule 5: an extension whose `type` has no + registered handler, whose `version` exceeds the handler's supported range, + or whose `config` the handler rejects (e.g. unknown target ref) is **skipped + and reported** via the existing `recordError` path → `PolicySyncStatus.errors`. + The policy itself stays valid — a dump extension is never "required for the + policy's behavior". +- Handlers get a `validateConfig(config: []const u8) error{...}!void` chance at + compile time so bad configs surface at sync, not at dispatch. + +### 3. Handlers: tagged union, not a vtable + +Extension types are a closed set the implementation must declare anyway (spec +rule 4), so runtime polymorphism buys nothing. Per Claude.md §"Encoding +Instead of Polymorphism", handlers are a tagged union with switch dispatch: + +```zig +pub const ExtensionTag = enum { s3_dump }; + +pub const ExtensionHandler = union(ExtensionTag) { + s3_dump: S3Dump, + + pub fn typeId(tag: ExtensionTag) []const u8 { + return switch (tag) { .s3_dump => "com.usetero/s3-dump" }; + } + + pub fn validateConfig(self: *const ExtensionHandler, config: []const u8) !void { + switch (self.*) { .s3_dump => |*h| try h.validateConfig(config) } + } + + /// Hot-path entry. MUST be non-blocking: append to a batch and return. + pub fn deliver(self: *ExtensionHandler, rec: DeliveredRecord, config: []const u8, slice: SliceTag) void { + switch (self.*) { .s3_dump => |*h| h.deliver(rec, config, slice) } + } + + /// SyncResponse.extension_configs entries for this type. + pub fn configure(self: *ExtensionHandler, entries: []const []const u8) void { ... } + + /// Capability descriptors for ClientMetadata.supported_extensions. + pub fn capabilities(self: *const ExtensionHandler, allocator: Allocator) ![]const []const u8 { ... } +}; +``` + +There is no registry object: the consumer constructs an +`Extensions = struct { handlers: std.EnumArray(ExtensionTag, ?ExtensionHandler) }` +with the handlers it enables, and the dispatch path is an array index by the +binding's pre-resolved tag — no string lookup, no pointer chase. Adding a +second extension type = one new enum tag + one new union arm; the compiler +enforces exhaustiveness everywhere. + +The only fn pointers in the design are the engine-side `ExtensionSink` +(one function, to break the policy_zig ↔ extensions module cycle — same +pattern as accessors) and the consumer's `encode` callback (unavoidable: the +record type is the consumer's). + +### 4. Record encoding — consumer contract + +The engine reads records through accessors and never owns bytes, so a dump +extension cannot serialize a record by itself. `DeliveredRecord` therefore +carries a consumer-wired encoder: + +```zig +pub const DeliveredRecord = struct { + signal: TelemetryType, + /// Borrowed. Valid ONLY for the duration of the deliver() call. + ctx: *const anyopaque, + /// Render the record (pre-transform) to bytes, e.g. OTLP JSON. + encode: *const fn (ctx: *const anyopaque, writer: *std.Io.Writer) anyerror!void, +}; +``` + +Consumers that already hold OTLP protobuf can emit it directly; others emit +whatever textual form they want in the object. The library does not prescribe +the record schema inside the dump — the extension prescribes the *container* +(ndjson: one encoded record per line). + +### Record lifetime and hot-path cost — copy at delivery, never hold + +This is the load-bearing invariant of the whole design: + +**Handlers encode synchronously inside `deliver()` and own only the encoded +bytes.** The batch buffer receives a self-contained ndjson line; the moment +`deliver()` returns, no extension holds any reference to the consumer's +record. Record memory has exactly the lifetime it has today — free/reuse it +when `evaluate` returns. Nothing is pinned until flush. + +Deferring the copy isn't an optimization we declined — it's impossible to do +correctly: + +1. **The spec requires pre-transform delivery**, and the engine applies + transforms (redact/remove/rename) *in place, immediately after dispatch, + in the same `evaluate` call*. A handler that stashed a reference and read + it at flush time would observe redacted/renamed data. +2. Holding references until flush (up to `max_batch_age` = 30 s) would pin + entire OTLP batches in memory on the edge — the exact memory-pressure + failure this design exists to avoid. + +This is also how we satisfy the spec's "dispatches copies" wording: the copy +is the encoded line; borrowing `ctx` for the duration of one synchronous call +is just how the copy gets made without a second intermediate allocation. + +What the hot path actually pays, worst to best case: + +| situation | cost per record | +| -------------------------------------- | ----------------------------------------- | +| no sink / no bindings in snapshot | one null/len check | +| bindings exist, no mode selects record | slice classification (a few compares) | +| record selected | one `encode` + append into batch buffer | +| record selected, sealed backlog full | capacity check only — **encode skipped**, drop counted | + +The selected-record cost is a bounded, allocation-free serialize (µs-scale +for a typical log record) on the processing thread — a memcpy-class +operation, never I/O, never a lock held across I/O. The capacity check runs +*before* encoding, so a destination outage degrades to a counter increment, +not wasted serialization. And for the flagship `mode: dropped` use, the +records paying the encode are ones the pipeline was discarding anyway. + +If that encode ever shows up in a profile, the knobs are consumer-side: a +cheaper `encode` (e.g. memcpy of pre-serialized bytes the consumer already +has), or narrower modes/matchers so fewer records are selected. A zero-copy +"raw bytes" fast path on `DeliveredRecord` is possible later without changing +this invariant. + +### 5. Sync plumbing (`provider_http`) + +- **Advertise**: when building `ClientMetadata`, iterate the enabled handlers + in the `Extensions` enum array, ask each for `capabilities()`, and fill + `supported_extensions` with + `{type, min_version, config}`. For s3-dump each descriptor is a serialized + `ExtensionTargetRef` for every target the client can currently reach. +- **Receive**: on `SyncResponse`, route each `extension_configs` entry to + `handler.configure(entries)`. For s3-dump the entries are serialized + `ExtensionTarget`s, merged with locally configured targets (broadcast wins + on `(kind, name)` collision; spec says clients merge). + +## `com.usetero/s3-dump` v1.0.0 + +### Policy-facing config + +Exactly what the spec's example shows — a target reference, nothing else: + +```yaml +extensions: + - type: com.usetero/s3-dump + version: 1.0.0 + mode: dropped + config: + target: { kind: s3, name: eu-bucket } +``` + +`config` bytes = serialized `ExtensionTargetRef`. `validateConfig` decodes it +and checks the target exists (locally configured or broadcast); unknown target +→ skip + sync error, per fail-open. + +### Target config (`ExtensionTarget.config` for kind `s3`) + +The spec leaves this kind-defined. Define it as JSON (matches the rest of the +repo's config surface and avoids another proto): + +```json +{ + "endpoint": "https://s3.eu-west-1.amazonaws.com", // any S3-compatible URL + "region": "eu-west-1", + "bucket": "tero-waste", + "prefix": "dumps/", + "force_path_style": false +} +``` + +**Credentials are never in the target or the policy.** The consumer supplies a +credentials callback at handler construction +(`fn () ?Credentials{access_key, secret_key, session_token}`); the default +reads `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`/`AWS_SESSION_TOKEN`. No +credentials → deliveries for that target are counted-and-dropped (fail-open). + +### Batching — the core of the handler + +Records are never uploaded individually. One S3 PUT per record would melt both +the pipeline and the S3 bill; the unit of upload is a **batch**, and the hot +path's only job is appending to one (spec rule 6: off the hot path, +non-blocking). + +**Batch key**: `(target, signal, policy_id)`. Signal and policy separate +naturally in storage (waste analysis is per-policy), and the object key below +encodes both. The number of batches is bounded by +`configured targets × 3 signals × policies-with-s3-dump` — small, known at +snapshot compile time. + +**Hot path** (`deliver()`): find the open batch — bindings are compiled per +snapshot, so each binding carries its batch index; array lookup, no hashing +per record — check capacity **first** (full backlog → count the drop and +return, no encode), then run the consumer `encode` into the batch's append +buffer, write the `\n`, bump the record count. Under a per-batch mutex; no +I/O, no allocation beyond the batch buffer. See +[Record lifetime](#record-lifetime-and-hot-path-cost--copy-at-delivery-never-hold) +for why the encode happens here and not later. + +**Batch limits** (defaults, overridable at handler construction): + +| limit | default | on hit | +| ------------------- | ------- | ---------------------------------------- | +| `max_batch_bytes` | 4 MiB | batch is sealed, ready for flush | +| `max_batch_records` | 10 000 | batch is sealed, ready for flush | +| `max_batch_age` | 30 s | checked in `flush()`, seals stale batches | +| `max_sealed_bytes` | 32 MiB | total sealed backlog — new records **drop** | + +A record arriving when the sealed backlog is at `max_sealed_bytes` is dropped +and counted (`records_dropped` on the event bus). Backpressure never +propagates to telemetry processing — a dump destination outage costs waste +records, never pipeline latency. + +**Flush** — the consumer drives I/O, per this repo's io-threading rule +(`std.Io` passed per call, never stored): `flush(io: std.Io) FlushResult` +seals any over-age open batches (age from `io.now`), uploads each sealed +batch as one object, and releases its buffer. The consumer calls it from a +background task/timer; the handler never spawns threads or sleeps. +Double-buffering (open batch swaps onto the sealed list) keeps `deliver()` +appending while `flush` uploads. + +### Data format: edge-native rows, not columnar + +Considered: dump in whatever form the edge already has (consumer-encoded +rows), vs. re-encoding into a columnar format (OTel Arrow / OTAP, Vortex, +Parquet). Decision: **edge-native rows at the edge; columnar belongs in a +downstream compaction job, if ever.** + +- **No Zig implementation exists** for Arrow, OTAP, or Vortex. Adopting one + means FFI to a Rust/C++ library or writing a columnar encoder — either + dwarfs the rest of this design for a waste stream that is written often and + read rarely. +- **Columnar encoding is a batch-side workload.** Good compression needs large + row groups, dictionary state, and schema unification across records — CPU + and memory spent on the edge, on the hot-adjacent path, for exactly the + records the policy decided were not worth keeping. The edge's job is to get + bytes out cheaply; fail-open dumping must stay near-free. +- **Rows rehydrate; columns analyze.** The primary consumer of a waste dump is + "replay these records back into the pipeline" (incident retro, sampling + regret). Consumer-encoded OTLP rows replay directly. If cheap analytics over + the dumps becomes real, run a server-side compaction job + (S3 → Parquet/Vortex) where cores are cheap and batches are huge — the + standard lake pattern, and it needs zero changes at the edge. +- The consumer `encode` callback already exists because the engine reads + records through accessors and owns no serialized form. A columnar format + would force a much fatter contract (full typed schema per record) on every + consumer. + +So the object body is rows in whatever form the edge is working with — +recommended `encode` output is OTLP JSON, but the container doesn't care. + +### Object format and key + +One object per batch: + +- Body: ndjson — each line is one record as produced by the consumer `encode`. +- Key: `{prefix}{signal}/{yyyy}/{mm}/{dd}/{hh}/{policy_id}-{unix_nanos}-{seq}.ndjson` + Time from `io.now` at flush; `seq` is a per-process counter (no uuid dep). +- `Content-Type: application/x-ndjson`. + +### Upload + +- One `z3` `putObject(bucket, key, body, options)` per sealed batch, over the + flush-supplied `io`. z3 handles SigV4, path- vs virtual-host-style (our + `force_path_style` maps directly), and custom endpoints. The sealed batch is + a contiguous buffer, so content length and payload hash are trivially + available (z3's no-chunked-signing limitation doesn't bite us). +- z3 does not do credential discovery — fine, because our design already + sources credentials from the consumer callback / env (see above) and passes + them at client init. One z3 client per target, created lazily at first + flush. +- Failure handling v1: non-2xx or transport error → count it, drop the batch, + log via event bus. No retry queue (the data is by definition the waste + stream; better to lose a batch than to grow unbounded state). + + +### Capability advertisement + +`capabilities()` returns one serialized `ExtensionTargetRef{kind: "s3", name}` +per configured target, so the provider only sends policies whose extensions +this client can satisfy. + +## Conformance note + +README gains a "Supported extensions" table — required by conformance item 9: + +| type | versions | notes | +| --------------------- | -------- | ---------------------------------- | +| `com.usetero/s3-dump` | 1.0.x | ndjson objects, z3 client, no retries | + +## Testing + +- **Classification**: engine-level table tests — one policy per mode × record + {matched-kept, matched-dropped, unmatched}, assert exactly the spec's slice + reaches a recording fake sink; assert transforms observe untouched records + after a sink that mutates nothing; assert final decision is identical with + and without a sink (MUST NOT change outcome). +- **Cross-policy drop**: policy A `keep: none`, policy B `keep: all` with + `mode: dropped` extension — B's extension receives the record (final-outcome + semantics). +- **Validation**: unknown type / bad version / unknown target → policy still + active, extension skipped, error string in `PolicySyncStatus.errors`. +- **Upload**: flush against a local HTTP stub asserting method, path, auth + header presence, and ndjson body (signing correctness itself is z3's tested + responsibility); sealed-backlog drop counting; missing-credentials no-op. +- **End-to-end (manual)**: MinIO via the `verify` flow. + +## Rollout + +1. Vendor v1.6.0 extension protos + regen (no behavior change). +2. `extensions` module: `ExtensionHandler` tagged union, sink, binding + compilation + validation in `IndexBuilder`, engine dispatch hook. + Fake-sink tests. +3. Vendor the z3 dependency; smoke-test `putObject` against MinIO. +4. `s3_dump.zig`: config/target parsing, batching, flush/upload, stub tests. +5. `provider_http`: capability advertisement + `extension_configs` routing. +6. README conformance table + consumer wiring example. + +Each step lands independently; 1–2 are useful on their own (dispatch mechanism +with no handlers registered is inert). + +## Open questions + +1. **`mode: all`/`unmatched` cost**: these force sink calls (and encodes) for + every record of the signal. Fine mechanically, but do we want a + per-snapshot cap or a warning event when such a policy is loaded? +2. **S3 target JSON vs proto**: JSON chosen above for authoring ergonomics; + if the tero backend already defines an S3 target proto, use that instead — + the bytes are opaque to everything but the handler either way. From acc775697c2f9019033fb163b3973fb4724e4cba Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Tue, 7 Jul 2026 18:07:31 -0400 Subject: [PATCH 4/8] feat: introduce s3 extension --- README.md | 15 + build.zig | 28 + build.zig.zon | 4 + design/extensions.md | 18 +- proto/tero/policy/v1/policy.proto | 17 + src/policy/matcher_index.zig | 136 +++-- src/policy/policy_engine.zig | 886 ++++++++++++++++++++++++++++++ src/policy/provider_http.zig | 48 ++ src/policy/registry.zig | 16 + src/policy/root.zig | 8 + src/policy/types.zig | 89 +++ src/proto/tero/policy/v1.pb.zig | 448 +++++++++++++++ 12 files changed, 1681 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 9945cb1..6447dbb 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ src/ trace_sampler.zig # Trace sampling rate_limiter.zig # Rate limiting hyperscan.zig # Vectorscan/Hyperscan C bindings + extensions/ # Policy extensions (spec v1.6.0): s3-dump handler observability/ # Event bus, spans, formatters proto/ # Generated protobuf definitions ``` @@ -160,6 +161,20 @@ src/ This gives O(k \* n) performance where k = unique matcher keys and n = input text length, independent of policy count. +## Supported Extensions + +Per policy-spec conformance item 9, this implementation supports the following +extension `type`/`version` pairs (see `design/extensions.md` and the +`extensions` module): + +| type | versions | notes | +| --------------------- | -------- | -------------------------------------------------- | +| `com.usetero/s3-dump` | 1.0.x | batched ndjson objects via z3 S3 client, no retry queue | + +Extensions declared by a policy but not supported (or with an unresolvable +target) are skipped fail-open and reported via `PolicySyncStatus.errors`; the +policy's core match/keep/transform still applies. + ## Task Commands This project uses [Task](https://taskfile.dev/) for common operations: diff --git a/build.zig b/build.zig index f030754..2e959fd 100644 --- a/build.zig +++ b/build.zig @@ -44,6 +44,23 @@ pub fn build(b: *std.Build) void { mod.link_libc = true; mod.linkSystemLibrary("hs", .{}); + // Extensions module (spec v1.6.0): concrete extension handlers (s3-dump). + // Separate module so policy_zig stays free of I/O and of the z3 + // dependency; only consumers that enable extensions link it. + const z3_dep = b.dependency("z3", .{ + .target = target, + .optimize = optimize, + }); + const ext_mod = b.addModule("extensions", .{ + .root_source_file = b.path("src/extensions/root.zig"), + .target = target, + .imports = &.{ + .{ .name = "proto", .module = proto_mod }, + .{ .name = "policy_zig", .module = mod }, + .{ .name = "s3", .module = z3_dep.module("s3") }, + }, + }); + // Tests const mod_tests = b.addTest(.{ .root_module = mod, @@ -59,9 +76,18 @@ pub fn build(b: *std.Build) void { }); const run_o11y_tests = b.addRunArtifact(o11y_tests); + // Extensions module tests (need libc/hyperscan transitively via policy_zig) + const ext_tests = b.addTest(.{ + .root_module = ext_mod, + }); + ext_tests.root_module.link_libc = true; + ext_tests.root_module.linkSystemLibrary("hs", .{}); + const run_ext_tests = b.addRunArtifact(ext_tests); + const test_step = b.step("test", "Run tests"); test_step.dependOn(&run_mod_tests.step); test_step.dependOn(&run_o11y_tests.step); + test_step.dependOn(&run_ext_tests.step); // Benchmarks const zbench_dep = b.dependency("zbench", .{ .target = target, .optimize = .ReleaseFast }); @@ -96,6 +122,8 @@ pub fn build(b: *std.Build) void { .destination_directory = b.path("src/proto"), .source_files = &.{ b.path("proto/tero/policy/v1/policy.proto"), + b.path("proto/tero/policy/v1/extension.proto"), + b.path("proto/tero/policy/v1/tero_extensions.proto"), b.path("proto/tero/policy/v1/log.proto"), b.path("proto/tero/policy/v1/metric.proto"), b.path("proto/tero/policy/v1/trace.proto"), diff --git a/build.zig.zon b/build.zig.zon index fd71d56..44478c2 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -50,6 +50,10 @@ .url = "git+https://github.com/hendriknielaender/zBench.git#b2b89c475e3ef1bb2bd71255c80478a82d3e0ca8", .hash = "zbench-0.13.0-YTdc7xVBAQCCMC-IdLLuotBeiNNNm8k9Pi2V4VYqrwfI", }, + .z3 = .{ + .url = "git+https://codeberg.org/fellowtraveler/z3#83b1f8d6222188f39c09966b8572bc9168813d04", + .hash = "z3-0.4.1-N25-cH-fAQAbj6ABBY0Bq85Kv-d7sdIvANgLCE9NCaEF", + }, }, // Specifies the set of files and directories that are included in this package. // Only files and directories listed here are included in the `hash` that diff --git a/design/extensions.md b/design/extensions.md index f2414c7..b03c043 100644 --- a/design/extensions.md +++ b/design/extensions.md @@ -3,7 +3,23 @@ Design for implementing policy-spec v1.6.0 extensions in policy-zig, and the first concrete extension: dumping telemetry to S3-compatible storage. -Status: proposal (no code yet). +Status: **implemented** (see `src/extensions/`, engine dispatch in +`src/policy/policy_engine.zig`, bindings in `src/policy/matcher_index.zig`). +Deviations from the text below, made during implementation: + +- The resolver seam passes `io` first (`resolve(io, ctx, signal, policy_id, + ext)`) per ziglint's io-ordering rule, and resolution returns an opaque + `{handler, slot}` pair — the slot is the s3-dump batch index, assigned at + compile time so dispatch is an array lookup. +- Per-signal `encode` callbacks are registered once on `Extensions.init` + rather than carried in each `DeliveredRecord` (the engine passes only + `(signal, record ctx)` across the module seam). +- Credentials are consumer-provided at `S3Dump.init` (no built-in env + discovery — z3 has none either, and the edge binary owns its env). +- `provider_http` gains `setExtensionSyncHooks` (a two-fn seam like + StatsCollector) for capability advertisement + config broadcast routing. +- All batching knobs live in `S3Dump.Options`, a plain-data struct with + defaults, deserializable from a config file. ## What the spec requires (v1.6.0) diff --git a/proto/tero/policy/v1/policy.proto b/proto/tero/policy/v1/policy.proto index 02d8d8a..b1ddbca 100644 --- a/proto/tero/policy/v1/policy.proto +++ b/proto/tero/policy/v1/policy.proto @@ -4,6 +4,7 @@ package tero.policy.v1; import "google/api/annotations.proto"; import "opentelemetry/proto/common/v1/common.proto"; +import "tero/policy/v1/extension.proto"; import "tero/policy/v1/log.proto"; import "tero/policy/v1/metric.proto"; import "tero/policy/v1/trace.proto"; @@ -49,6 +50,10 @@ message Policy { MetricTarget metric = 11; TraceTarget trace = 12; } + + // Implementation-specific extensions attached to this policy. Each extension + // references a pre-configured ExtensionTarget by (kind, name). + repeated Extension extensions = 20; } // ============================================================================= @@ -83,6 +88,11 @@ message ClientMetadata { // * service.namespace // * service.version repeated opentelemetry.proto.common.v1.KeyValue resource_attributes = 3; + + // Extension types this client supports, and the target kinds/names it can + // route each type to. Providers SHOULD only send policies whose extensions + // reference targets advertised here. + repeated ExtensionCapability supported_extensions = 4; } // TransformStageStatus reports hits and misses for a single transform stage. @@ -160,6 +170,13 @@ message SyncResponse { // Error message if sync failed string error_message = 6; + + // Opaque, type-defined extension configuration broadcast to the client, keyed + // by extension type. The policy engine does not interpret it; each extension + // handler parses its own entries. For tero's s3-dump this carries the + // available ExtensionTargets. A client merges these with any locally + // configured extension state. + repeated ExtensionConfig extension_configs = 7; } // ============================================================================= diff --git a/src/policy/matcher_index.zig b/src/policy/matcher_index.zig index 1c6c897..7f73912 100644 --- a/src/policy/matcher_index.zig +++ b/src/policy/matcher_index.zig @@ -878,6 +878,9 @@ fn IndexBuilder(comptime T: TelemetryType) type { /// policies are compiled without validation (used by direct build() /// callers such as unit tests). errors: ?*CompilationErrors, + /// Optional resolver for policy extensions (v1.6.0). When null, any + /// declared extension is skipped and reported (fail-open). + extension_resolver: ?policy_types.ExtensionResolver, patterns_by_key: std.HashMap( MatcherKeyT, PatternsPerKey, @@ -886,6 +889,7 @@ fn IndexBuilder(comptime T: TelemetryType) type { ), policy_info_list: std.ArrayList(PolicyInfo), typed_checks_list: std.ArrayList(TypedCheckT), + extension_bindings_list: std.ArrayList(policy_types.ExtensionBinding), path_storage: std.ArrayList([]const []const u8), policy_id_storage: std.ArrayList([]const u8), policy_index: PolicyIndex, @@ -903,12 +907,14 @@ fn IndexBuilder(comptime T: TelemetryType) type { temp_allocator: std.mem.Allocator, bus: *EventBus, errors: ?*CompilationErrors, + extension_resolver: ?policy_types.ExtensionResolver, ) Self { return .{ .allocator = allocator, .temp_allocator = temp_allocator, .bus = bus, .errors = errors, + .extension_resolver = extension_resolver, .patterns_by_key = std.HashMap( MatcherKeyT, PatternsPerKey, @@ -917,6 +923,7 @@ fn IndexBuilder(comptime T: TelemetryType) type { ).init(temp_allocator), .policy_info_list = .empty, .typed_checks_list = .empty, + .extension_bindings_list = .empty, .path_storage = .empty, .policy_id_storage = .empty, .policy_index = 0, @@ -997,6 +1004,47 @@ fn IndexBuilder(comptime T: TelemetryType) type { const keep_value = parseKeepValue(target); try self.storePolicyInfo(policy, target, keep_value, global_index); + try self.compileExtensions(policy); + } + + /// Compile the policy's extension declarations into bindings (v1.6.0). + /// Fail-open per spec rule 5: an extension that cannot be resolved (no + /// resolver wired, unsupported type/version, invalid config) is skipped + /// and reported WITHOUT invalidating the policy — its core + /// match/keep/transform still applies. Must run after storePolicyInfo + /// so the binding can reference the stored policy info. + fn compileExtensions(self: *Self, policy: *const Policy) !void { + if (policy.extensions.items.len == 0) return; + const info = &self.policy_info_list.items[self.policy_info_list.items.len - 1]; + for (policy.extensions.items) |*ext| { + const resolution: ?policy_types.ExtensionResolution = blk: { + const r = self.extension_resolver orelse break :blk null; + break :blk r.resolve(self.bus.io, r.ctx, T, policy.id, ext); + }; + const resolved = resolution orelse { + if (self.errors) |sink| { + try sink.addFmt( + @tagName(T) ++ ": extension \"{s}\" unsupported or invalid; skipped", + policy.id, + .{ext.type}, + ); + } + continue; + }; + // Config bytes go into the index's owned-string storage so the + // binding stays valid for the snapshot's lifetime regardless of + // registry-side policy memory churn (same treatment as ids). + const config_copy = try self.allocator.dupe(u8, ext.config); + try self.policy_id_storage.append(self.allocator, config_copy); + try self.extension_bindings_list.append(self.temp_allocator, .{ + .policy_index = info.index, + .policy_id = info.id, + .handler = resolved.handler, + .slot = resolved.slot, + .slices = policy_types.extensionSliceSet(ext.mode), + .config = config_copy, + }); + } } /// Record a compilation error for the policy being compiled: marks it @@ -1573,6 +1621,10 @@ fn IndexBuilder(comptime T: TelemetryType) type { const typed_checks = try self.allocator.dupe(TypedCheckT, self.typed_checks_list.items); // Free the builder's backing storage; the items are now in `typed_checks`. self.typed_checks_list.deinit(self.allocator); + const extension_bindings = try self.allocator.dupe( + policy_types.ExtensionBinding, + self.extension_bindings_list.items, + ); return IndexT{ .allocator = self.allocator, @@ -1581,6 +1633,7 @@ fn IndexBuilder(comptime T: TelemetryType) type { .policies_with_negation = policies_with_negation, .matcher_keys = matcher_keys, .typed_checks = typed_checks, + .extension_bindings = extension_bindings, .path_storage = self.path_storage, .policy_id_storage = self.policy_id_storage, .bus = self.bus, @@ -1605,6 +1658,7 @@ pub const LogMatcherIndex = struct { policies_with_negation: []PolicyIndex, matcher_keys: []LogMatcherKey, typed_checks: []TypedCheckType(.log), + extension_bindings: []policy_types.ExtensionBinding, path_storage: std.ArrayList([]const []const u8), policy_id_storage: std.ArrayList([]const u8), bus: *EventBus, @@ -1620,6 +1674,7 @@ pub const LogMatcherIndex = struct { bus: *EventBus, policies_slice: []const Policy, errors: ?*CompilationErrors, + extension_resolver: ?policy_types.ExtensionResolver, ) !LogMatcherIndex { const started_event: MatcherIndexBuildStarted = .{ .policy_count = policies_slice.len, @@ -1634,7 +1689,7 @@ pub const LogMatcherIndex = struct { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - var builder = IndexBuilder(.log).init(allocator, arena.allocator(), bus, errors); + var builder = IndexBuilder(.log).init(allocator, arena.allocator(), bus, errors, extension_resolver); for (policies_slice, 0..) |*policy, i| { try builder.processPolicy(policy, @intCast(i)); @@ -1684,6 +1739,10 @@ pub const LogMatcherIndex = struct { return self.typed_checks; } + pub fn getExtensionBindings(self: *const LogMatcherIndex) []const policy_types.ExtensionBinding { + return self.extension_bindings; + } + pub fn isEmpty(self: *const LogMatcherIndex) bool { return self.matcher_keys.len == 0 and self.typed_checks.len == 0; } @@ -1731,6 +1790,7 @@ pub const LogMatcherIndex = struct { } } self.allocator.free(self.typed_checks); + self.allocator.free(self.extension_bindings); for (self.path_storage.items) |path| { for (path) |segment| { @@ -1763,6 +1823,7 @@ pub const MetricMatcherIndex = struct { policies_with_negation: []PolicyIndex, matcher_keys: []MetricMatcherKey, typed_checks: []TypedCheckType(.metric), + extension_bindings: []policy_types.ExtensionBinding, path_storage: std.ArrayList([]const []const u8), policy_id_storage: std.ArrayList([]const u8), bus: *EventBus, @@ -1774,6 +1835,7 @@ pub const MetricMatcherIndex = struct { bus: *EventBus, policies_slice: []const Policy, errors: ?*CompilationErrors, + extension_resolver: ?policy_types.ExtensionResolver, ) !MetricMatcherIndex { const started_event: MatcherIndexBuildStarted = .{ .policy_count = policies_slice.len, @@ -1788,7 +1850,7 @@ pub const MetricMatcherIndex = struct { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - var builder = IndexBuilder(.metric).init(allocator, arena.allocator(), bus, errors); + var builder = IndexBuilder(.metric).init(allocator, arena.allocator(), bus, errors, extension_resolver); for (policies_slice, 0..) |*policy, i| { try builder.processPolicy(policy, @intCast(i)); @@ -1838,6 +1900,10 @@ pub const MetricMatcherIndex = struct { return self.typed_checks; } + pub fn getExtensionBindings(self: *const MetricMatcherIndex) []const policy_types.ExtensionBinding { + return self.extension_bindings; + } + pub fn isEmpty(self: *const MetricMatcherIndex) bool { return self.matcher_keys.len == 0 and self.typed_checks.len == 0; } @@ -1880,6 +1946,7 @@ pub const MetricMatcherIndex = struct { } } self.allocator.free(self.typed_checks); + self.allocator.free(self.extension_bindings); for (self.path_storage.items) |path| { for (path) |segment| { @@ -1912,6 +1979,7 @@ pub const TraceMatcherIndex = struct { policies_with_negation: []PolicyIndex, matcher_keys: []TraceMatcherKey, typed_checks: []TypedCheckType(.trace), + extension_bindings: []policy_types.ExtensionBinding, path_storage: std.ArrayList([]const []const u8), policy_id_storage: std.ArrayList([]const u8), bus: *EventBus, @@ -1923,6 +1991,7 @@ pub const TraceMatcherIndex = struct { bus: *EventBus, policies_slice: []const Policy, errors: ?*CompilationErrors, + extension_resolver: ?policy_types.ExtensionResolver, ) !TraceMatcherIndex { const started_event: MatcherIndexBuildStarted = .{ .policy_count = policies_slice.len, @@ -1937,7 +2006,7 @@ pub const TraceMatcherIndex = struct { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - var builder = IndexBuilder(.trace).init(allocator, arena.allocator(), bus, errors); + var builder = IndexBuilder(.trace).init(allocator, arena.allocator(), bus, errors, extension_resolver); for (policies_slice, 0..) |*policy, i| { try builder.processPolicy(policy, @intCast(i)); @@ -1987,6 +2056,10 @@ pub const TraceMatcherIndex = struct { return self.typed_checks; } + pub fn getExtensionBindings(self: *const TraceMatcherIndex) []const policy_types.ExtensionBinding { + return self.extension_bindings; + } + pub fn isEmpty(self: *const TraceMatcherIndex) bool { return self.matcher_keys.len == 0 and self.typed_checks.len == 0; } @@ -2029,6 +2102,7 @@ pub const TraceMatcherIndex = struct { } } self.allocator.free(self.typed_checks); + self.allocator.free(self.extension_bindings); for (self.path_storage.items) |path| { for (path) |segment| { @@ -2353,7 +2427,7 @@ test "LogMatcherIndex: build empty" { var noop_bus: NoopEventBus = undefined; noop_bus.init(std.Options.debug_io); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{}, null, null); defer index.deinit(); try testing.expect(index.isEmpty()); @@ -2365,7 +2439,7 @@ test "MetricMatcherIndex: build empty" { var noop_bus: NoopEventBus = undefined; noop_bus.init(std.Options.debug_io); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{}, null, null); defer index.deinit(); try testing.expect(index.isEmpty()); @@ -2391,7 +2465,7 @@ test "LogMatcherIndex: build with single policy" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2428,7 +2502,7 @@ test "LogMatcherIndex: invalid regex policy is skipped (inert) and reported" { var errors = CompilationErrors.init(allocator); defer errors.deinit(); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, &errors); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, &errors, null); defer index.deinit(); // Inert: not compiled into the index at all. @@ -2467,7 +2541,7 @@ test "LogMatcherIndex: empty attribute path is skipped (inert) and reported" { var errors = CompilationErrors.init(allocator); defer errors.deinit(); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, &errors); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, &errors, null); defer index.deinit(); try testing.expectEqual(@as(usize, 0), index.getPolicyCount()); @@ -2514,7 +2588,7 @@ test "LogMatcherIndex: valid and invalid policies in one batch — valid still c var errors = CompilationErrors.init(allocator); defer errors.deinit(); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{ valid, bad_keep }, &errors); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{ valid, bad_keep }, &errors, null); defer index.deinit(); // Valid policy compiled; malformed-keep policy is inert and reported. @@ -2543,7 +2617,7 @@ test "LogMatcherIndex: invalid policy is inert even without an error sink" { }; defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expectEqual(@as(usize, 0), index.getPolicyCount()); @@ -2568,7 +2642,7 @@ test "MetricMatcherIndex: build with single policy" { }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2602,7 +2676,7 @@ test "LogMatcherIndex: build with keyed matchers" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2637,7 +2711,7 @@ test "LogMatcherIndex: negated matcher creates negated database" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2668,7 +2742,7 @@ test "LogMatcherIndex: scan database" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }); @@ -2705,7 +2779,7 @@ test "LogMatcherIndex: exists=true matcher is bucketed into exists_entries" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); // exists matchers no longer compile to Hyperscan patterns; they live in @@ -2746,7 +2820,7 @@ test "LogMatcherIndex: exists=false matcher creates negated pattern" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); // exists=false lives in exists_entries with negate=true; no Hyperscan DB. @@ -2791,7 +2865,7 @@ test "MetricMatcherIndex: metric_type with null match (implicit exists)" { }, }; - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); // The index should have registered one policy with one matcher @@ -2837,7 +2911,7 @@ test "TraceMatcherIndex: span_kind null match with second resource_attribute mat }, }; - var index = try TraceMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try TraceMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2880,7 +2954,7 @@ test "TraceMatcherIndex: span_kind with null match (implicit exists)" { }, }; - var index = try TraceMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try TraceMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2915,7 +2989,7 @@ test "MetricMatcherIndex: exists=true matcher is bucketed into exists_entries" { }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); // exists=true is dispatched via accessor.callExists, not Hyperscan. @@ -2954,7 +3028,7 @@ test "MetricMatcherIndex: metric_type exists is bucketed into exists_entries" { }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -2993,7 +3067,7 @@ test "MetricMatcherIndex: metric_type with regex pattern" { }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -3036,7 +3110,7 @@ test "MetricMatcherIndex: aggregation_temporality field creates Hyperscan databa }); defer policy.deinit(allocator); - var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null); + var index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), &.{policy}, null, null); defer index.deinit(); try testing.expect(!index.isEmpty()); @@ -3096,7 +3170,7 @@ test "Mixed log and metric policies: each index only gets its type" { const policies = &[_]Policy{ log_policy, metric_policy }; // Log index should only have log policy - var log_index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), policies, null); + var log_index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), policies, null, null); defer log_index.deinit(); try testing.expectEqual(@as(usize, 1), log_index.getPolicyCount()); try testing.expectEqual(@as(usize, 1), log_index.getDatabaseCount()); @@ -3108,7 +3182,7 @@ test "Mixed log and metric policies: each index only gets its type" { try testing.expect(metric_in_log == null); // Metric index should only have metric policy - var metric_index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), policies, null); + var metric_index = try MetricMatcherIndex.build(allocator, noop_bus.eventBus(), policies, null, null); defer metric_index.deinit(); try testing.expectEqual(@as(usize, 1), metric_index.getPolicyCount()); try testing.expectEqual(@as(usize, 1), metric_index.getDatabaseCount()); @@ -3180,7 +3254,7 @@ test "Log matcher with starts_with" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3218,7 +3292,7 @@ test "Log matcher with ends_with" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3256,7 +3330,7 @@ test "Log matcher with contains" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3297,7 +3371,7 @@ test "Log matcher with exact" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3336,7 +3410,7 @@ test "Log matcher with case_insensitive" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; @@ -3378,7 +3452,7 @@ test "Log matcher with starts_with case_insensitive" { }); defer policy.deinit(allocator); - var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null); + var index = try LogMatcherIndex.build(allocator, noop_bus.eventBus(), &[_]Policy{policy}, null, null); defer index.deinit(); const db = index.getDatabase(.{ .field = .{ .log_field = .LOG_FIELD_BODY } }).?; diff --git a/src/policy/policy_engine.zig b/src/policy/policy_engine.zig index 6b04477..5a9e03c 100644 --- a/src/policy/policy_engine.zig +++ b/src/policy/policy_engine.zig @@ -232,6 +232,12 @@ pub const EvaluateOptions = struct { /// degrades those paths gracefully (rate limit defaults to keep, regex /// redact no-ops) exactly like a null `scratch`. io: ?std.Io = null, + /// Extension dispatch sink (v1.6.0). When set and the snapshot compiled + /// extension bindings, each record is classified per binding + /// (kept/dropped/unmatched against the final keep outcome) and selected + /// records are handed to the sink after keep resolution, before + /// transforms. Null costs a single branch. + extension_sink: ?policy_types.ExtensionSink = null, }; // ============================================================================= @@ -385,6 +391,30 @@ pub const PolicyEngine = struct { // Record hit/miss stats using lock-free atomics self.recordMatchedPolicyStats(snapshot, &match_state); + // Extension dispatch (v1.6.0): classify this record for every compiled + // (policy, extension) binding against the final keep outcome and hand + // selected records to the sink. Runs after keep resolution and BEFORE + // transforms (and before the drop return below — `mode: dropped` is + // the flagship use), so sinks observe pre-transform records. The + // record is borrowed only for the duration of each deliver call. + if (options.extension_sink) |sink| { + const bindings = index.getExtensionBindings(); + const kept = match_state.decision != .drop; + for (bindings) |*binding| { + const info = index.getPolicyByIndex(binding.policy_index) orelse continue; + // A disabled policy's extensions are inactive with it. + if (!info.enabled) continue; + const matched = scan_state.match_counts[binding.policy_index] == info.required_match_count; + const slice: policy_types.ExtensionSlice = if (matched) + (if (kept) .kept else .dropped) + else + .unmatched; + if (binding.slices.contains(slice)) { + sink.deliver(sink.ctx, options.io, T, ctx, binding, slice); + } + } + } + if (match_state.decision == .drop) { return .dropped; } @@ -5225,3 +5255,859 @@ test "PolicyEngine: mixed signal policies scope stats to own signal type" { const trace_stats = snapshot.getStats(2).?; try testing.expectEqual(@as(i64, 1), trace_stats.hits.load(.monotonic)); } + +// ============================================================================= +// Extension dispatch tests (v1.6.0) +// ============================================================================= + +/// Test resolver: accepts extensions of type "test/record", assigns handler 7 +/// and sequential slots; anything else is unsupported (returns null). +const TestExtensionResolver = struct { + next_slot: u32 = 0, + + fn resolveFn( + io: std.Io, + ctx_ptr: *anyopaque, + signal: TelemetryType, + policy_id: []const u8, + extension: *const proto.policy.Extension, + ) ?policy_types.ExtensionResolution { + _ = io; + _ = signal; + _ = policy_id; + const self: *TestExtensionResolver = @ptrCast(@alignCast(ctx_ptr)); + if (!std.mem.eql(u8, extension.type, "test/record")) return null; + defer self.next_slot += 1; + return .{ .handler = 7, .slot = self.next_slot }; + } + + fn resolver(self: *TestExtensionResolver) policy_types.ExtensionResolver { + return .{ .ctx = self, .resolve = resolveFn }; + } +}; + +/// Test sink: records every delivery. Policy ids are duped because they are +/// only guaranteed valid for the snapshot's lifetime. +const RecordingExtensionSink = struct { + const Delivery = struct { + policy_id: []const u8, + handler: u8, + slot: u32, + slice: policy_types.ExtensionSlice, + }; + + allocator: std.mem.Allocator, + deliveries: std.ArrayList(Delivery) = .empty, + + fn deinit(self: *RecordingExtensionSink) void { + defer self.* = undefined; + for (self.deliveries.items) |d| self.allocator.free(d.policy_id); + self.deliveries.deinit(self.allocator); + } + + fn deliverFn( + ctx_ptr: *anyopaque, + io: ?std.Io, + signal: TelemetryType, + record: *const anyopaque, + binding: *const policy_types.ExtensionBinding, + slice: policy_types.ExtensionSlice, + ) void { + _ = io; + _ = signal; + _ = record; + const self: *RecordingExtensionSink = @ptrCast(@alignCast(ctx_ptr)); + const id = self.allocator.dupe(u8, binding.policy_id) catch return; + self.deliveries.append(self.allocator, .{ + .policy_id = id, + .handler = binding.handler, + .slot = binding.slot, + .slice = slice, + }) catch self.allocator.free(id); + } + + fn sink(self: *RecordingExtensionSink) policy_types.ExtensionSink { + return .{ .ctx = self, .deliver = deliverFn }; + } +}; + +fn makeExtensionLogPolicy( + allocator: std.mem.Allocator, + id: []const u8, + pattern: []const u8, + keep: []const u8, + ext_type: []const u8, + mode: proto.policy.ExtensionMode, +) !Policy { + var policy: Policy = .{ + .id = try allocator.dupe(u8, id), + .name = try allocator.dupe(u8, id), + .enabled = true, + .target = .{ .log = .{ + .keep = try allocator.dupe(u8, keep), + } }, + }; + try policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, pattern) }, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, ext_type), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "cfg-bytes"), + .mode = mode, + }); + return policy; +} + +fn evalTestLogWithSink( + engine: *const PolicyEngine, + ctx: *TestLogContext, + policy_id_buf: [][]const u8, + sink: policy_types.ExtensionSink, +) PolicyResult { + return engine.evaluate(.log, &TestLogContext.accessor, ctx, policy_id_buf, .{ + .io = std.Options.debug_io, + .extension_sink = sink, + }); +} + +test "extensions: mode dropped delivers matched-and-dropped records only" { + const allocator = testing.allocator; + + var policy = try makeExtensionLogPolicy( + allocator, + "dump-waste", + "^drop", + "none", + "test/record", + .EXTENSION_MODE_DROPPED, + ); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + // Matched and dropped: delivered with slice .dropped. + var log1: TestLogContext = .{ .message = "drop me" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(FilterDecision.drop, r1.decision); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); + try testing.expectEqual(policy_types.ExtensionSlice.dropped, rec.deliveries.items[0].slice); + try testing.expectEqualStrings("dump-waste", rec.deliveries.items[0].policy_id); + try testing.expectEqual(@as(u8, 7), rec.deliveries.items[0].handler); + + // Unmatched: mode dropped does not select unmatched records. + var log2: TestLogContext = .{ .message = "keep me" }; + _ = evalTestLogWithSink(&engine, &log2, &policy_id_buf, rec.sink()); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); +} + +test "extensions: default mode (unspecified) selects matched, kept and dropped" { + const allocator = testing.allocator; + + var policy = try makeExtensionLogPolicy( + allocator, + "observe", + "payment", + "all", + "test/record", + .EXTENSION_MODE_UNSPECIFIED, + ); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + // Matched and kept → delivered as .kept. + var log1: TestLogContext = .{ .message = "payment ok" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(FilterDecision.keep, r1.decision); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); + try testing.expectEqual(policy_types.ExtensionSlice.kept, rec.deliveries.items[0].slice); + + // Unmatched → not selected by MATCHED. + var log2: TestLogContext = .{ .message = "healthcheck" }; + _ = evalTestLogWithSink(&engine, &log2, &policy_id_buf, rec.sink()); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); +} + +test "extensions: mode all delivers unmatched records" { + const allocator = testing.allocator; + + var policy = try makeExtensionLogPolicy(allocator, "mirror", "payment", "all", "test/record", .EXTENSION_MODE_ALL); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + var log1: TestLogContext = .{ .message = "healthcheck" }; + _ = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); + try testing.expectEqual(policy_types.ExtensionSlice.unmatched, rec.deliveries.items[0].slice); +} + +test "extensions: dropped slice reflects final outcome across policies" { + const allocator = testing.allocator; + + // Policy A drops payment logs; policy B keeps them but wants the waste. + // B's extension must see the record as .dropped because the FINAL pipeline + // outcome (most restrictive across all policies) is drop. + var drop_policy: Policy = .{ + .id = try allocator.dupe(u8, "a-drop"), + .name = try allocator.dupe(u8, "a-drop"), + .enabled = true, + .target = .{ .log = .{ .keep = try allocator.dupe(u8, "none") } }, + }; + try drop_policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "payment") }, + }); + defer drop_policy.deinit(allocator); + + var dump_policy = try makeExtensionLogPolicy( + allocator, + "b-dump", + "payment", + "all", + "test/record", + .EXTENSION_MODE_DROPPED, + ); + defer dump_policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{ drop_policy, dump_policy }, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + var log1: TestLogContext = .{ .message = "payment failed" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(FilterDecision.drop, r1.decision); + try testing.expectEqual(@as(usize, 1), rec.deliveries.items.len); + try testing.expectEqualStrings("b-dump", rec.deliveries.items[0].policy_id); + try testing.expectEqual(policy_types.ExtensionSlice.dropped, rec.deliveries.items[0].slice); +} + +test "extensions: unsupported type is skipped fail-open and reported" { + const allocator = testing.allocator; + + var policy = try makeExtensionLogPolicy( + allocator, + "unknown-ext", + "^drop", + "none", + "vendor/unknown", + .EXTENSION_MODE_DROPPED, + ); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + // The unsupported extension is reported via the compile-error channel... + try testing.expect(registry.policy_errors.count() == 1); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + var rec: RecordingExtensionSink = .{ .allocator = allocator }; + defer rec.deinit(); + + // ...but the policy's core keep semantics still apply (fail-open), and no + // delivery happens. + var log1: TestLogContext = .{ .message = "drop me" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, rec.sink()); + try testing.expectEqual(FilterDecision.drop, r1.decision); + try testing.expectEqual(@as(usize, 0), rec.deliveries.items.len); +} + +test "extensions: sink observes pre-transform record values" { + const allocator = testing.allocator; + + var transform: proto.policy.LogTransform = .{}; + try transform.redact.append(allocator, .{ + .field = .{ .log_attribute = try testMakeAttrPath(allocator, "service") }, + .replacement = try allocator.dupe(u8, "[REDACTED]"), + }); + + var policy: Policy = .{ + .id = try allocator.dupe(u8, "redact-and-dump"), + .name = try allocator.dupe(u8, "redact-and-dump"), + .enabled = true, + .target = .{ .log = .{ + .keep = try allocator.dupe(u8, "all"), + .transform = transform, + } }, + }; + try policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "sensitive") }, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, "test/record"), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, ""), + .mode = .EXTENSION_MODE_KEPT, + }); + defer policy.deinit(allocator); + + var res: TestExtensionResolver = .{}; + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(res.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + + // Sink that snapshots the record's `service` value at delivery time. + const ObservingSink = struct { + observed: ?[]const u8 = null, + buf: [64]u8 = undefined, + + fn deliverFn( + ctx_ptr: *anyopaque, + io: ?std.Io, + signal: TelemetryType, + record: *const anyopaque, + binding: *const policy_types.ExtensionBinding, + slice: policy_types.ExtensionSlice, + ) void { + _ = io; + _ = signal; + _ = binding; + _ = slice; + const Self = @This(); + const self: *Self = @ptrCast(@alignCast(ctx_ptr)); + const rec_ctx: *const MutableTestLogContext = @ptrCast(@alignCast(record)); + const service = rec_ctx.service orelse return; + const n = @min(service.len, self.buf.len); + @memcpy(self.buf[0..n], service[0..n]); + self.observed = self.buf[0..n]; + } + }; + var obs: ObservingSink = .{}; + + var ctx = MutableTestLogContext.init(allocator); + defer ctx.deinit(); + ctx.message = "sensitive data here"; + ctx.service = "secret-service"; + + var policy_id_buf: [16][]const u8 = undefined; + const result = engine.evaluate(.log, &MutableTestLogContext.accessor, &ctx, &policy_id_buf, .{ + .io = std.Options.debug_io, + .extension_sink = .{ .ctx = &obs, .deliver = ObservingSink.deliverFn }, + }); + + try testing.expectEqual(FilterDecision.keep, result.decision); + // The sink saw the pre-transform value... + try testing.expectEqualStrings("secret-service", obs.observed.?); + // ...and the transform still applied afterwards. + try testing.expectEqualStrings("[REDACTED]", ctx.service.?); +} + +// ============================================================================= +// Mock extension: spec-conformance tests (v1.6.0 §Extensions) +// ============================================================================= + +/// A full mock extension implementation, acting as both resolver and sink the +/// way a real handler module would: it declares a type id, gates on semver +/// major, validates config at resolve time, and records every delivery. Used +/// to verify the engine's dispatch behavior matches the spec. +const MockExtension = struct { + const type_id = "test/mock"; + const handler_id: u8 = 42; + + const Delivery = struct { + policy_id: []const u8, + signal: TelemetryType, + slice: policy_types.ExtensionSlice, + config: []const u8, + slot: u32, + }; + + allocator: std.mem.Allocator, + deliveries: std.ArrayList(Delivery) = .empty, + resolve_rejections: u32 = 0, + next_slot: u32 = 0, + + fn deinit(self: *MockExtension) void { + defer self.* = undefined; + for (self.deliveries.items) |d| { + self.allocator.free(d.policy_id); + self.allocator.free(d.config); + } + self.deliveries.deinit(self.allocator); + } + + fn resolver(self: *MockExtension) policy_types.ExtensionResolver { + return .{ .ctx = self, .resolve = resolveFn }; + } + + fn sink(self: *MockExtension) policy_types.ExtensionSink { + return .{ .ctx = self, .deliver = deliverFn }; + } + + fn resolveFn( + io: std.Io, + ctx_ptr: *anyopaque, + signal: TelemetryType, + policy_id: []const u8, + extension: *const proto.policy.Extension, + ) ?policy_types.ExtensionResolution { + _ = io; + _ = signal; + _ = policy_id; + const self: *MockExtension = @ptrCast(@alignCast(ctx_ptr)); + // Spec rule 4: implementations declare which types they support. + if (!std.mem.eql(u8, extension.type, type_id)) { + self.resolve_rejections += 1; + return null; + } + // Semver gate: only major 1 of this mock's schema is supported. + const version = std.SemanticVersion.parse(extension.version) catch { + self.resolve_rejections += 1; + return null; + }; + if (version.major != 1) { + self.resolve_rejections += 1; + return null; + } + // Config validation at compile time (spec: reject unsatisfiable + // config; the engine must skip the extension fail-open). + if (std.mem.eql(u8, extension.config, "invalid")) { + self.resolve_rejections += 1; + return null; + } + defer self.next_slot += 1; + return .{ .handler = handler_id, .slot = self.next_slot }; + } + + fn deliverFn( + ctx_ptr: *anyopaque, + io: ?std.Io, + signal: TelemetryType, + record: *const anyopaque, + binding: *const policy_types.ExtensionBinding, + slice: policy_types.ExtensionSlice, + ) void { + _ = io; + _ = record; + const self: *MockExtension = @ptrCast(@alignCast(ctx_ptr)); + std.debug.assert(binding.handler == handler_id); + const id = self.allocator.dupe(u8, binding.policy_id) catch return; + const config = self.allocator.dupe(u8, binding.config) catch { + self.allocator.free(id); + return; + }; + self.deliveries.append(self.allocator, .{ + .policy_id = id, + .signal = signal, + .slice = slice, + .config = config, + .slot = binding.slot, + }) catch { + self.allocator.free(id); + self.allocator.free(config); + }; + } + + fn count(self: *const MockExtension, policy_id: []const u8, slice: policy_types.ExtensionSlice) usize { + var n: usize = 0; + for (self.deliveries.items) |d| { + if (d.slice == slice and std.mem.eql(u8, d.policy_id, policy_id)) n += 1; + } + return n; + } + + fn total(self: *const MockExtension, policy_id: []const u8) usize { + var n: usize = 0; + for (self.deliveries.items) |d| { + if (std.mem.eql(u8, d.policy_id, policy_id)) n += 1; + } + return n; + } +}; + +test "mock extension: full mode x record-class dispatch matrix (spec Extension Input table)" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + // One policy per mode, all matching "payment" with keep "all", plus a + // separate drop policy so a record can be removed by a *different* + // policy's keep (the spec's final-pipeline-outcome semantics). + var policies: [6]Policy = undefined; + var initialized: usize = 0; + defer for (policies[0..initialized]) |*p| p.deinit(allocator); + + policies[0] = .{ + .id = try allocator.dupe(u8, "0-drop-declined"), + .name = try allocator.dupe(u8, "0-drop-declined"), + .enabled = true, + .target = .{ .log = .{ .keep = try allocator.dupe(u8, "none") } }, + }; + try policies[0].target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "declined") }, + }); + initialized = 1; + + const mode_policies = [_]struct { id: []const u8, mode: proto.policy.ExtensionMode }{ + .{ .id = "mode-kept", .mode = .EXTENSION_MODE_KEPT }, + .{ .id = "mode-dropped", .mode = .EXTENSION_MODE_DROPPED }, + .{ .id = "mode-unmatched", .mode = .EXTENSION_MODE_UNMATCHED }, + .{ .id = "mode-matched", .mode = .EXTENSION_MODE_MATCHED }, + .{ .id = "mode-all", .mode = .EXTENSION_MODE_ALL }, + }; + for (mode_policies, 1..) |mp, i| { + policies[i] = try makeExtensionLogPolicy( + allocator, + mp.id, + "payment", + "all", + MockExtension.type_id, + mp.mode, + ); + initialized = i + 1; + } + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&policies, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + // Record class 1: matches the mode policies, kept. + var kept_log: TestLogContext = .{ .message = "payment ok" }; + const r1 = evalTestLogWithSink(&engine, &kept_log, &policy_id_buf, mock.sink()); + try testing.expectEqual(FilterDecision.keep, r1.decision); + + // Record class 2: matches the mode policies AND the drop policy — the + // final outcome is drop, caused by a different policy. + var dropped_log: TestLogContext = .{ .message = "payment declined" }; + const r2 = evalTestLogWithSink(&engine, &dropped_log, &policy_id_buf, mock.sink()); + try testing.expectEqual(FilterDecision.drop, r2.decision); + + // Record class 3: matches nothing. + var unmatched_log: TestLogContext = .{ .message = "healthcheck" }; + const r3 = evalTestLogWithSink(&engine, &unmatched_log, &policy_id_buf, mock.sink()); + try testing.expectEqual(FilterDecision.unset, r3.decision); + + // The spec's Extension Input table, row by row: + // kept: matched and survived keep. + try testing.expectEqual(@as(usize, 1), mock.count("mode-kept", .kept)); + try testing.expectEqual(@as(usize, 1), mock.total("mode-kept")); + // dropped: matched but removed by the final keep outcome. + try testing.expectEqual(@as(usize, 1), mock.count("mode-dropped", .dropped)); + try testing.expectEqual(@as(usize, 1), mock.total("mode-dropped")); + // unmatched: did not match the policy's matchers. + try testing.expectEqual(@as(usize, 1), mock.count("mode-unmatched", .unmatched)); + try testing.expectEqual(@as(usize, 1), mock.total("mode-unmatched")); + // matched = kept + dropped, never unmatched. + try testing.expectEqual(@as(usize, 1), mock.count("mode-matched", .kept)); + try testing.expectEqual(@as(usize, 1), mock.count("mode-matched", .dropped)); + try testing.expectEqual(@as(usize, 2), mock.total("mode-matched")); + // all = every record of the signal. + try testing.expectEqual(@as(usize, 1), mock.count("mode-all", .kept)); + try testing.expectEqual(@as(usize, 1), mock.count("mode-all", .dropped)); + try testing.expectEqual(@as(usize, 1), mock.count("mode-all", .unmatched)); + try testing.expectEqual(@as(usize, 8), mock.deliveries.items.len); +} + +test "mock extension: multiple extensions per policy, opaque config routed per declaration" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + // Spec rule 1: a policy MAY declare multiple extensions. Each has its own + // mode and its own opaque config. + var policy = try makeExtensionLogPolicy( + allocator, + "multi-ext", + "payment", + "all", + MockExtension.type_id, + .EXTENSION_MODE_KEPT, + ); + defer policy.deinit(allocator); + // Overwrite first extension's config, then add a second extension. + allocator.free(@constCast(policy.extensions.items[0].config)); + policy.extensions.items[0].config = try allocator.dupe(u8, "cfg-A"); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.2.3"), + .config = try allocator.dupe(u8, "cfg-B"), + .mode = .EXTENSION_MODE_DROPPED, + }); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + // Kept record: only the mode-kept extension (cfg-A) fires. + var log1: TestLogContext = .{ .message = "payment ok" }; + _ = evalTestLogWithSink(&engine, &log1, &policy_id_buf, mock.sink()); + try testing.expectEqual(@as(usize, 1), mock.deliveries.items.len); + try testing.expectEqualStrings("cfg-A", mock.deliveries.items[0].config); + try testing.expectEqual(policy_types.ExtensionSlice.kept, mock.deliveries.items[0].slice); + + // Re-evaluating delivers through the same binding with a stable slot. + const slot_a = mock.deliveries.items[0].slot; + var log2: TestLogContext = .{ .message = "payment ok" }; + _ = evalTestLogWithSink(&engine, &log2, &policy_id_buf, mock.sink()); + try testing.expectEqual(@as(usize, 2), mock.deliveries.items.len); + try testing.expectEqual(slot_a, mock.deliveries.items[1].slot); +} + +test "mock extension: version and config validation reject fail-open at compile time" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + // Three extensions: unsupported major, invalid config, and a good one. + var policy = try makeExtensionLogPolicy( + allocator, + "gate-ext", + "^drop", + "none", + MockExtension.type_id, + .EXTENSION_MODE_DROPPED, + ); + defer policy.deinit(allocator); + allocator.free(@constCast(policy.extensions.items[0].version)); + policy.extensions.items[0].version = try allocator.dupe(u8, "2.0.0"); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "invalid"), + .mode = .EXTENSION_MODE_DROPPED, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "good"), + .mode = .EXTENSION_MODE_DROPPED, + }); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + // Both rejects were reported under the policy's id... + try testing.expectEqual(@as(u32, 2), mock.resolve_rejections); + try testing.expectEqual(@as(usize, 2), registry.policy_errors.get("gate-ext").?.items.len); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + // ...the policy's core keep still applies (fail-open), and only the valid + // extension fires. + var log1: TestLogContext = .{ .message = "drop me" }; + const r1 = evalTestLogWithSink(&engine, &log1, &policy_id_buf, mock.sink()); + try testing.expectEqual(FilterDecision.drop, r1.decision); + try testing.expectEqual(@as(usize, 1), mock.deliveries.items.len); + try testing.expectEqualStrings("good", mock.deliveries.items[0].config); +} + +test "mock extension: dispatch MUST NOT change the evaluation outcome" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + // keep-all policy with a transform AND a mode-all extension: the result + // with a sink wired must be identical to the result without one, and the + // transform must still apply. + var transform: proto.policy.LogTransform = .{}; + try transform.redact.append(allocator, .{ + .field = .{ .log_attribute = try testMakeAttrPath(allocator, "service") }, + .replacement = try allocator.dupe(u8, "[REDACTED]"), + }); + var policy: Policy = .{ + .id = try allocator.dupe(u8, "redact-observed"), + .name = try allocator.dupe(u8, "redact-observed"), + .enabled = true, + .target = .{ .log = .{ + .keep = try allocator.dupe(u8, "all"), + .transform = transform, + } }, + }; + try policy.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "sensitive") }, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "cfg"), + .mode = .EXTENSION_MODE_ALL, + }); + defer policy.deinit(allocator); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var with_sink = MutableTestLogContext.init(allocator); + defer with_sink.deinit(); + with_sink.message = "sensitive data"; + with_sink.service = "secret-service"; + const r_with = engine.evaluate(.log, &MutableTestLogContext.accessor, &with_sink, &policy_id_buf, .{ + .io = std.Options.debug_io, + .extension_sink = mock.sink(), + }); + + var without_sink = MutableTestLogContext.init(allocator); + defer without_sink.deinit(); + without_sink.message = "sensitive data"; + without_sink.service = "secret-service"; + var policy_id_buf2: [16][]const u8 = undefined; + const r_without = engine.evaluate(.log, &MutableTestLogContext.accessor, &without_sink, &policy_id_buf2, .{ + .io = std.Options.debug_io, + }); + + // Identical decision, matches, and transform effects either way. + try testing.expectEqual(r_without.decision, r_with.decision); + try testing.expectEqual(r_without.was_transformed, r_with.was_transformed); + try testing.expectEqual(r_without.matched_policy_ids.len, r_with.matched_policy_ids.len); + try testing.expectEqualStrings("[REDACTED]", with_sink.service.?); + try testing.expectEqualStrings("[REDACTED]", without_sink.service.?); + try testing.expectEqual(@as(usize, 1), mock.deliveries.items.len); +} + +test "mock extension: disabled policy's extensions are inactive" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + var policy = try makeExtensionLogPolicy( + allocator, + "disabled-ext", + "payment", + "all", + MockExtension.type_id, + .EXTENSION_MODE_ALL, + ); + policy.enabled = false; + defer policy.deinit(allocator); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var log1: TestLogContext = .{ .message = "payment ok" }; + _ = evalTestLogWithSink(&engine, &log1, &policy_id_buf, mock.sink()); + try testing.expectEqual(@as(usize, 0), mock.deliveries.items.len); +} + +test "mock extension: metric policies dispatch with the metric signal" { + const allocator = testing.allocator; + var mock: MockExtension = .{ .allocator = allocator }; + defer mock.deinit(); + + var policy: Policy = .{ + .id = try allocator.dupe(u8, "metric-ext"), + .name = try allocator.dupe(u8, "metric-ext"), + .enabled = true, + .target = .{ .metric = .{ .keep = true } }, + }; + try policy.target.?.metric.match.append(allocator, .{ + .field = .{ .metric_field = .METRIC_FIELD_NAME }, + .match = .{ .regex = try allocator.dupe(u8, "^cpu") }, + }); + try policy.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, MockExtension.type_id), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, "cfg"), + .mode = .EXTENSION_MODE_MATCHED, + }); + defer policy.deinit(allocator); + + var noop_bus: NoopEventBus = undefined; + noop_bus.init(std.Options.debug_io); + var registry = PolicyRegistry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(mock.resolver()); + try registry.updatePolicies(&.{policy}, "test", .file); + + const engine = PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var metric: TestMetricContext = .{ .name = "cpu.usage" }; + const result = engine.evaluate(.metric, &TestMetricContext.accessor, &metric, &policy_id_buf, .{ + .io = std.Options.debug_io, + .extension_sink = mock.sink(), + }); + try testing.expectEqual(FilterDecision.keep, result.decision); + try testing.expectEqual(@as(usize, 1), mock.deliveries.items.len); + try testing.expectEqual(TelemetryType.metric, mock.deliveries.items[0].signal); + try testing.expectEqual(policy_types.ExtensionSlice.kept, mock.deliveries.items[0].slice); +} diff --git a/src/policy/provider_http.zig b/src/policy/provider_http.zig index 4bdaf2f..e7e872f 100644 --- a/src/policy/provider_http.zig +++ b/src/policy/provider_http.zig @@ -42,6 +42,28 @@ const HttpSyncStatusReported = struct { id: []const u8, match_hits: i64, match_m const HttpFetchStarted = struct {}; const HttpFetchCompleted = struct {}; +/// Extension sync plumbing (spec v1.6.0), implemented outside policy_zig by +/// the extensions module: capability advertisement for sync requests and +/// routing of broadcast extension configs from responses. A fn-pointer seam +/// like StatsCollector — not a vtable of behaviors, just the two crossings. +pub const ExtensionSyncHooks = struct { + ctx: *anyopaque, + /// Build ClientMetadata.supported_extensions. Allocate from `arena`. + capabilities: *const fn ( + io: std.Io, + ctx: *anyopaque, + arena: std.mem.Allocator, + ) anyerror![]proto.policy.ExtensionCapability, + /// Receive SyncResponse.extension_configs. Called before policies are + /// handed to the registry, so broadcast targets exist by the time the + /// snapshot compiles extension bindings against them. + apply_configs: *const fn ( + io: std.Io, + ctx: *anyopaque, + configs: []const proto.policy.ExtensionConfig, + ) void, +}; + /// HTTP-based policy provider that polls a remote endpoint pub const HttpProvider = struct { allocator: std.mem.Allocator, @@ -73,6 +95,9 @@ pub const HttpProvider = struct { // each sync to obtain the per-policy hit/miss/error rows to report. stats_collector: ?StatsCollector, + // Extension sync plumbing (v1.6.0), set via setExtensionSyncHooks. + extension_hooks: ?ExtensionSyncHooks, + pub const Config = struct { id: []const u8, url: []const u8, @@ -133,6 +158,7 @@ pub const HttpProvider = struct { .sync_state_mutex = .init, .bus = bus, .stats_collector = null, + .extension_hooks = null, }; return self; @@ -143,6 +169,11 @@ pub const HttpProvider = struct { return self.id; } + /// Wire extension sync plumbing (v1.6.0). Call before start(). + pub fn setExtensionSyncHooks(self: *HttpProvider, hooks: ExtensionSyncHooks) void { + self.extension_hooks = hooks; + } + /// Record the hash from a successful sync. /// This hash will be sent in subsequent sync requests. pub fn recordSyncedHash(self: *HttpProvider, hash: []const u8) !void { @@ -275,6 +306,12 @@ pub const HttpProvider = struct { self.bus.info(event); } + // Route broadcast extension configs first, so targets they carry are + // known before the registry compiles the policies that reference them. + if (self.extension_hooks) |hooks| { + hooks.apply_configs(self.bus.io, hooks.ctx, response.extension_configs.items); + } + // Notify callback with policies from response if (self.callback) |cb| { try cb.call(.{ @@ -354,9 +391,20 @@ pub const HttpProvider = struct { last_hash = self.last_successful_hash orelse &.{}; } + // Extension capability advertisement (v1.6.0). Failure here must not + // block a sync: advertise nothing instead. + const supported_extensions: []proto.policy.ExtensionCapability = if (self.extension_hooks) |hooks| + hooks.capabilities(self.bus.io, hooks.ctx, temp_allocator) catch &.{} + else + &.{}; + // Create SyncRequest with the new structure const sync_request: SyncRequest = .{ .client_metadata = .{ + .supported_extensions = .{ + .items = supported_extensions, + .capacity = supported_extensions.len, + }, .supported_policy_stages = .{ .items = @constCast(supported_policy_stages), .capacity = supported_policy_stages.len, diff --git a/src/policy/registry.zig b/src/policy/registry.zig index a5f9c7b..99fda78 100644 --- a/src/policy/registry.zig +++ b/src/policy/registry.zig @@ -250,6 +250,10 @@ pub const PolicyRegistry = struct { // Event bus for observability bus: *EventBus, + // Optional extension resolver (v1.6.0), consulted at snapshot compile + // time to resolve/validate each policy's extension declarations. + extension_resolver: ?policy_types.ExtensionResolver, + /// Subscription context for provider callbacks. /// Allocated with stable address so the callback pointer remains valid. const Subscription = struct { @@ -284,9 +288,18 @@ pub const PolicyRegistry = struct { .policy_errors = .empty, .subscriptions = .empty, .bus = bus, + .extension_resolver = null, }; } + /// Wire an extension resolver (v1.6.0), consulted at snapshot compile + /// time. Call before providers deliver policies; takes effect on the next + /// recompile. Without one, declared extensions are skipped (fail-open) + /// and reported via PolicySyncStatus.errors. + pub fn setExtensionResolver(self: *PolicyRegistry, resolver: policy_types.ExtensionResolver) void { + self.extension_resolver = resolver; + } + /// Subscribe a provider to the registry in one step: hand it our stats /// collector (so it can pull per-policy hit/miss/error rows before each sync) /// and subscribe to its policy updates with an internal callback that feeds @@ -663,6 +676,7 @@ pub const PolicyRegistry = struct { self.bus, policies_slice, &comp_errors, + self.extension_resolver, ); errdefer log_idx.deinit(); @@ -671,6 +685,7 @@ pub const PolicyRegistry = struct { self.bus, policies_slice, &comp_errors, + self.extension_resolver, ); errdefer metric_idx.deinit(); @@ -679,6 +694,7 @@ pub const PolicyRegistry = struct { self.bus, policies_slice, &comp_errors, + self.extension_resolver, ); errdefer trace_idx.deinit(); diff --git a/src/policy/root.zig b/src/policy/root.zig index c8cece5..8434fbd 100644 --- a/src/policy/root.zig +++ b/src/policy/root.zig @@ -101,6 +101,14 @@ pub const TraceAccessor = types.TraceAccessor; pub const TypedValue = types.TypedValue; pub const TelemetryType = types.TelemetryType; +// Extensions (spec v1.6.0) +pub const ExtensionSlice = types.ExtensionSlice; +pub const ExtensionSliceSet = types.ExtensionSliceSet; +pub const ExtensionBinding = types.ExtensionBinding; +pub const ExtensionResolution = types.ExtensionResolution; +pub const ExtensionResolver = types.ExtensionResolver; +pub const ExtensionSink = types.ExtensionSink; + // ============================================================================= // Matcher Index (Hyperscan-based pattern matching) // ============================================================================= diff --git a/src/policy/types.zig b/src/policy/types.zig index 909ccd0..25b2894 100644 --- a/src/policy/types.zig +++ b/src/policy/types.zig @@ -22,6 +22,95 @@ pub const TelemetryType = enum { trace, }; +// ============================================================================= +// Extensions (spec v1.6.0) — dispatch types +// ============================================================================= + +/// Which disjoint slice of the telemetry stream a record fell into, relative +/// to one policy: matched and kept, matched but removed by the final keep +/// outcome (which another policy may have caused), or unmatched. The proto +/// ExtensionMode values MATCHED and ALL are unions over these, resolved to an +/// ExtensionSliceSet at snapshot compile time. +pub const ExtensionSlice = enum(u2) { kept, dropped, unmatched }; + +pub const ExtensionSliceSet = std.EnumSet(ExtensionSlice); + +/// Resolve a proto ExtensionMode to the slice set it selects. UNSPECIFIED +/// and unknown future modes resolve to MATCHED, the spec default. +pub fn extensionSliceSet(mode: proto.policy.ExtensionMode) ExtensionSliceSet { + return switch (mode) { + .EXTENSION_MODE_KEPT => .init(.{ .kept = true }), + .EXTENSION_MODE_DROPPED => .init(.{ .dropped = true }), + .EXTENSION_MODE_UNMATCHED => .init(.{ .unmatched = true }), + .EXTENSION_MODE_ALL => .init(.{ .kept = true, .dropped = true, .unmatched = true }), + .EXTENSION_MODE_MATCHED, .EXTENSION_MODE_UNSPECIFIED, _ => .init(.{ .kept = true, .dropped = true }), + }; +} + +/// One (policy, extension) pair compiled into a signal's matcher index. +/// `handler` and `slot` are assigned by the resolver at snapshot compile time +/// and are opaque to policy_zig — the sink that receives the binding +/// interprets them. Dispatch never re-parses the extension declaration or +/// compares type strings. +pub const ExtensionBinding = struct { + /// Index into the signal index's policy list (matcher_index.PolicyIndex). + policy_index: u16, + /// Policy id, owned by the index (valid for the snapshot's lifetime). + policy_id: []const u8, + /// Opaque handler identifier from the resolver (e.g. an enum tag value). + handler: u8, + /// Opaque per-binding slot from the resolver (e.g. a batch index). + slot: u32, + /// Slices of the stream this binding's mode selects. + slices: ExtensionSliceSet, + /// Extension config bytes, owned by the index. + config: []const u8, +}; + +/// Result of resolving one extension declaration to a handler. +pub const ExtensionResolution = struct { + handler: u8, + slot: u32, +}; + +/// Resolves and validates one extension declaration at snapshot compile time. +/// Implemented outside policy_zig (the extensions module); policy_zig only +/// stores the opaque resolution in the binding. Return null when the +/// extension is unsupported or its config is invalid: per spec rule 5 the +/// extension is skipped (fail-open) and reported via PolicySyncStatus.errors, +/// while the policy's core match/keep/transform still applies. +pub const ExtensionResolver = struct { + ctx: *anyopaque, + resolve: *const fn ( + io: std.Io, + ctx: *anyopaque, + signal: TelemetryType, + policy_id: []const u8, + extension: *const proto.policy.Extension, + ) ?ExtensionResolution, +}; + +/// Receives records selected by an extension's mode. One function pointer + +/// ctx (the shape of the accessor primitives, not a vtable): it exists only +/// to break the module cycle — policy_zig cannot import the extensions +/// module that implements it. +/// +/// Called synchronously inside `evaluate`, after the final keep outcome is +/// known and before transforms run, so implementations observe pre-transform +/// records. `record` is borrowed and valid ONLY for the duration of the +/// call; implementations copy (encode) what they keep and MUST NOT block. +pub const ExtensionSink = struct { + ctx: *anyopaque, + deliver: *const fn ( + ctx: *anyopaque, + io: ?std.Io, + signal: TelemetryType, + record: *const anyopaque, + binding: *const ExtensionBinding, + slice: ExtensionSlice, + ) void, +}; + // ============================================================================= // Service and Provider Configuration // ============================================================================= diff --git a/src/proto/tero/policy/v1.pb.zig b/src/proto/tero/policy/v1.pb.zig index 3f05abd..981994e 100644 --- a/src/proto/tero/policy/v1.pb.zig +++ b/src/proto/tero/policy/v1.pb.zig @@ -9,6 +9,276 @@ const google_api = @import("../../google/api.pb.zig"); /// import package opentelemetry.proto.common.v1 const opentelemetry_proto_common_v1 = @import("../../opentelemetry/proto/common/v1.pb.zig"); +/// ExtensionMode selects which slice of the telemetry stream the engine delivers +/// to an extension, relative to its policy. {KEPT, DROPPED, UNMATCHED} are +/// disjoint and partition the stream; MATCHED and ALL are convenience unions. +pub const ExtensionMode = enum(i32) { + EXTENSION_MODE_UNSPECIFIED = 0, + EXTENSION_MODE_KEPT = 1, + EXTENSION_MODE_DROPPED = 2, + EXTENSION_MODE_UNMATCHED = 3, + EXTENSION_MODE_MATCHED = 4, + EXTENSION_MODE_ALL = 5, + _, +}; + +/// Extension attaches implementation-specific behavior to a policy. +/// +/// Implementations MUST declare which extension types they support. An extension +/// with an unrecognized type is handled per the spec (fail-open, or reject policy +/// load when the extension is required for the policy's behavior). +pub const Extension = struct { + type: []const u8 = &.{}, + version: []const u8 = &.{}, + config: []const u8 = &.{}, + mode: ExtensionMode = @enumFromInt(0), + + pub const _desc_table = .{ + .type = fd(1, .{ .scalar = .string }), + .version = fd(2, .{ .scalar = .string }), + .config = fd(3, .{ .scalar = .bytes }), + .mode = fd(4, .@"enum"), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +}; + +/// ExtensionCapability declares a client's support for one extension type. The +/// repeated `config` entries are opaque, type-defined capability descriptors — +/// for example, the destinations the client can route to. The policy engine does +/// not interpret them; the extension handler does. Providers use this to avoid +/// sending policies whose extensions the client cannot satisfy. +pub const ExtensionCapability = struct { + type: []const u8 = &.{}, + min_version: []const u8 = &.{}, + config: std.ArrayList([]const u8) = .empty, + + pub const _desc_table = .{ + .type = fd(1, .{ .scalar = .string }), + .min_version = fd(2, .{ .scalar = .string }), + .config = fd(3, .{ .repeated = .{ .scalar = .bytes } }), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +}; + +/// ExtensionConfig broadcasts opaque, type-defined configuration for one +/// extension type to clients (via SyncResponse.extension_configs). Each `config` +/// entry is parsed by the handler registered for `type`. For tero's s3-dump +/// these are serialized ExtensionTargets (see tero_extensions.proto). +pub const ExtensionConfig = struct { + type: []const u8 = &.{}, + config: std.ArrayList([]const u8) = .empty, + + pub const _desc_table = .{ + .type = fd(1, .{ .scalar = .string }), + .config = fd(2, .{ .repeated = .{ .scalar = .bytes } }), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +}; + /// AttributePath specifies how to access an attribute value. /// /// The path is represented as an array of string segments. Each segment represents @@ -1873,6 +2143,7 @@ pub const Policy = struct { created_at_unix_nano: u64 = 0, modified_at_unix_nano: u64 = 0, labels: std.ArrayList(opentelemetry_proto_common_v1.KeyValue) = .empty, + extensions: std.ArrayList(Extension) = .empty, target: ?target_union = null, pub const _target_case = enum { @@ -1899,6 +2170,7 @@ pub const Policy = struct { .created_at_unix_nano = fd(5, .{ .scalar = .fixed64 }), .modified_at_unix_nano = fd(6, .{ .scalar = .fixed64 }), .labels = fd(7, .{ .repeated = .submessage }), + .extensions = fd(20, .{ .repeated = .submessage }), .target = fd(null, .{ .oneof = target_union }), }; @@ -1977,11 +2249,13 @@ pub const ClientMetadata = struct { supported_policy_stages: std.ArrayList(PolicyStage) = .empty, labels: std.ArrayList(opentelemetry_proto_common_v1.KeyValue) = .empty, resource_attributes: std.ArrayList(opentelemetry_proto_common_v1.KeyValue) = .empty, + supported_extensions: std.ArrayList(ExtensionCapability) = .empty, pub const _desc_table = .{ .supported_policy_stages = fd(1, .{ .packed_repeated = .@"enum" }), .labels = fd(2, .{ .repeated = .submessage }), .resource_attributes = fd(3, .{ .repeated = .submessage }), + .supported_extensions = fd(4, .{ .repeated = .submessage }), }; /// Encodes the message to the writer @@ -2321,6 +2595,7 @@ pub const SyncResponse = struct { recommended_sync_interval_seconds: u32 = 0, sync_type: SyncType = @enumFromInt(0), error_message: []const u8 = &.{}, + extension_configs: std.ArrayList(ExtensionConfig) = .empty, pub const _desc_table = .{ .policies = fd(1, .{ .repeated = .submessage }), @@ -2329,6 +2604,7 @@ pub const SyncResponse = struct { .recommended_sync_interval_seconds = fd(4, .{ .scalar = .uint32 }), .sync_type = fd(5, .@"enum"), .error_message = fd(6, .{ .scalar = .string }), + .extension_configs = fd(7, .{ .repeated = .submessage }), }; /// Encodes the message to the writer @@ -2411,3 +2687,175 @@ pub fn PolicyService(comptime UserDataType: type, comptime ErrorSet: type) type Sync: *const fn (userdata: *UserDataType, request: SyncRequest) ErrorSet!SyncResponse, }; } + +/// ExtensionTarget is a pre-configured, named destination an extension can route +/// telemetry to (for example, an S3 bucket or a downstream OTLP endpoint). +/// +/// Targets are configured out of band on the implementation, or broadcast to +/// clients via SyncResponse.extension_configs (serialized into the s3-dump +/// extension's config). Policies reference a target by (kind, name) via +/// ExtensionTargetRef and never carry its configuration inline. +pub const ExtensionTarget = struct { + kind: []const u8 = &.{}, + name: []const u8 = &.{}, + config: []const u8 = &.{}, + + pub const _desc_table = .{ + .kind = fd(1, .{ .scalar = .string }), + .name = fd(2, .{ .scalar = .string }), + .config = fd(3, .{ .scalar = .bytes }), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +}; + +/// ExtensionTargetRef references a pre-configured ExtensionTarget by identity. +/// Carried inside an Extension's config (to name where it routes) and inside an +/// ExtensionCapability's config (to advertise reachable destinations). The +/// referenced target MUST be known to the implementation — locally configured or +/// received via SyncResponse.extension_configs. +pub const ExtensionTargetRef = struct { + kind: []const u8 = &.{}, + name: []const u8 = &.{}, + + pub const _desc_table = .{ + .kind = fd(1, .{ .scalar = .string }), + .name = fd(2, .{ .scalar = .string }), + }; + + /// Encodes the message to the writer + /// The allocator is used to generate submessages internally. + /// Hence, an ArenaAllocator is a preferred choice if allocations are a bottleneck. + pub fn encode( + self: @This(), + writer: *std.Io.Writer, + allocator: std.mem.Allocator, + ) (std.Io.Writer.Error || std.mem.Allocator.Error)!void { + return protobuf.encode(writer, allocator, self); + } + + /// Decodes the message from the bytes read from the reader. + pub fn decode( + reader: *std.Io.Reader, + allocator: std.mem.Allocator, + ) (protobuf.DecodingError || std.Io.Reader.Error || std.mem.Allocator.Error)!@This() { + return protobuf.decode(@This(), reader, allocator); + } + + /// Deinitializes and frees the memory associated with the message. + pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void { + return protobuf.deinit(allocator, self); + } + + /// Duplicates the message. + pub fn dupe(self: @This(), allocator: std.mem.Allocator) std.mem.Allocator.Error!@This() { + return protobuf.dupe(@This(), self, allocator); + } + + /// Decodes the message from the JSON string. + pub fn jsonDecode( + input: []const u8, + options: std.json.ParseOptions, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decode(@This(), input, options, allocator); + } + + /// Decodes the message from the JSON string, honoring pb options + /// (e.g. hex_bytes_fields for OTLP trace_id/span_id). + pub fn jsonDecodeOpts( + input: []const u8, + options: std.json.ParseOptions, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) !std.json.Parsed(@This()) { + return protobuf.json.decodeOpts(@This(), input, options, pb_options, allocator); + } + + /// Encodes the message to a JSON string. + pub fn jsonEncode( + self: @This(), + options: std.json.Stringify.Options, + pb_options: protobuf.json.Options, + allocator: std.mem.Allocator, + ) ![]const u8 { + return protobuf.json.encode(self, options, pb_options, allocator); + } + + /// This method is used by std.json + /// internally for deserialization. DO NOT RENAME! + pub fn jsonParse( + allocator: std.mem.Allocator, + source: anytype, + options: std.json.ParseOptions, + ) !@This() { + return protobuf.json.parse(@This(), allocator, source, options); + } +}; From 9841894cc9709dd6349906bd3e6ec2a6a332dc32 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Tue, 7 Jul 2026 18:07:37 -0400 Subject: [PATCH 5/8] s3 extension wiring --- proto/tero/policy/v1/extension.proto | 96 ++++ proto/tero/policy/v1/tero_extensions.proto | 41 ++ src/extensions/root.zig | 381 +++++++++++++ src/extensions/s3_dump.zig | 594 +++++++++++++++++++++ 4 files changed, 1112 insertions(+) create mode 100644 proto/tero/policy/v1/extension.proto create mode 100644 proto/tero/policy/v1/tero_extensions.proto create mode 100644 src/extensions/root.zig create mode 100644 src/extensions/s3_dump.zig diff --git a/proto/tero/policy/v1/extension.proto b/proto/tero/policy/v1/extension.proto new file mode 100644 index 0000000..809a766 --- /dev/null +++ b/proto/tero/policy/v1/extension.proto @@ -0,0 +1,96 @@ +syntax = "proto3"; + +package tero.policy.v1; + +option go_package = "github.com/usetero/policy/gen/go/tero/policy/v1"; + +// ============================================================================= +// Extensions +// ============================================================================= +// +// Extensions are a fully generic mechanism for attaching implementation-specific +// behavior to a policy. The policy engine matches telemetry using the core +// target (log/metric/trace), computes the final keep outcome, then dispatches +// selected records to the handler registered for the extension `type` before +// transforms run. All extension payloads are opaque bytes: the engine never +// interprets them, so any implementation can define its own extension types +// without changing the core schema. +// +// tero's own extension types (e.g. the s3-dump destination) live in +// tero_extensions.proto. + +// ExtensionMode selects which slice of the telemetry stream the engine delivers +// to an extension, relative to its policy. {KEPT, DROPPED, UNMATCHED} are +// disjoint and partition the stream; MATCHED and ALL are convenience unions. +enum ExtensionMode { + EXTENSION_MODE_UNSPECIFIED = 0; // treated as MATCHED + + // Matched the policy and survived the keep stage. + EXTENSION_MODE_KEPT = 1; + + // Matched the policy but was removed by the keep stage (dropped, sampled out, + // or rate limited). Its keep verdict is the record's final pipeline outcome, + // which another matching policy may have caused. This is the "dump waste" mode. + EXTENSION_MODE_DROPPED = 2; + + // Did not match the policy's matchers. + EXTENSION_MODE_UNMATCHED = 3; + + // Matched the policy, regardless of keep outcome (KEPT + DROPPED). Default. + EXTENSION_MODE_MATCHED = 4; + + // Every record of the signal type, regardless of match or keep + // (KEPT + DROPPED + UNMATCHED). + EXTENSION_MODE_ALL = 5; +} + +// Extension attaches implementation-specific behavior to a policy. +// +// Implementations MUST declare which extension types they support. An extension +// with an unrecognized type is handled per the spec (fail-open, or reject policy +// load when the extension is required for the policy's behavior). +message Extension { + // Type identifier using reverse-FQDN notation (e.g. "com.usetero/s3-dump"). + // Implementations use this to route to the correct handler. + string type = 1; + + // Version of the extension schema (semver, e.g. "1.0.0"). + string version = 2; + + // Opaque, type-defined configuration payload, deserialized by the handler + // registered for `type`. The policy engine does not interpret it. + bytes config = 3; + + // Which slice of traffic the engine delivers to this extension. Defaults to + // MATCHED. + ExtensionMode mode = 4; +} + +// ExtensionCapability declares a client's support for one extension type. The +// repeated `config` entries are opaque, type-defined capability descriptors — +// for example, the destinations the client can route to. The policy engine does +// not interpret them; the extension handler does. Providers use this to avoid +// sending policies whose extensions the client cannot satisfy. +message ExtensionCapability { + // Extension type identifier (e.g. "com.usetero/s3-dump"). + string type = 1; + + // Minimum supported extension version (semver, inclusive). + string min_version = 2; + + // Opaque, type-defined capability descriptors. For tero's s3-dump these are + // serialized ExtensionTargetRefs (see tero_extensions.proto). + repeated bytes config = 3; +} + +// ExtensionConfig broadcasts opaque, type-defined configuration for one +// extension type to clients (via SyncResponse.extension_configs). Each `config` +// entry is parsed by the handler registered for `type`. For tero's s3-dump +// these are serialized ExtensionTargets (see tero_extensions.proto). +message ExtensionConfig { + // Extension type identifier (e.g. "com.usetero/s3-dump"). + string type = 1; + + // Opaque, type-defined configuration payloads. + repeated bytes config = 2; +} diff --git a/proto/tero/policy/v1/tero_extensions.proto b/proto/tero/policy/v1/tero_extensions.proto new file mode 100644 index 0000000..2df6401 --- /dev/null +++ b/proto/tero/policy/v1/tero_extensions.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package tero.policy.v1; + +option go_package = "github.com/usetero/policy/gen/go/tero/policy/v1"; + +// ============================================================================= +// tero Extension Types +// ============================================================================= +// +// Concrete extension types defined by tero, carried inside the opaque `config` +// bytes of the generic extension messages in extension.proto. These are not +// part of the core policy mechanism — other implementations define their own. + +// ExtensionTarget is a pre-configured, named destination an extension can route +// telemetry to (for example, an S3 bucket or a downstream OTLP endpoint). +// +// Targets are configured out of band on the implementation, or broadcast to +// clients via SyncResponse.extension_configs (serialized into the s3-dump +// extension's config). Policies reference a target by (kind, name) via +// ExtensionTargetRef and never carry its configuration inline. +message ExtensionTarget { + // Destination kind, e.g. "s3" or "otlp". + string kind = 1; + + // Name, unique within a kind, e.g. "eu-bucket". Used as the reference key. + string name = 2; + + // Opaque, kind-defined configuration (e.g. region, bucket, prefix). + bytes config = 3; +} + +// ExtensionTargetRef references a pre-configured ExtensionTarget by identity. +// Carried inside an Extension's config (to name where it routes) and inside an +// ExtensionCapability's config (to advertise reachable destinations). The +// referenced target MUST be known to the implementation — locally configured or +// received via SyncResponse.extension_configs. +message ExtensionTargetRef { + string kind = 1; + string name = 2; +} diff --git a/src/extensions/root.zig b/src/extensions/root.zig new file mode 100644 index 0000000..2ac2275 --- /dev/null +++ b/src/extensions/root.zig @@ -0,0 +1,381 @@ +//! Policy extensions (spec v1.6.0): concrete extension handlers and the +//! adapters that plug them into policy_zig's resolver/sink seams. +//! +//! Handlers are a tagged union, not a vtable (see Claude.md "Encoding Instead +//! of Polymorphism") — extension types are a closed set the implementation +//! must declare anyway (spec rule 4). Adding a type = one enum tag + one +//! union arm; the compiler enforces exhaustiveness everywhere. + +const std = @import("std"); +const policy = @import("policy_zig"); +const proto = @import("proto"); + +const s3_dump_mod = @import("./s3_dump.zig"); +pub const S3Dump = s3_dump_mod.S3Dump; +pub const EncodeFn = s3_dump_mod.EncodeFn; + +pub const ExtensionTag = enum(u8) { + s3_dump, + + pub fn typeId(tag: ExtensionTag) []const u8 { + return switch (tag) { + .s3_dump => "com.usetero/s3-dump", + }; + } + + /// Highest supported major version per type (semver gate: an extension + /// declaring a different major is unsupported). + pub fn supportedMajor(tag: ExtensionTag) usize { + return switch (tag) { + .s3_dump => 1, + }; + } + + fn fromTypeId(type_id: []const u8) ?ExtensionTag { + inline for (comptime std.enums.values(ExtensionTag)) |tag| { + if (std.mem.eql(u8, type_id, comptime tag.typeId())) return tag; + } + return null; + } +}; + +pub const ExtensionHandler = union(ExtensionTag) { + s3_dump: S3Dump, + + pub fn deinit(self: *ExtensionHandler) void { + switch (self.*) { + .s3_dump => |*h| h.deinit(), + } + } +}; + +/// Owns the enabled handlers and adapts them to policy_zig's fn-pointer +/// seams (`ExtensionResolver` at snapshot compile time, `ExtensionSink` at +/// evaluate time). Encoders are wired per signal at construction because the +/// record types behind `*const anyopaque` are the consumer's. +/// +/// Usage: +/// var exts = Extensions.init(allocator, .{ .log = encodeLog }); +/// defer exts.deinit(); +/// exts.enable(.{ .s3_dump = S3Dump.init(allocator, .{}, creds) }); +/// try exts.s3Dump().?.addTarget(io, "eu-bucket", target_json); +/// registry.setExtensionResolver(exts.resolver()); +/// // per evaluate call: .{ .io = io, .extension_sink = exts.sink() } +/// // periodically: exts.flush(io, .{}); +pub const Extensions = struct { + pub const Encoders = struct { + log: ?EncodeFn = null, + metric: ?EncodeFn = null, + trace: ?EncodeFn = null, + }; + + allocator: std.mem.Allocator, + handlers: std.EnumArray(ExtensionTag, ?ExtensionHandler), + encoders: Encoders, + + pub fn init(allocator: std.mem.Allocator, encoders: Encoders) Extensions { + return .{ + .allocator = allocator, + .handlers = .initFill(null), + .encoders = encoders, + }; + } + + pub fn deinit(self: *Extensions) void { + defer self.* = undefined; + var it = self.handlers.iterator(); + while (it.next()) |entry| { + if (entry.value.*) |*handler| handler.deinit(); + } + } + + /// Enable a handler. Takes ownership; replaces (and deinits) any handler + /// already enabled for the same tag. + pub fn enable(self: *Extensions, handler: ExtensionHandler) void { + const tag = std.meta.activeTag(handler); + if (self.handlers.getPtr(tag).*) |*old| old.deinit(); + self.handlers.set(tag, handler); + } + + pub fn s3Dump(self: *Extensions) ?*S3Dump { + if (self.handlers.getPtr(.s3_dump).*) |*handler| return &handler.s3_dump; + return null; + } + + // ========================================================================= + // policy_zig seams + // ========================================================================= + + pub fn resolver(self: *Extensions) policy.ExtensionResolver { + return .{ .ctx = self, .resolve = resolveFn }; + } + + pub fn sink(self: *Extensions) policy.ExtensionSink { + return .{ .ctx = self, .deliver = deliverFn }; + } + + fn resolveFn( + io: std.Io, + ctx: *anyopaque, + signal: policy.TelemetryType, + policy_id: []const u8, + extension: *const proto.policy.Extension, + ) ?policy.ExtensionResolution { + const self: *Extensions = @ptrCast(@alignCast(ctx)); + const tag = ExtensionTag.fromTypeId(extension.type) orelse return null; + const version = std.SemanticVersion.parse(extension.version) catch return null; + if (version.major != tag.supportedMajor()) return null; + + switch (tag) { + .s3_dump => { + const handler = self.s3Dump() orelse return null; + const slot = handler.resolve(io, signal, policy_id, extension.config) orelse return null; + return .{ .handler = @intFromEnum(tag), .slot = slot }; + }, + } + } + + fn deliverFn( + ctx: *anyopaque, + io: ?std.Io, + signal: policy.TelemetryType, + record: *const anyopaque, + binding: *const policy.ExtensionBinding, + slice: policy.ExtensionSlice, + ) void { + _ = slice; // mode selection already applied by the engine + const self: *Extensions = @ptrCast(@alignCast(ctx)); + const encode = switch (signal) { + .log => self.encoders.log, + .metric => self.encoders.metric, + .trace => self.encoders.trace, + } orelse return; + const tag = std.enums.fromInt(ExtensionTag, binding.handler) orelse return; + switch (tag) { + .s3_dump => if (self.s3Dump()) |handler| { + handler.deliver(io, binding.slot, record, encode); + }, + } + } + + // ========================================================================= + // Sync plumbing helpers + // ========================================================================= + + /// Adapter for HttpProvider.setExtensionSyncHooks: advertises + /// capabilities in sync requests and routes broadcast extension configs + /// from sync responses. + pub fn syncHooks(self: *Extensions) policy.HttpProvider.ExtensionSyncHooks { + return .{ + .ctx = self, + .capabilities = capabilitiesHook, + .apply_configs = applyConfigsHook, + }; + } + + fn capabilitiesHook( + io: std.Io, + ctx: *anyopaque, + arena: std.mem.Allocator, + ) anyerror![]proto.policy.ExtensionCapability { + const self: *Extensions = @ptrCast(@alignCast(ctx)); + return self.supportedExtensions(io, arena); + } + + fn applyConfigsHook( + io: std.Io, + ctx: *anyopaque, + configs: []const proto.policy.ExtensionConfig, + ) void { + const self: *Extensions = @ptrCast(@alignCast(ctx)); + self.applyExtensionConfigs(io, configs); + } + + /// Route `SyncResponse.extension_configs` entries to their handlers. + pub fn applyExtensionConfigs(self: *Extensions, io: std.Io, configs: []const proto.policy.ExtensionConfig) void { + for (configs) |config| { + const tag = ExtensionTag.fromTypeId(config.type) orelse continue; + switch (tag) { + .s3_dump => if (self.s3Dump()) |handler| { + handler.configure(io, config.config.items); + }, + } + } + } + + /// Build `ClientMetadata.supported_extensions` for the enabled handlers. + /// All memory (including descriptor bytes) is allocated with `arena` — + /// pass an arena allocator and free it wholesale. + pub fn supportedExtensions(self: *Extensions, io: std.Io, arena: std.mem.Allocator) ![]proto.policy.ExtensionCapability { + var out: std.ArrayList(proto.policy.ExtensionCapability) = .empty; + var it = self.handlers.iterator(); + while (it.next()) |entry| { + if (entry.value.* == null) continue; + const descriptors: []const []const u8 = switch (entry.key) { + .s3_dump => try self.s3Dump().?.capabilities(io, arena), + }; + var config_list: std.ArrayList([]const u8) = .empty; + try config_list.appendSlice(arena, descriptors); + try out.append(arena, .{ + .type = entry.key.typeId(), + .min_version = "1.0.0", + .config = config_list, + }); + } + return out.toOwnedSlice(arena); + } + + /// Flush all handlers that buffer for upload. Forward to the consumer's + /// background loop. + pub fn flush(self: *Extensions, io: std.Io, opts: S3Dump.FlushOptions) S3Dump.FlushResult { + if (self.s3Dump()) |handler| return handler.flush(io, opts); + return .{}; + } +}; + +test { + _ = @import("./s3_dump.zig"); +} + +// ============================================================================= +// Integration test: engine → resolver → sink → batch +// ============================================================================= + +const testing = std.testing; +const test_io = std.Options.debug_io; + +const TestLogRecord = struct { + body: []const u8, + + fn typedValue(ctx: *const anyopaque, field: policy.FieldRef) ?policy.TypedValue { + const self: *const TestLogRecord = @ptrCast(@alignCast(ctx)); + return switch (field) { + .log_field => |lf| if (lf == .LOG_FIELD_BODY) .{ .string = self.body } else null, + else => null, + }; + } + + const accessor: policy.LogAccessor = .{ .typed_value = typedValue }; + + fn encode(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + const self: *const TestLogRecord = @ptrCast(@alignCast(record)); + try writer.print("{{\"body\":\"{s}\"}}", .{self.body}); + } +}; + +fn encodeTargetRefAlloc(allocator: std.mem.Allocator, kind: []const u8, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = kind, .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +test "end to end: dropped records land in the s3-dump batch as ndjson" { + const allocator = testing.allocator; + const o11y = policy.observability; + + var exts = Extensions.init(allocator, .{ .log = TestLogRecord.encode }); + defer exts.deinit(); + exts.enable(.{ .s3_dump = S3Dump.init(allocator, .{}, null) }); + try exts.s3Dump().?.addTarget( + test_io, + "eu-bucket", + \\{"bucket": "waste", "region": "eu-west-1"} + , + ); + + // Policy: drop payment logs, dump the waste to eu-bucket. + const ref_bytes = try encodeTargetRefAlloc(allocator, "s3", "eu-bucket"); + defer allocator.free(ref_bytes); + var pol: proto.policy.Policy = .{ + .id = try allocator.dupe(u8, "drop-payments"), + .name = try allocator.dupe(u8, "drop-payments"), + .enabled = true, + .target = .{ .log = .{ .keep = try allocator.dupe(u8, "none") } }, + }; + try pol.target.?.log.match.append(allocator, .{ + .field = .{ .log_field = .LOG_FIELD_BODY }, + .match = .{ .regex = try allocator.dupe(u8, "payment") }, + }); + try pol.extensions.append(allocator, .{ + .type = try allocator.dupe(u8, "com.usetero/s3-dump"), + .version = try allocator.dupe(u8, "1.0.0"), + .config = try allocator.dupe(u8, ref_bytes), + .mode = .EXTENSION_MODE_DROPPED, + }); + defer pol.deinit(allocator); + + var noop_bus: o11y.NoopEventBus = undefined; + noop_bus.init(test_io); + var registry = policy.Registry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + registry.setExtensionResolver(exts.resolver()); + try registry.updatePolicies(&.{pol}, "test", .file); + + const engine = policy.PolicyEngine.init(noop_bus.eventBus(), ®istry); + var policy_id_buf: [16][]const u8 = undefined; + + var dropped_record: TestLogRecord = .{ .body = "payment failed" }; + const r1 = engine.evaluate(.log, &TestLogRecord.accessor, &dropped_record, &policy_id_buf, .{ + .io = test_io, + .extension_sink = exts.sink(), + }); + try testing.expectEqual(policy.FilterDecision.drop, r1.decision); + + var kept_record: TestLogRecord = .{ .body = "healthcheck" }; + _ = engine.evaluate(.log, &TestLogRecord.accessor, &kept_record, &policy_id_buf, .{ + .io = test_io, + .extension_sink = exts.sink(), + }); + + // Only the dropped record was batched, encoded by the consumer encoder. + const dump = exts.s3Dump().?; + try testing.expectEqual(@as(usize, 1), dump.batches.items.len); + try testing.expectEqualStrings( + "{\"body\":\"payment failed\"}\n", + dump.batches.items[0].buf.items, + ); + + // Capability advertisement covers the configured target. + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const caps = try exts.supportedExtensions(test_io, arena.allocator()); + try testing.expectEqual(@as(usize, 1), caps.len); + try testing.expectEqualStrings("com.usetero/s3-dump", caps[0].type); + try testing.expectEqual(@as(usize, 1), caps[0].config.items.len); +} + +test "Extensions: applyExtensionConfigs routes broadcast targets by type" { + const allocator = testing.allocator; + var exts = Extensions.init(allocator, .{}); + defer exts.deinit(); + exts.enable(.{ .s3_dump = S3Dump.init(allocator, .{}, null) }); + + var aw = std.Io.Writer.Allocating.init(allocator); + defer aw.deinit(); + const target: proto.policy.ExtensionTarget = .{ + .kind = "s3", + .name = "bcast-bucket", + .config = "{\"bucket\": \"waste\"}", + }; + try target.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + const target_bytes = try list.toOwnedSlice(allocator); + defer allocator.free(target_bytes); + + var entries: std.ArrayList([]const u8) = .empty; + defer entries.deinit(allocator); + try entries.append(allocator, target_bytes); + + // A config for an unknown type is ignored; the s3-dump one lands. + const configs = [_]proto.policy.ExtensionConfig{ + .{ .type = "vendor/unknown", .config = entries }, + .{ .type = "com.usetero/s3-dump", .config = entries }, + }; + exts.applyExtensionConfigs(test_io, &configs); + + try testing.expectEqual(@as(usize, 1), exts.s3Dump().?.targets.items.len); + try testing.expectEqualStrings("bcast-bucket", exts.s3Dump().?.targets.items[0].name); +} diff --git a/src/extensions/s3_dump.zig b/src/extensions/s3_dump.zig new file mode 100644 index 0000000..375fd6b --- /dev/null +++ b/src/extensions/s3_dump.zig @@ -0,0 +1,594 @@ +//! `com.usetero/s3-dump` extension handler (policy spec v1.6.0). +//! +//! Routes records selected by a policy extension's mode to a pre-configured +//! S3-compatible destination as batched ndjson objects. See +//! design/extensions.md for the full design. +//! +//! Invariants: +//! - `deliver` copies (encodes) synchronously and holds no reference to the +//! record after it returns; the hot path never performs network I/O. +//! - A destination outage costs waste records (counted drops), never pipeline +//! latency: the sealed-backlog cap is checked before encoding. +//! - I/O happens only in consumer-driven `flush(io)`, per this repo's rule +//! that `std.Io` is passed per call and never stored. + +const std = @import("std"); +const policy = @import("policy_zig"); +const proto = @import("proto"); +const s3 = @import("s3"); + +/// Renders one record to bytes (one ndjson line, no trailing newline). +/// Wired by the consumer per signal — the record type is the consumer's. +pub const EncodeFn = *const fn (record: *const anyopaque, writer: *std.Io.Writer) anyerror!void; + +pub const S3Dump = struct { + /// All knobs, as plain data with defaults so a consumer can deserialize + /// them straight from a config file (e.g. std.json) later. + pub const Options = struct { + /// Open batch is sealed for upload once it reaches this size. + max_batch_bytes: usize = 4 << 20, + /// Open batch is sealed for upload once it holds this many records. + max_batch_records: u32 = 10_000, + /// flush() seals a non-empty open batch older than this. + max_batch_age_ms: u64 = 30_000, + /// Cap on sealed-but-not-uploaded bytes; past it new records drop. + max_sealed_bytes: usize = 32 << 20, + /// Per-request attempts passed to the S3 client (its built-in retry). + max_attempts: usize = 3, + content_type: []const u8 = "application/x-ndjson", + }; + + /// Static credentials, consumer-provided (e.g. read from env or a + /// credentials file by the edge binary). Never carried in policies or + /// targets. Null → uploads are counted as failed and batches dropped. + pub const Credentials = struct { + access_key_id: []const u8, + secret_access_key: []const u8, + }; + + /// Kind-defined config carried in `ExtensionTarget.config` for kind + /// "s3", authored as JSON. + pub const TargetConfig = struct { + endpoint: ?[]const u8 = null, + region: []const u8 = "us-east-1", + bucket: []const u8, + prefix: []const u8 = "", + force_path_style: bool = true, + }; + + const Target = struct { + name: []const u8, + parsed: std.json.Parsed(TargetConfig), + }; + + /// One open batch per (target, signal, policy). Created at resolve time + /// (snapshot compile) so `deliver` is an array index, never a hash or + /// string compare. Batches persist across snapshot rebuilds so pending + /// data survives policy updates. + const Batch = struct { + target: u16, + signal: policy.TelemetryType, + policy_id: []const u8, + buf: std.ArrayList(u8) = .empty, + records: u32 = 0, + /// Monotonic (.awake) time of the first record, for age sealing. + first_ns: i128 = 0, + }; + + const Sealed = struct { + target: u16, + signal: policy.TelemetryType, + /// Borrowed from the owning Batch (batches live until deinit). + policy_id: []const u8, + buf: []u8, + records: u32, + }; + + pub const FlushOptions = struct { + /// Seal and upload all open batches regardless of age (shutdown). + force: bool = false, + }; + + pub const FlushResult = struct { + objects_uploaded: u32 = 0, + objects_failed: u32 = 0, + records_uploaded: u64 = 0, + records_failed: u64 = 0, + /// Records dropped at delivery time (backlog full, encode failure, + /// null io) since the previous flush. + records_dropped: u64 = 0, + }; + + allocator: std.mem.Allocator, + options: Options, + credentials: ?Credentials, + mutex: std.Io.Mutex = .init, + targets: std.ArrayList(Target) = .empty, + batches: std.ArrayList(*Batch) = .empty, + sealed: std.ArrayList(Sealed) = .empty, + sealed_bytes: usize = 0, + /// Reusable encode buffer, guarded by `mutex`. + scratch: std.ArrayList(u8) = .empty, + records_dropped: std.atomic.Value(u64) = .init(0), + object_seq: std.atomic.Value(u64) = .init(0), + + pub fn init(allocator: std.mem.Allocator, options: Options, credentials: ?Credentials) S3Dump { + return .{ + .allocator = allocator, + .options = options, + .credentials = credentials, + }; + } + + pub fn deinit(self: *S3Dump) void { + defer self.* = undefined; + for (self.batches.items) |batch| { + batch.buf.deinit(self.allocator); + self.allocator.free(batch.policy_id); + self.allocator.destroy(batch); + } + self.batches.deinit(self.allocator); + for (self.sealed.items) |s| self.allocator.free(s.buf); + self.sealed.deinit(self.allocator); + for (self.targets.items) |*t| { + self.allocator.free(t.name); + t.parsed.deinit(); + } + self.targets.deinit(self.allocator); + self.scratch.deinit(self.allocator); + } + + // ========================================================================= + // Target configuration + // ========================================================================= + + /// Add or replace a named s3 target. `config_json` is the kind-defined + /// JSON TargetConfig. Used for local (out-of-band) configuration; the + /// broadcast path (`configure`) lands here too, so broadcast wins on a + /// name collision. + pub fn addTarget(self: *S3Dump, io: std.Io, name: []const u8, config_json: []const u8) !void { + const parsed = try std.json.parseFromSlice(TargetConfig, self.allocator, config_json, .{ + .ignore_unknown_fields = true, + .allocate = .alloc_always, + }); + errdefer parsed.deinit(); + const name_copy = try self.allocator.dupe(u8, name); + errdefer self.allocator.free(name_copy); + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + for (self.targets.items) |*t| { + if (std.mem.eql(u8, t.name, name)) { + self.allocator.free(t.name); + t.parsed.deinit(); + t.* = .{ .name = name_copy, .parsed = parsed }; + return; + } + } + try self.targets.append(self.allocator, .{ .name = name_copy, .parsed = parsed }); + } + + /// Ingest broadcast target entries (`SyncResponse.extension_configs` for + /// this type): each entry is a serialized ExtensionTarget whose `config` + /// is JSON TargetConfig. Non-s3 kinds and malformed entries are skipped + /// (fail-open). + pub fn configure(self: *S3Dump, io: std.Io, entries: []const []const u8) void { + for (entries) |bytes| { + var reader = std.Io.Reader.fixed(bytes); + var target = proto.policy.ExtensionTarget.decode(&reader, self.allocator) catch continue; + defer target.deinit(self.allocator); + if (!std.mem.eql(u8, target.kind, "s3")) continue; + self.addTarget(io, target.name, target.config) catch continue; + } + } + + /// One serialized ExtensionTargetRef per configured target — the + /// capability descriptors for `ClientMetadata.supported_extensions`. + /// Caller owns the returned slices (allocated with `allocator`). + pub fn capabilities(self: *S3Dump, io: std.Io, allocator: std.mem.Allocator) ![]const []const u8 { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + var out: std.ArrayList([]const u8) = .empty; + errdefer { + for (out.items) |item| allocator.free(item); + out.deinit(allocator); + } + for (self.targets.items) |t| { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = "s3", .name = t.name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + try out.append(allocator, try list.toOwnedSlice(allocator)); + } + return try out.toOwnedSlice(allocator); + } + + // ========================================================================= + // Resolution (snapshot compile time, cold) + // ========================================================================= + + /// Resolve one policy extension config (a serialized ExtensionTargetRef) + /// to a batch slot. Null → unsupported/invalid, the caller skips the + /// extension fail-open. The same (target, signal, policy) triple always + /// resolves to the same slot, so batches survive snapshot rebuilds. + pub fn resolve( + self: *S3Dump, + io: std.Io, + signal: policy.TelemetryType, + policy_id: []const u8, + config: []const u8, + ) ?u32 { + var reader = std.Io.Reader.fixed(config); + var ref = proto.policy.ExtensionTargetRef.decode(&reader, self.allocator) catch return null; + defer ref.deinit(self.allocator); + if (!std.mem.eql(u8, ref.kind, "s3")) return null; + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + const target_index: u16 = blk: { + for (self.targets.items, 0..) |t, i| { + if (std.mem.eql(u8, t.name, ref.name)) break :blk @intCast(i); + } + return null; + }; + + for (self.batches.items, 0..) |batch, i| { + if (batch.target == target_index and batch.signal == signal and + std.mem.eql(u8, batch.policy_id, policy_id)) + { + return @intCast(i); + } + } + + const batch = self.allocator.create(Batch) catch return null; + const id_copy = self.allocator.dupe(u8, policy_id) catch { + self.allocator.destroy(batch); + return null; + }; + batch.* = .{ .target = target_index, .signal = signal, .policy_id = id_copy }; + self.batches.append(self.allocator, batch) catch { + self.allocator.free(id_copy); + self.allocator.destroy(batch); + return null; + }; + return @intCast(self.batches.items.len - 1); + } + + // ========================================================================= + // Delivery (hot path — encode + append only, never I/O) + // ========================================================================= + + pub fn deliver( + self: *S3Dump, + io_opt: ?std.Io, + slot: u32, + record: *const anyopaque, + encode: EncodeFn, + ) void { + const io = io_opt orelse return self.drop(); + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + // Stale/foreign slot: fail-open. + if (slot >= self.batches.items.len) return self.drop(); + const batch = self.batches.items[slot]; + + // Seal a full batch first — and check the sealed-backlog cap BEFORE + // encoding, so a destination outage degrades to a counter increment. + const must_seal = batch.buf.items.len >= self.options.max_batch_bytes or + batch.records >= self.options.max_batch_records; + if (must_seal) { + if (self.sealed_bytes + batch.buf.items.len > self.options.max_sealed_bytes) { + return self.drop(); + } + self.sealLocked(batch) catch return self.drop(); + } + + // Encode into the reusable scratch buffer so a failed encode never + // corrupts the batch. + self.scratch.clearRetainingCapacity(); + var aw = std.Io.Writer.Allocating.fromArrayList(self.allocator, &self.scratch); + const encode_err = encode(record, &aw.writer); + self.scratch = aw.toArrayList(); + encode_err catch return self.drop(); + + const line = self.scratch.items; + if (line.len + 1 > self.options.max_batch_bytes) return self.drop(); + + batch.buf.ensureUnusedCapacity(self.allocator, line.len + 1) catch return self.drop(); + batch.buf.appendSliceAssumeCapacity(line); + batch.buf.appendAssumeCapacity('\n'); + batch.records += 1; + if (batch.first_ns == 0) { + batch.first_ns = std.Io.Timestamp.now(io, .awake).nanoseconds; + } + } + + fn drop(self: *S3Dump) void { + _ = self.records_dropped.fetchAdd(1, .monotonic); + } + + /// Move the open batch's buffer onto the sealed list and reset the batch. + fn sealLocked(self: *S3Dump, batch: *Batch) !void { + if (batch.buf.items.len == 0) return; + const buf = try batch.buf.toOwnedSlice(self.allocator); + errdefer self.allocator.free(buf); + try self.sealed.append(self.allocator, .{ + .target = batch.target, + .signal = batch.signal, + .policy_id = batch.policy_id, + .buf = buf, + .records = batch.records, + }); + self.sealed_bytes += buf.len; + batch.records = 0; + batch.first_ns = 0; + } + + // ========================================================================= + // Flush (consumer-driven I/O) + // ========================================================================= + + /// Seal over-age (or, with `force`, all) open batches and upload every + /// sealed batch as one object. Failed uploads are counted and their + /// batches dropped — no retry queue by design (design/extensions.md). + pub fn flush(self: *S3Dump, io: std.Io, opts: FlushOptions) FlushResult { + var result: FlushResult = .{}; + + const now: i128 = std.Io.Timestamp.now(io, .awake).nanoseconds; + const max_age_ns: i128 = @as(i128, self.options.max_batch_age_ms) * std.time.ns_per_ms; + + self.mutex.lockUncancelable(io); + for (self.batches.items) |batch| { + if (batch.buf.items.len == 0) continue; + const expired = batch.first_ns != 0 and now - batch.first_ns >= max_age_ns; + if (opts.force or expired) { + // OOM here just leaves the batch for the next flush. + self.sealLocked(batch) catch {}; + } + } + const to_upload = self.sealed.toOwnedSlice(self.allocator) catch { + self.mutex.unlock(io); + result.records_dropped = self.records_dropped.swap(0, .monotonic); + return result; + }; + self.sealed_bytes = 0; + self.mutex.unlock(io); + defer self.allocator.free(to_upload); + + for (to_upload) |*sealed_batch| { + defer self.allocator.free(sealed_batch.buf); + if (self.uploadOne(io, sealed_batch)) { + result.objects_uploaded += 1; + result.records_uploaded += sealed_batch.records; + } else { + result.objects_failed += 1; + result.records_failed += sealed_batch.records; + } + } + + result.records_dropped = self.records_dropped.swap(0, .monotonic); + return result; + } + + fn uploadOne(self: *S3Dump, io: std.Io, sealed_batch: *const Sealed) bool { + const creds = self.credentials orelse return false; + // Target config is index-stable: targets are only appended or + // replaced in place, never removed. + const cfg = self.targets.items[sealed_batch.target].parsed.value; + + // ponytail: one client (connection) per object; reuse a per-target + // client within a flush if profiles show connection setup mattering. + var client = s3.S3Client.init(self.allocator, .{ + .access_key_id = creds.access_key_id, + .secret_access_key = creds.secret_access_key, + .region = cfg.region, + .endpoint = cfg.endpoint, + .virtual_host_style = !cfg.force_path_style, + }, .{ .io = io }) catch return false; + defer client.deinit(); + + const key = self.buildKey(io, cfg.prefix, sealed_batch) catch return false; + defer self.allocator.free(key); + + var response = client.putObject(cfg.bucket, key, sealed_batch.buf, .{ + .content_type = self.options.content_type, + .request = .{ .max_attempts = self.options.max_attempts }, + }) catch return false; + defer response.deinit(); + return response.http_head.status.class() == .success; + } + + /// `{prefix}{signal}/{yyyy}/{mm}/{dd}/{hh}/{policy_id}-{unix_nanos}-{seq}.ndjson` + fn buildKey(self: *S3Dump, io: std.Io, prefix: []const u8, sealed_batch: *const Sealed) ![]u8 { + const wall_ns: i128 = std.Io.Timestamp.now(io, .real).nanoseconds; + const secs: u64 = @intCast(@divTrunc(wall_ns, std.time.ns_per_s)); + const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = secs }; + const year_day = epoch_secs.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const seq = self.object_seq.fetchAdd(1, .monotonic); + return std.fmt.allocPrint( + self.allocator, + "{s}{s}/{d:0>4}/{d:0>2}/{d:0>2}/{d:0>2}/{s}-{d}-{d}.ndjson", + .{ + prefix, + @tagName(sealed_batch.signal), + year_day.year, + month_day.month.numeric(), + @as(u32, month_day.day_index) + 1, + epoch_secs.getDaySeconds().getHoursIntoDay(), + sealed_batch.policy_id, + wall_ns, + seq, + }, + ); + } +}; + +// ============================================================================= +// Tests +// ============================================================================= + +const testing = std.testing; +const test_io = std.Options.debug_io; + +fn encodeTargetRef(allocator: std.mem.Allocator, kind: []const u8, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = kind, .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +const target_json = + \\{"endpoint": "http://127.0.0.1:1", "region": "eu-west-1", "bucket": "waste", "prefix": "dumps/"} +; + +fn testEncodeRecord(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + const msg: *const []const u8 = @ptrCast(@alignCast(record)); + try writer.writeAll(msg.*); +} + +test "S3Dump: resolve validates kind and target, slots are stable" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const good = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(good); + const bad_kind = try encodeTargetRef(testing.allocator, "gcs", "eu-bucket"); + defer testing.allocator.free(bad_kind); + const bad_name = try encodeTargetRef(testing.allocator, "s3", "nope"); + defer testing.allocator.free(bad_name); + + try testing.expect(dump.resolve(test_io, .log, "p1", bad_kind) == null); + try testing.expect(dump.resolve(test_io, .log, "p1", bad_name) == null); + + const slot = dump.resolve(test_io, .log, "p1", good).?; + // Same triple → same slot (snapshot rebuilds keep their batches). + try testing.expectEqual(slot, dump.resolve(test_io, .log, "p1", good).?); + // Different policy or signal → different slot. + try testing.expect(dump.resolve(test_io, .log, "p2", good).? != slot); + try testing.expect(dump.resolve(test_io, .trace, "p1", good).? != slot); +} + +test "S3Dump: deliver batches records as ndjson lines" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec1: []const u8 = "{\"body\":\"one\"}"; + const rec2: []const u8 = "{\"body\":\"two\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec1), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec2), testEncodeRecord); + + const batch = dump.batches.items[slot]; + try testing.expectEqual(@as(u32, 2), batch.records); + try testing.expectEqualStrings("{\"body\":\"one\"}\n{\"body\":\"two\"}\n", batch.buf.items); + try testing.expectEqual(@as(u64, 0), dump.records_dropped.load(.monotonic)); +} + +test "S3Dump: batch seals at record limit; backlog cap drops instead of growing" { + var dump = S3Dump.init(testing.allocator, .{ + .max_batch_records = 2, + .max_sealed_bytes = 40, + }, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "0123456789"; // 11 bytes per line with \n + // Two records fill the batch; the third seals it (22 bytes ≤ 40) and + // starts a new one. + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + try testing.expectEqual(@as(usize, 1), dump.sealed.items.len); + try testing.expectEqual(@as(u32, 2), dump.sealed.items[0].records); + try testing.expectEqual(@as(u32, 1), dump.batches.items[slot].records); + + // Fill and roll again: sealing would put the backlog at 44 > 40, so the + // record that forces the seal is dropped before encoding. + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + try testing.expectEqual(@as(u64, 1), dump.records_dropped.load(.monotonic)); + try testing.expectEqual(@as(usize, 1), dump.sealed.items.len); +} + +test "S3Dump: flush without credentials fails the batch, counts it, and resets state" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + + const result = dump.flush(test_io, .{ .force = true }); + try testing.expectEqual(@as(u32, 0), result.objects_uploaded); + try testing.expectEqual(@as(u32, 1), result.objects_failed); + try testing.expectEqual(@as(u64, 1), result.records_failed); + try testing.expectEqual(@as(usize, 0), dump.sealed.items.len); + try testing.expectEqual(@as(usize, 0), dump.sealed_bytes); + try testing.expectEqual(@as(usize, 0), dump.batches.items[slot].buf.items.len); +} + +test "S3Dump: non-forced flush leaves young batches open" { + var dump = S3Dump.init(testing.allocator, .{ .max_batch_age_ms = 60_000 }, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + + const result = dump.flush(test_io, .{}); + try testing.expectEqual(@as(u32, 0), result.objects_uploaded + result.objects_failed); + try testing.expectEqual(@as(u32, 1), dump.batches.items[slot].records); +} + +test "S3Dump: configure ingests broadcast targets; capabilities round-trips refs" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + + // Broadcast one s3 target and one foreign-kind target (skipped). + var aw = std.Io.Writer.Allocating.init(testing.allocator); + defer aw.deinit(); + const bcast: proto.policy.ExtensionTarget = .{ .kind = "s3", .name = "us-bucket", .config = target_json }; + try bcast.encode(&aw.writer, testing.allocator); + var list = aw.toArrayList(); + const bcast_bytes = try list.toOwnedSlice(testing.allocator); + defer testing.allocator.free(bcast_bytes); + + dump.configure(test_io, &.{ bcast_bytes, "not-a-proto" }); + try testing.expectEqual(@as(usize, 1), dump.targets.items.len); + + const caps = try dump.capabilities(test_io, testing.allocator); + defer { + for (caps) |c| testing.allocator.free(c); + testing.allocator.free(caps); + } + try testing.expectEqual(@as(usize, 1), caps.len); + var reader = std.Io.Reader.fixed(caps[0]); + var ref = try proto.policy.ExtensionTargetRef.decode(&reader, testing.allocator); + defer ref.deinit(testing.allocator); + try testing.expectEqualStrings("s3", ref.kind); + try testing.expectEqualStrings("us-bucket", ref.name); +} From b06b85c4beecbd103d70c5269380d05743e99585 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Thu, 9 Jul 2026 11:39:11 -0400 Subject: [PATCH 6/8] feat: introduce S3 dump capabilities --- README.md | 78 +++++-- Taskfile.yml | 72 +++++++ build.zig | 42 ++++ design/extensions.md | 91 ++++++-- src/extensions/root.zig | 89 +++++++- src/extensions/s3_dump.zig | 389 +++++++++++++++++++++++++++++------ src/policy/provider.zig | 27 +++ src/policy/provider_http.zig | 26 +-- src/policy/registry.zig | 24 ++- src/policy/root.zig | 1 + src/policy/types.zig | 12 ++ 11 files changed, 729 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 6447dbb..75eb0c7 100644 --- a/README.md +++ b/README.md @@ -167,31 +167,77 @@ Per policy-spec conformance item 9, this implementation supports the following extension `type`/`version` pairs (see `design/extensions.md` and the `extensions` module): -| type | versions | notes | -| --------------------- | -------- | -------------------------------------------------- | -| `com.usetero/s3-dump` | 1.0.x | batched ndjson objects via z3 S3 client, no retry queue | +| type | versions | notes | +| --------------------- | -------- | ------------------------------------------------------------------------------------------ | +| `com.usetero/s3-dump` | 1.0.x | batched ndjson objects via z3; failed uploads retried in-memory (bounded), idempotent keys | Extensions declared by a policy but not supported (or with an unresolvable target) are skipped fail-open and reported via `PolicySyncStatus.errors`; the policy's core match/keep/transform still applies. +Enabling extensions in a consumer (the `extensions` module owns the handlers +and adapts them to policy_zig's seams): + +```zig +const ext = @import("extensions"); + +// One Extensions object owns every enabled handler. Encoders are per-signal +// because the engine hands the sink an opaque record it can't serialize. +var exts = ext.Extensions.init(allocator, .{ .log = encodeLog }); +defer exts.deinit(); + +// Enable s3-dump and configure targets directly on the returned handle. +// Credentials come from the consumer (env/secrets) — never from policies. +const s3 = exts.enableS3Dump(allocator, .{}, .{ + .access_key_id = key, + .secret_access_key = secret, +}); +try s3.addTarget(io, "eu-bucket", + \\{"bucket": "tero-waste", "region": "eu-west-1", "prefix": "dumps/"} +); + +// One call wires the resolver (snapshot compile) + sync hooks (advertised on +// each provider). subscribe() then pushes the hooks to every provider — the +// HTTP provider advertises capabilities and receives broadcast targets; a +// file provider accepts and ignores them. +exts.register(®istry); +try registry.subscribe(.{ .http = http_provider }); + +// Per evaluate: pass the sink so selected records are delivered. +_ = engine.evaluate(.log, &accessor, &record, &buf, .{ + .io = io, + .extension_sink = exts.sink(), +}); + +// From a background loop: flush uploads and read the result as metrics. +const r = exts.flush(io, .{}); +// r.objects_uploaded / .records_dropped / .backlog_bytes → your metrics client +``` + +`com.usetero/s3-dump`'s upload path is verified two ways: `zig build test` +(covered by `task test`) includes a hermetic test against an in-process HTTP +stub — no network or docker needed — and `task test:s3-e2e` spins up a real +MinIO container via docker and round-trips an upload through it. + ## Task Commands This project uses [Task](https://taskfile.dev/) for common operations: -| Command | Description | -| -------------------- | ------------------------------------------------ | -| `task` | Build (debug) | -| `task test` | Run all tests | -| `task build:release` | Build with ReleaseFast | -| `task build:safe` | Build with ReleaseSafe | -| `task format` | Format source files | -| `task format:check` | Check formatting | -| `task lint` | Run linting checks | -| `task clean` | Clean build artifacts | -| `task do` | Run all pre-commit checks (format + lint + test) | -| `task signoff` | Full signoff (do + build:safe + gh signoff) | -| `task ci:setup` | Install CI/dev dependencies | +| Command | Description | +| -------------------- | --------------------------------------------------------------- | +| `task` | Build (debug) | +| `task test` | Run all tests | +| `task test:s3-e2e` | Real-storage s3-dump smoke test against MinIO (requires docker) | +| `task bench:s3` | s3-dump scale benchmarks against MinIO (requires docker) | +| `task build:release` | Build with ReleaseFast | +| `task build:safe` | Build with ReleaseSafe | +| `task format` | Format source files | +| `task format:check` | Check formatting | +| `task lint` | Run linting checks | +| `task clean` | Clean build artifacts | +| `task do` | Run all pre-commit checks (format + lint + test) | +| `task signoff` | Full signoff (do + build:safe + gh signoff) | +| `task ci:setup` | Install CI/dev dependencies | ## License diff --git a/Taskfile.yml b/Taskfile.yml index 97db170..5f57bb5 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -71,6 +71,78 @@ tasks: cmds: - zig build test --summary all + test:s3-e2e: + desc: "Real-storage smoke test for the s3-dump extension against a local MinIO container (requires docker)" + cmds: + - | + if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for task test:s3-e2e" + exit 1 + fi + CONTAINER=policy-zig-minio-e2e + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + docker run -d --rm --name "$CONTAINER" \ + -p 9000:9000 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data >/dev/null + trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1' EXIT + + echo "Waiting for MinIO..." + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; then + break + fi + if [ "$i" = 30 ]; then + echo "MinIO did not become healthy in time" + docker logs "$CONTAINER" || true + exit 1 + fi + sleep 1 + done + + AWS_ACCESS_KEY_ID=minioadmin \ + AWS_SECRET_ACCESS_KEY=minioadmin \ + S3_ENDPOINT=http://127.0.0.1:9000 \ + S3_BUCKET=policy-zig-e2e \ + zig build test-s3-e2e --summary all + + bench:s3: + desc: "s3-dump scale benchmarks against a local MinIO container (requires docker)" + cmds: + - | + if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for task bench:s3" + exit 1 + fi + CONTAINER=policy-zig-minio-bench + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + docker run -d --rm --name "$CONTAINER" \ + -p 9000:9000 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data >/dev/null + trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1' EXIT + + echo "Waiting for MinIO..." + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; then + break + fi + if [ "$i" = 30 ]; then + echo "MinIO did not become healthy in time" + docker logs "$CONTAINER" || true + exit 1 + fi + sleep 1 + done + + AWS_ACCESS_KEY_ID=minioadmin \ + AWS_SECRET_ACCESS_KEY=minioadmin \ + S3_ENDPOINT=http://127.0.0.1:9000 \ + S3_BUCKET=policy-zig-bench \ + zig build bench-s3 + format: desc: "Format all Zig source files" aliases: [fmt, f] diff --git a/build.zig b/build.zig index 2e959fd..263fd28 100644 --- a/build.zig +++ b/build.zig @@ -89,6 +89,48 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_o11y_tests.step); test_step.dependOn(&run_ext_tests.step); + // Real-storage s3-dump smoke test (needs a running S3-compatible + // server — see `task test:s3-e2e`). Deliberately NOT part of `test`: + // it requires network I/O and external state, unlike every other test. + const s3_e2e_tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("src/extensions/s3_minio_test.zig"), + .target = target, + .imports = &.{ + .{ .name = "proto", .module = proto_mod }, + .{ .name = "policy_zig", .module = mod }, + .{ .name = "s3", .module = z3_dep.module("s3") }, + }, + }), + }); + s3_e2e_tests.root_module.link_libc = true; + s3_e2e_tests.root_module.linkSystemLibrary("hs", .{}); + const run_s3_e2e_tests = b.addRunArtifact(s3_e2e_tests); + const s3_e2e_step = b.step("test-s3-e2e", "Run the s3-dump smoke test against a real S3-compatible server"); + s3_e2e_step.dependOn(&run_s3_e2e_tests.step); + + // s3-dump scale benchmarks (needs MinIO — see `task bench:s3`). + // ReleaseFast like the other benchmarks; excluded from `test`. A plain + // executable, not a test: it reports numbers, not pass/fail. + const s3_bench = b.addExecutable(.{ + .name = "bench-s3", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/extensions/s3_bench.zig"), + .target = target, + .optimize = .ReleaseFast, + .imports = &.{ + .{ .name = "proto", .module = proto_mod }, + .{ .name = "policy_zig", .module = mod }, + .{ .name = "s3", .module = z3_dep.module("s3") }, + }, + }), + }); + s3_bench.root_module.link_libc = true; + s3_bench.root_module.linkSystemLibrary("hs", .{}); + const run_s3_bench = b.addRunArtifact(s3_bench); + const s3_bench_step = b.step("bench-s3", "Run s3-dump scale benchmarks against a real S3-compatible server"); + s3_bench_step.dependOn(&run_s3_bench.step); + // Benchmarks const zbench_dep = b.dependency("zbench", .{ .target = target, .optimize = .ReleaseFast }); const zbench_mod = zbench_dep.module("zbench"); diff --git a/design/extensions.md b/design/extensions.md index b03c043..7101df9 100644 --- a/design/extensions.md +++ b/design/extensions.md @@ -16,8 +16,16 @@ Deviations from the text below, made during implementation: `(signal, record ctx)` across the module seam). - Credentials are consumer-provided at `S3Dump.init` (no built-in env discovery — z3 has none either, and the edge binary owns its env). -- `provider_http` gains `setExtensionSyncHooks` (a two-fn seam like - StatsCollector) for capability advertisement + config broadcast routing. +- Extension sync hooks (a two-fn seam like `StatsCollector`) live in the + generic provider layer (`provider.zig`); `Provider.setExtensionSyncHooks` + dispatches per variant (`.http` wires, `.file`/`.testing` no-op), and + `registry.subscribe` pushes them to every provider automatically — the + consumer wires them once via `registry.setExtensionSyncHooks` (or + `Extensions.register`), not per provider. +- `Extensions.enableS3Dump(alloc, opts, creds) *S3Dump` constructs the + handler in place and returns its pointer, so targets are configured on the + handler directly (no move, no `s3Dump().?` reach-back). +- `Extensions.register(®istry)` wires resolver + sync hooks in one call. - All batching knobs live in `S3Dump.Options`, a plain-data struct with defaults, deserializable from a config file. @@ -433,12 +441,21 @@ One object per batch: available (z3's no-chunked-signing limitation doesn't bite us). - z3 does not do credential discovery — fine, because our design already sources credentials from the consumer callback / env (see above) and passes - them at client init. One z3 client per target, created lazily at first - flush. -- Failure handling v1: non-2xx or transport error → count it, drop the batch, - log via event bus. No retry queue (the data is by definition the waste - stream; better to lose a batch than to grow unbounded state). - + them at client init. One z3 client per target per flush (lazily created), + so std.http.Client's connection pool is reused across that flush's uploads. +- Target configs are snapshotted into a flush-local arena under the mutex; + uploads never read live target state (a concurrent broadcast `configure` + can therefore never invalidate memory mid-upload). +- Failure handling: z3's `max_attempts` only retries connection + establishment — send/receive failures and 5xx responses are not retried by + the client. So a failed upload is **requeued** onto the sealed backlog and + retried on subsequent flushes, bounded by the same `max_sealed_bytes` cap + (overflow drops, counted). The object key is fixed at seal time, so + retries are idempotent same-key PUTs — a request that succeeded + server-side before the failure surfaced overwrites itself instead of + duplicating. Records are lost only when the backlog cap overflows; there + is still no unbounded queue and no disk spill. + ### Capability advertisement @@ -446,6 +463,44 @@ One object per batch: per configured target, so the provider only sends policies whose extensions this client can satisfy. +## Scale characteristics (measured) + +`task bench:s3` runs scale benchmarks against a local MinIO container +(ReleaseFast, 191-byte records, Apple Silicon dev machine — treat as shape, +not absolutes): + +| scenario | result | +| --- | --- | +| deliver, 1 thread | ~16M records/s (~2.9 GiB/s) — ~62ns/record | +| deliver, 4 threads | ~11.5M records/s aggregate — **slower than 1 thread** | +| flush, 64KiB objects | ~250–320 objects/s, ~16–20 MiB/s | +| flush, 1MiB objects | ~100 objects/s, ~100 MiB/s | +| flush, 4MiB objects | ~76–80 objects/s, ~135–142 MiB/s | +| fan-out, 64 policies × 128KiB | ~260 objects/s (one flush ≈ 0.25s) | +| outage flush, max_attempts=1 | ~1ms (connect-refused + requeue is free) | +| outage flush, max_attempts=3 | ~2.3–2.8s for 8 objects (~290ms/object of retry sleeps) | + +What this says about scale: + +1. **The hot path is not the bottleneck.** Deliver costs ~62ns/record — + comparable to a whole engine evaluation (~40–60ns) and paid only by + records a mode selects. +2. **The single dump-wide mutex is a real ceiling under concurrent + delivery**: 4 threads are ~30% *slower* in aggregate than 1. Irrelevant at + realistic waste rates (1M records/s is ~6% of the ceiling), but per-batch + mutexes are the upgrade path if a many-threaded consumer attaches + `mode: all` extensions to high-volume signals. +3. **Object size dominates upload throughput** (~3–4ms fixed cost per PUT: + SigV4 + HTTP round trip). 64KiB objects move ~16 MiB/s; 4MiB objects move + ~140 MiB/s. Keep `max_batch_bytes` ≥ 1MiB and avoid fragmenting batches + across very many (policy × signal × target) combinations. Uploads within + a flush are serial; hundreds of batches per flush cycle means seconds of + flush latency. +4. **z3's in-request connect retries are redundant with our requeue** and + serialize ~100ms sleeps per object into the flush during outages — + hence `max_attempts` defaults to 1: a failed batch just waits for the + next flush instead of stalling the current one. + ## Conformance note README gains a "Supported extensions" table — required by conformance item 9: @@ -466,10 +521,22 @@ README gains a "Supported extensions" table — required by conformance item 9: semantics). - **Validation**: unknown type / bad version / unknown target → policy still active, extension skipped, error string in `PolicySyncStatus.errors`. -- **Upload**: flush against a local HTTP stub asserting method, path, auth - header presence, and ndjson body (signing correctness itself is z3's tested - responsibility); sealed-backlog drop counting; missing-credentials no-op. -- **End-to-end (manual)**: MinIO via the `verify` flow. +- **Upload (hermetic)**: `src/extensions/s3_stub_test.zig` runs flush against + an in-process `std.http.Server` stub over a real socket — asserts method, + path-style URL/key layout, the SigV4 auth header, the payload-sha256 header + S3 uses for integrity verification, and exact ndjson body bytes; a second + test scripts a 500 response and asserts the batch is requeued under the + identical object key and the retry succeeds. Runs in every + `zig build test`, no docker or network required. +- **Upload (real backend)**: `src/extensions/s3_minio_test.zig`, built as the + separate `zig build test-s3-e2e` step (excluded from `test` — needs a live + server) and driven end-to-end by `task test:s3-e2e`, which starts a MinIO + container via docker, waits for its health check, runs the test, and tears + the container down (`trap ... EXIT`). Skips itself (`error.SkipZigTest`) if + the AWS_* / S3_ENDPOINT env vars aren't set, so it's inert when built + without the task. Verifies against the real backend's own APIs — lists the + uploaded prefix, fetches the object, and diffs its body byte-for-byte — + rather than trusting our own response parsing. ## Rollout diff --git a/src/extensions/root.zig b/src/extensions/root.zig index 2ac2275..1912133 100644 --- a/src/extensions/root.zig +++ b/src/extensions/root.zig @@ -13,13 +13,14 @@ const proto = @import("proto"); const s3_dump_mod = @import("./s3_dump.zig"); pub const S3Dump = s3_dump_mod.S3Dump; pub const EncodeFn = s3_dump_mod.EncodeFn; +pub const TeroS3DumpTag = "com.usetero/s3-dump"; pub const ExtensionTag = enum(u8) { s3_dump, pub fn typeId(tag: ExtensionTag) []const u8 { return switch (tag) { - .s3_dump => "com.usetero/s3-dump", + .s3_dump => TeroS3DumpTag, }; } @@ -57,9 +58,10 @@ pub const ExtensionHandler = union(ExtensionTag) { /// Usage: /// var exts = Extensions.init(allocator, .{ .log = encodeLog }); /// defer exts.deinit(); -/// exts.enable(.{ .s3_dump = S3Dump.init(allocator, .{}, creds) }); -/// try exts.s3Dump().?.addTarget(io, "eu-bucket", target_json); -/// registry.setExtensionResolver(exts.resolver()); +/// const s3 = exts.enableS3Dump(allocator, .{}, creds); +/// try s3.addTarget(io, "eu-bucket", target_json); +/// exts.register(®istry); // resolver + sync hooks, once +/// try registry.subscribe(.{ .http = http_provider }); // hooks auto-pushed /// // per evaluate call: .{ .io = io, .extension_sink = exts.sink() } /// // periodically: exts.flush(io, .{}); pub const Extensions = struct { @@ -97,11 +99,36 @@ pub const Extensions = struct { self.handlers.set(tag, handler); } + /// Enable the s3-dump handler, constructing it directly in place, and + /// return a stable pointer so targets are configured on the handler + /// itself — no move, no `s3Dump().?` reach-back: + /// const s3 = exts.enableS3Dump(allocator, .{}, creds); + /// try s3.addTarget(io, "eu-bucket", target_json); + /// The pointer is valid for the life of this `Extensions` (don't move it). + pub fn enableS3Dump( + self: *Extensions, + allocator: std.mem.Allocator, + options: S3Dump.Options, + credentials: ?S3Dump.Credentials, + ) *S3Dump { + self.enable(.{ .s3_dump = S3Dump.init(allocator, options, credentials) }); + return self.s3Dump().?; + } + pub fn s3Dump(self: *Extensions) ?*S3Dump { if (self.handlers.getPtr(.s3_dump).*) |*handler| return &handler.s3_dump; return null; } + /// Wire this Extensions into a registry in one call: the resolver (used + /// at snapshot compile) and the sync hooks (pushed to providers on + /// subscribe). Call before `registry.subscribe(...)`. The per-evaluate + /// `sink()` and the background `flush()` are still the consumer's to wire. + pub fn register(self: *Extensions, reg: *policy.Registry) void { + reg.setExtensionResolver(self.resolver()); + reg.setExtensionSyncHooks(self.syncHooks()); + } + // ========================================================================= // policy_zig seams // ========================================================================= @@ -165,7 +192,7 @@ pub const Extensions = struct { /// Adapter for HttpProvider.setExtensionSyncHooks: advertises /// capabilities in sync requests and routes broadcast extension configs /// from sync responses. - pub fn syncHooks(self: *Extensions) policy.HttpProvider.ExtensionSyncHooks { + pub fn syncHooks(self: *Extensions) policy.ExtensionSyncHooks { return .{ .ctx = self, .capabilities = capabilitiesHook, @@ -235,6 +262,7 @@ pub const Extensions = struct { test { _ = @import("./s3_dump.zig"); + _ = @import("./s3_stub_test.zig"); } // ============================================================================= @@ -278,8 +306,9 @@ test "end to end: dropped records land in the s3-dump batch as ndjson" { var exts = Extensions.init(allocator, .{ .log = TestLogRecord.encode }); defer exts.deinit(); - exts.enable(.{ .s3_dump = S3Dump.init(allocator, .{}, null) }); - try exts.s3Dump().?.addTarget( + // Configure the target directly on the handler pointer — no reach-back. + const s3 = exts.enableS3Dump(allocator, .{}, null); + try s3.addTarget( test_io, "eu-bucket", \\{"bucket": "waste", "region": "eu-west-1"} @@ -311,7 +340,7 @@ test "end to end: dropped records land in the s3-dump batch as ndjson" { noop_bus.init(test_io); var registry = policy.Registry.init(allocator, noop_bus.eventBus()); defer registry.deinit(); - registry.setExtensionResolver(exts.resolver()); + exts.register(®istry); // resolver + sync hooks in one call try registry.updatePolicies(&.{pol}, "test", .file); const engine = policy.PolicyEngine.init(noop_bus.eventBus(), ®istry); @@ -343,7 +372,7 @@ test "end to end: dropped records land in the s3-dump batch as ndjson" { defer arena.deinit(); const caps = try exts.supportedExtensions(test_io, arena.allocator()); try testing.expectEqual(@as(usize, 1), caps.len); - try testing.expectEqualStrings("com.usetero/s3-dump", caps[0].type); + try testing.expectEqualStrings(TeroS3DumpTag, caps[0].type); try testing.expectEqual(@as(usize, 1), caps[0].config.items.len); } @@ -372,10 +401,50 @@ test "Extensions: applyExtensionConfigs routes broadcast targets by type" { // A config for an unknown type is ignored; the s3-dump one lands. const configs = [_]proto.policy.ExtensionConfig{ .{ .type = "vendor/unknown", .config = entries }, - .{ .type = "com.usetero/s3-dump", .config = entries }, + .{ .type = TeroS3DumpTag, .config = entries }, }; exts.applyExtensionConfigs(test_io, &configs); try testing.expectEqual(@as(usize, 1), exts.s3Dump().?.targets.items.len); try testing.expectEqualStrings("bcast-bucket", exts.s3Dump().?.targets.items[0].name); } + +test "Extensions.register wires resolver + sync hooks; subscribe pushes hooks per provider" { + const allocator = testing.allocator; + const o11y = policy.observability; + + var exts = Extensions.init(allocator, .{ .log = TestLogRecord.encode }); + defer exts.deinit(); + _ = exts.enableS3Dump(allocator, .{}, null); + + var noop_bus: o11y.NoopEventBus = undefined; + noop_bus.init(test_io); + var registry = policy.Registry.init(allocator, noop_bus.eventBus()); + defer registry.deinit(); + + // One call wires both seams onto the registry. + exts.register(®istry); + try testing.expect(registry.extension_resolver != null); + try testing.expect(registry.extension_sync_hooks != null); + + // A file provider accepts the hooks and ignores them (no control plane): + // the union dispatch must not crash and leaves nothing to observe. + const file_provider = try policy.FileProvider.init(allocator, test_io, noop_bus.eventBus(), .{ + .id = "local", + .path = "does-not-exist.json", + }); + defer file_provider.deinit(); + const file_prov: policy.Provider = .{ .file = file_provider }; + file_prov.setExtensionSyncHooks(registry.extension_sync_hooks.?); + + // An HTTP provider actually stores them — this is what `subscribe` pushes. + const http_provider = try policy.HttpProvider.init(allocator, test_io, noop_bus.eventBus(), .{ + .id = "control-plane", + .url = "http://127.0.0.1:1/policies", + }); + defer http_provider.deinit(); + try testing.expect(http_provider.extension_hooks == null); + const http_prov: policy.Provider = .{ .http = http_provider }; + http_prov.setExtensionSyncHooks(registry.extension_sync_hooks.?); + try testing.expect(http_provider.extension_hooks != null); +} diff --git a/src/extensions/s3_dump.zig b/src/extensions/s3_dump.zig index 375fd6b..1dd3d11 100644 --- a/src/extensions/s3_dump.zig +++ b/src/extensions/s3_dump.zig @@ -7,8 +7,11 @@ //! Invariants: //! - `deliver` copies (encodes) synchronously and holds no reference to the //! record after it returns; the hot path never performs network I/O. -//! - A destination outage costs waste records (counted drops), never pipeline -//! latency: the sealed-backlog cap is checked before encoding. +//! - A destination outage costs pipeline latency never: the sealed-backlog +//! cap bounds all buffered memory, and past it records drop (counted). +//! - Failed uploads are requeued (within the same cap) and retried on later +//! flushes under the SAME object key, so retries are idempotent. Records +//! are lost only when the backlog cap overflows, never silently. //! - I/O happens only in consumer-driven `flush(io)`, per this repo's rule //! that `std.Io` is passed per call and never stored. @@ -33,8 +36,14 @@ pub const S3Dump = struct { max_batch_age_ms: u64 = 30_000, /// Cap on sealed-but-not-uploaded bytes; past it new records drop. max_sealed_bytes: usize = 32 << 20, - /// Per-request attempts passed to the S3 client (its built-in retry). - max_attempts: usize = 3, + /// Per-request attempts passed to the S3 client. The client only + /// retries connection establishment, sleeping ~100ms between tries + /// serialized per object — measured at ~290ms/object of added flush + /// latency during an outage (`task bench:s3`). Since failed batches + /// are requeued and retried on the next flush anyway, the default + /// keeps flush fast; raise it only if the next-flush wait is too + /// coarse for transient connect blips. + max_attempts: usize = 1, content_type: []const u8 = "application/x-ndjson", }; @@ -80,23 +89,63 @@ pub const S3Dump = struct { signal: policy.TelemetryType, /// Borrowed from the owning Batch (batches live until deinit). policy_id: []const u8, + /// Object key, fixed at seal time so retried uploads are idempotent + /// (same key, identical content). Owned. + key: []u8, buf: []u8, records: u32, }; + /// Flush-local copy of a target's connection config, duped into the + /// flush arena under the mutex. Uploads read only snapshots, so a + /// concurrent addTarget/configure (provider sync thread) can never + /// invalidate memory mid-upload. + const TargetSnapshot = struct { + bucket: []const u8, + region: []const u8, + endpoint: ?[]const u8, + virtual_host_style: bool, + }; + + /// PUT responses are tiny (empty or an XML error); never buffer more. + const max_response_body: usize = 64 * 1024; + pub const FlushOptions = struct { /// Seal and upload all open batches regardless of age (shutdown). force: bool = false, }; + /// Everything a consumer needs to turn one `flush()` call into metrics: + /// counters (deltas since the previous flush) plus one gauge + /// (`backlog_bytes`, a point-in-time snapshot, not a delta) for + /// early-warning before the `max_sealed_bytes` cap starts dropping data. + /// The library emits no metrics itself — this struct is the seam; the + /// edge decides where counts and gauge go (OTel, Prometheus, statsd, ...). pub const FlushResult = struct { objects_uploaded: u32 = 0, + /// Objects that failed to upload this flush. Unless the backlog cap + /// forced a drop they are requeued and retried on the next flush. objects_failed: u32 = 0, + /// Of the failed objects, how many were requeued for retry. + objects_requeued: u32 = 0, records_uploaded: u64 = 0, records_failed: u64 = 0, - /// Records dropped at delivery time (backlog full, encode failure, - /// null io) since the previous flush. + /// Records dropped since the previous flush: at delivery time + /// (backlog full, encode failure, null io) or because a failed batch + /// could not be requeued within the backlog cap. records_dropped: u64 = 0, + /// Object body bytes successfully PUT this flush. Feeds a + /// throughput/cost counter (S3 PUT + egress pricing scales with this). + bytes_uploaded: u64 = 0, + /// Object body bytes in batches that failed to upload this flush + /// (whether requeued or dropped). + bytes_failed: u64 = 0, + /// GAUGE, not a delta: sealed-but-not-yet-uploaded bytes remaining + /// after this flush (open batches are excluded — they aren't at risk + /// of the drop cliff yet). Alert on this approaching + /// `Options.max_sealed_bytes`: that's the leading indicator of the + /// destination falling behind, before `records_dropped` climbs. + backlog_bytes: usize = 0, }; allocator: std.mem.Allocator, @@ -107,8 +156,6 @@ pub const S3Dump = struct { batches: std.ArrayList(*Batch) = .empty, sealed: std.ArrayList(Sealed) = .empty, sealed_bytes: usize = 0, - /// Reusable encode buffer, guarded by `mutex`. - scratch: std.ArrayList(u8) = .empty, records_dropped: std.atomic.Value(u64) = .init(0), object_seq: std.atomic.Value(u64) = .init(0), @@ -128,14 +175,16 @@ pub const S3Dump = struct { self.allocator.destroy(batch); } self.batches.deinit(self.allocator); - for (self.sealed.items) |s| self.allocator.free(s.buf); + for (self.sealed.items) |s| { + self.allocator.free(s.buf); + self.allocator.free(s.key); + } self.sealed.deinit(self.allocator); for (self.targets.items) |*t| { self.allocator.free(t.name); t.parsed.deinit(); } self.targets.deinit(self.allocator); - self.scratch.deinit(self.allocator); } // ========================================================================= @@ -284,23 +333,34 @@ pub const S3Dump = struct { if (self.sealed_bytes + batch.buf.items.len > self.options.max_sealed_bytes) { return self.drop(); } - self.sealLocked(batch) catch return self.drop(); + self.sealLocked(io, batch) catch return self.drop(); } - // Encode into the reusable scratch buffer so a failed encode never - // corrupts the batch. - self.scratch.clearRetainingCapacity(); - var aw = std.Io.Writer.Allocating.fromArrayList(self.allocator, &self.scratch); - const encode_err = encode(record, &aw.writer); - self.scratch = aw.toArrayList(); - encode_err catch return self.drop(); - - const line = self.scratch.items; - if (line.len + 1 > self.options.max_batch_bytes) return self.drop(); - - batch.buf.ensureUnusedCapacity(self.allocator, line.len + 1) catch return self.drop(); - batch.buf.appendSliceAssumeCapacity(line); - batch.buf.appendAssumeCapacity('\n'); + // Encode directly into the batch buffer — one write, no intermediate + // copy. A failed or oversized encode is rolled back by truncating to + // the pre-record length, so the batch is never corrupted. + const len_before = batch.buf.items.len; + var aw = std.Io.Writer.Allocating.fromArrayList(self.allocator, &batch.buf); + var write_failed = false; + encode(record, &aw.writer) catch { + write_failed = true; + }; + if (!write_failed) aw.writer.writeByte('\n') catch { + write_failed = true; + }; + batch.buf = aw.toArrayList(); + if (write_failed or batch.buf.items.len - len_before > self.options.max_batch_bytes) { + // A rogue oversized record may have grown the buffer far past + // anything a legal batch needs; release that memory rather than + // pinning it on this slot forever. Legal batches never need more + // than ~2x max_batch_bytes (seal-before-encode + one record). + if (batch.buf.capacity > self.options.max_batch_bytes * 2) { + batch.buf.shrinkAndFree(self.allocator, len_before); + } else { + batch.buf.shrinkRetainingCapacity(len_before); + } + return self.drop(); + } batch.records += 1; if (batch.first_ns == 0) { batch.first_ns = std.Io.Timestamp.now(io, .awake).nanoseconds; @@ -311,15 +371,23 @@ pub const S3Dump = struct { _ = self.records_dropped.fetchAdd(1, .monotonic); } - /// Move the open batch's buffer onto the sealed list and reset the batch. - fn sealLocked(self: *S3Dump, batch: *Batch) !void { + /// Move the open batch's buffer onto the sealed list and reset the + /// batch. The object key is fixed here, at seal time: a batch that fails + /// to upload retries under the SAME key, so a PUT that actually + /// succeeded server-side before the failure surfaced is overwritten with + /// identical content instead of duplicated under a second key. + fn sealLocked(self: *S3Dump, io: std.Io, batch: *Batch) !void { if (batch.buf.items.len == 0) return; + const prefix = self.targets.items[batch.target].parsed.value.prefix; + const key = try self.buildKey(io, prefix, batch.signal, batch.policy_id); + errdefer self.allocator.free(key); const buf = try batch.buf.toOwnedSlice(self.allocator); errdefer self.allocator.free(buf); try self.sealed.append(self.allocator, .{ .target = batch.target, .signal = batch.signal, .policy_id = batch.policy_id, + .key = key, .buf = buf, .records = batch.records, }); @@ -328,26 +396,58 @@ pub const S3Dump = struct { batch.first_ns = 0; } + /// Requeue a failed batch for the next flush, bounded by the same + /// backlog cap as delivery-time sealing. Returns false when the cap (or + /// OOM) forced a drop instead. + fn requeue(self: *S3Dump, io: std.Io, sealed_batch: Sealed) bool { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + const fits = self.sealed_bytes + sealed_batch.buf.len <= self.options.max_sealed_bytes; + if (fits) { + if (self.sealed.append(self.allocator, sealed_batch)) |_| { + self.sealed_bytes += sealed_batch.buf.len; + return true; + } else |_| {} + } + _ = self.records_dropped.fetchAdd(sealed_batch.records, .monotonic); + self.allocator.free(sealed_batch.buf); + self.allocator.free(sealed_batch.key); + return false; + } + // ========================================================================= // Flush (consumer-driven I/O) // ========================================================================= /// Seal over-age (or, with `force`, all) open batches and upload every - /// sealed batch as one object. Failed uploads are counted and their - /// batches dropped — no retry queue by design (design/extensions.md). + /// sealed batch as one object. A failed upload is requeued for the next + /// flush (bounded by max_sealed_bytes) rather than dropped — z3 only + /// retries connection establishment, not send/receive failures or 5xx + /// responses, so handler-level retry is what makes a transient + /// destination outage lossless. pub fn flush(self: *S3Dump, io: std.Io, opts: FlushOptions) FlushResult { var result: FlushResult = .{}; const now: i128 = std.Io.Timestamp.now(io, .awake).nanoseconds; const max_age_ns: i128 = @as(i128, self.options.max_batch_age_ms) * std.time.ns_per_ms; + var arena_state = std.heap.ArenaAllocator.init(self.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var no_snapshots = [0]?TargetSnapshot{}; + var no_clients = [0]?s3.S3Client{}; + + // Phase 1 (locked): seal due batches, steal the sealed list, and + // snapshot target configs into the flush arena. Uploads read only + // the snapshots, never `self.targets`. self.mutex.lockUncancelable(io); for (self.batches.items) |batch| { if (batch.buf.items.len == 0) continue; const expired = batch.first_ns != 0 and now - batch.first_ns >= max_age_ns; if (opts.force or expired) { - // OOM here just leaves the batch for the next flush. - self.sealLocked(batch) catch {}; + // OOM here just leaves the batch open for the next flush. + self.sealLocked(io, batch) catch {}; } } const to_upload = self.sealed.toOwnedSlice(self.allocator) catch { @@ -356,54 +456,130 @@ pub const S3Dump = struct { return result; }; self.sealed_bytes = 0; + const snapshots: []?TargetSnapshot = + arena.alloc(?TargetSnapshot, self.targets.items.len) catch no_snapshots[0..]; + for (self.targets.items, 0..) |t, i| { + if (i >= snapshots.len) break; + snapshots[i] = snapshotTarget(arena, t.parsed.value) catch null; + } self.mutex.unlock(io); defer self.allocator.free(to_upload); - for (to_upload) |*sealed_batch| { - defer self.allocator.free(sealed_batch.buf); - if (self.uploadOne(io, sealed_batch)) { + // Phase 2 (unlocked): create one client per target used this flush, + // then upload. Client setup is a distinct step from the PUT — one + // client per target so the std.http.Client connection pool is reused + // across all of this flush's uploads to that target. + const clients = self.makeClients(io, arena, snapshots, to_upload, no_clients[0..]); + defer for (clients) |*c| { + if (c.*) |*client| client.deinit(); + }; + + for (to_upload) |sealed_batch| { + const uploaded = ok: { + const client = clientFor(clients, sealed_batch.target) orelse break :ok false; + break :ok self.uploadOne(client, snapshots[sealed_batch.target].?, &sealed_batch); + }; + if (uploaded) { result.objects_uploaded += 1; result.records_uploaded += sealed_batch.records; + result.bytes_uploaded += sealed_batch.buf.len; + self.allocator.free(sealed_batch.buf); + self.allocator.free(sealed_batch.key); } else { result.objects_failed += 1; result.records_failed += sealed_batch.records; + result.bytes_failed += sealed_batch.buf.len; + if (self.requeue(io, sealed_batch)) result.objects_requeued += 1; } } result.records_dropped = self.records_dropped.swap(0, .monotonic); + // Gauge read AFTER requeue has put failed-but-kept batches back onto + // the backlog, so it reflects what's actually still queued. + // `sealed_bytes` is plain state guarded by `mutex` (like the rest of + // this struct), not an atomic — take the lock rather than tear it. + self.mutex.lockUncancelable(io); + result.backlog_bytes = self.sealed_bytes; + self.mutex.unlock(io); return result; } - fn uploadOne(self: *S3Dump, io: std.Io, sealed_batch: *const Sealed) bool { - const creds = self.credentials orelse return false; - // Target config is index-stable: targets are only appended or - // replaced in place, never removed. - const cfg = self.targets.items[sealed_batch.target].parsed.value; - - // ponytail: one client (connection) per object; reuse a per-target - // client within a flush if profiles show connection setup mattering. - var client = s3.S3Client.init(self.allocator, .{ - .access_key_id = creds.access_key_id, - .secret_access_key = creds.secret_access_key, - .region = cfg.region, - .endpoint = cfg.endpoint, + fn snapshotTarget(arena: std.mem.Allocator, cfg: TargetConfig) !TargetSnapshot { + return .{ + .bucket = try arena.dupe(u8, cfg.bucket), + .region = try arena.dupe(u8, cfg.region), + .endpoint = if (cfg.endpoint) |e| try arena.dupe(u8, e) else null, .virtual_host_style = !cfg.force_path_style, - }, .{ .io = io }) catch return false; - defer client.deinit(); + }; + } - const key = self.buildKey(io, cfg.prefix, sealed_batch) catch return false; - defer self.allocator.free(key); + /// Create one S3 client per target that has a batch to upload this flush + /// (deduped — several batches for one target share a client, hence one + /// connection pool). Indexed by target, parallel to `snapshots`. A target + /// with no credentials, no snapshot, or a failed client init is left null; + /// its batches then fail the upload and requeue. Clients are owned by the + /// caller (freed via the flush's `defer`), allocated in the flush arena. + fn makeClients( + self: *S3Dump, + io: std.Io, + arena: std.mem.Allocator, + snapshots: []const ?TargetSnapshot, + to_upload: []const Sealed, + fallback: []?s3.S3Client, + ) []?s3.S3Client { + const clients = arena.alloc(?s3.S3Client, snapshots.len) catch return fallback; + for (clients) |*c| c.* = null; + + const creds = self.credentials orelse return clients; // no creds → no clients + for (to_upload) |sealed_batch| { + const target = sealed_batch.target; + if (target >= clients.len or clients[target] != null) continue; + const snap = snapshots[target] orelse continue; + clients[target] = s3.S3Client.init(self.allocator, .{ + .access_key_id = creds.access_key_id, + .secret_access_key = creds.secret_access_key, + .region = snap.region, + .endpoint = snap.endpoint, + .virtual_host_style = snap.virtual_host_style, + }, .{ .io = io }) catch continue; + } + return clients; + } + + /// The client for a target, or null if none was created for it (out of + /// range, or setup skipped it — see `makeClients`). + fn clientFor(clients: []?s3.S3Client, target: u16) ?*s3.S3Client { + if (target >= clients.len) return null; + return if (clients[target]) |*c| c else null; + } - var response = client.putObject(cfg.bucket, key, sealed_batch.buf, .{ + /// PUT one sealed batch as a single object. No client lifecycle here — the + /// client is created up front by `makeClients`. + fn uploadOne( + self: *S3Dump, + client: *s3.S3Client, + snap: TargetSnapshot, + sealed_batch: *const Sealed, + ) bool { + var response = client.putObject(snap.bucket, sealed_batch.key, sealed_batch.buf, .{ .content_type = self.options.content_type, - .request = .{ .max_attempts = self.options.max_attempts }, + .request = .{ + .max_attempts = self.options.max_attempts, + .max_body_size = max_response_body, + }, }) catch return false; defer response.deinit(); return response.http_head.status.class() == .success; } /// `{prefix}{signal}/{yyyy}/{mm}/{dd}/{hh}/{policy_id}-{unix_nanos}-{seq}.ndjson` - fn buildKey(self: *S3Dump, io: std.Io, prefix: []const u8, sealed_batch: *const Sealed) ![]u8 { + fn buildKey( + self: *S3Dump, + io: std.Io, + prefix: []const u8, + signal: policy.TelemetryType, + policy_id: []const u8, + ) ![]u8 { const wall_ns: i128 = std.Io.Timestamp.now(io, .real).nanoseconds; const secs: u64 = @intCast(@divTrunc(wall_ns, std.time.ns_per_s)); const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = secs }; @@ -415,12 +591,12 @@ pub const S3Dump = struct { "{s}{s}/{d:0>4}/{d:0>2}/{d:0>2}/{d:0>2}/{s}-{d}-{d}.ndjson", .{ prefix, - @tagName(sealed_batch.signal), + @tagName(signal), year_day.year, month_day.month.numeric(), @as(u32, month_day.day_index) + 1, epoch_secs.getDaySeconds().getHoursIntoDay(), - sealed_batch.policy_id, + policy_id, wall_ns, seq, }, @@ -496,6 +672,35 @@ test "S3Dump: deliver batches records as ndjson lines" { try testing.expectEqual(@as(u64, 0), dump.records_dropped.load(.monotonic)); } +test "S3Dump: flush result carries byte counters and a backlog gauge" { + var dump = S3Dump.init(testing.allocator, .{}, null); // no credentials → upload fails + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; // 14 bytes + '\n' = 15 + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + + const result = dump.flush(test_io, .{ .force = true }); + + // Failed upload: bytes counted as failed, none as uploaded, and the + // gauge reflects the batch sitting in the backlog after requeue. + try testing.expectEqual(@as(u64, 30), result.bytes_failed); + try testing.expectEqual(@as(u64, 0), result.bytes_uploaded); + try testing.expectEqual(@as(usize, 30), result.backlog_bytes); + try testing.expectEqual(dump.sealed_bytes, result.backlog_bytes); + + // A flush with nothing to do reports a zero gauge, not the last value. + var empty_dump = S3Dump.init(testing.allocator, .{}, null); + defer empty_dump.deinit(); + const empty_result = empty_dump.flush(test_io, .{ .force = true }); + try testing.expectEqual(@as(usize, 0), empty_result.backlog_bytes); +} + test "S3Dump: batch seals at record limit; backlog cap drops instead of growing" { var dump = S3Dump.init(testing.allocator, .{ .max_batch_records = 2, @@ -526,7 +731,7 @@ test "S3Dump: batch seals at record limit; backlog cap drops instead of growing" try testing.expectEqual(@as(usize, 1), dump.sealed.items.len); } -test "S3Dump: flush without credentials fails the batch, counts it, and resets state" { +test "S3Dump: failed upload is requeued under a stable key, not dropped" { var dump = S3Dump.init(testing.allocator, .{}, null); defer dump.deinit(); try dump.addTarget(test_io, "eu-bucket", target_json); @@ -535,16 +740,80 @@ test "S3Dump: flush without credentials fails the batch, counts it, and resets s defer testing.allocator.free(cfg); const slot = dump.resolve(test_io, .log, "p1", cfg).?; + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + const batch_len = dump.batches.items[slot].buf.items.len; + + // No credentials → upload fails → the batch stays queued for retry. + const r1 = dump.flush(test_io, .{ .force = true }); + try testing.expectEqual(@as(u32, 0), r1.objects_uploaded); + try testing.expectEqual(@as(u32, 1), r1.objects_failed); + try testing.expectEqual(@as(u32, 1), r1.objects_requeued); + try testing.expectEqual(@as(u64, 1), r1.records_failed); + try testing.expectEqual(@as(u64, 0), r1.records_dropped); + try testing.expectEqual(@as(usize, 1), dump.sealed.items.len); + try testing.expectEqual(batch_len, dump.sealed_bytes); + try testing.expectEqual(@as(usize, 0), dump.batches.items[slot].buf.items.len); + + // The key was fixed at seal time and survives across retries, so a + // retried PUT is idempotent. Snapshot it, retry, compare. + const key_copy = try testing.allocator.dupe(u8, dump.sealed.items[0].key); + defer testing.allocator.free(key_copy); + try testing.expect(std.mem.startsWith(u8, key_copy, "dumps/log/")); + try testing.expect(std.mem.endsWith(u8, key_copy, ".ndjson")); + + const r2 = dump.flush(test_io, .{ .force = true }); + try testing.expectEqual(@as(u32, 1), r2.objects_failed); + try testing.expectEqual(@as(u32, 1), r2.objects_requeued); + try testing.expectEqual(@as(usize, 1), dump.sealed.items.len); + try testing.expectEqualStrings(key_copy, dump.sealed.items[0].key); +} + +test "S3Dump: requeue drops when the backlog cap would be exceeded" { + // Cap smaller than one batch: the failed upload cannot be requeued. + var dump = S3Dump.init(testing.allocator, .{ .max_sealed_bytes = 4 }, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + const rec: []const u8 = "{\"body\":\"one\"}"; dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); const result = dump.flush(test_io, .{ .force = true }); - try testing.expectEqual(@as(u32, 0), result.objects_uploaded); try testing.expectEqual(@as(u32, 1), result.objects_failed); - try testing.expectEqual(@as(u64, 1), result.records_failed); + try testing.expectEqual(@as(u32, 0), result.objects_requeued); + try testing.expectEqual(@as(u64, 1), result.records_dropped); try testing.expectEqual(@as(usize, 0), dump.sealed.items.len); try testing.expectEqual(@as(usize, 0), dump.sealed_bytes); - try testing.expectEqual(@as(usize, 0), dump.batches.items[slot].buf.items.len); +} + +fn testEncodePartialFail(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + _ = record; + try writer.writeAll("partial garbage"); + return error.EncodeFailed; +} + +test "S3Dump: failed encode rolls the batch back to its previous contents" { + var dump = S3Dump.init(testing.allocator, .{}, null); + defer dump.deinit(); + try dump.addTarget(test_io, "eu-bucket", target_json); + + const cfg = try encodeTargetRef(testing.allocator, "s3", "eu-bucket"); + defer testing.allocator.free(cfg); + const slot = dump.resolve(test_io, .log, "p1", cfg).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodeRecord); + // Partial write, then failure: everything it wrote must be truncated. + dump.deliver(test_io, slot, @ptrCast(&rec), testEncodePartialFail); + + const batch = dump.batches.items[slot]; + try testing.expectEqual(@as(u32, 1), batch.records); + try testing.expectEqualStrings("{\"body\":\"one\"}\n", batch.buf.items); + try testing.expectEqual(@as(u64, 1), dump.records_dropped.load(.monotonic)); } test "S3Dump: non-forced flush leaves young batches open" { diff --git a/src/policy/provider.zig b/src/policy/provider.zig index 45b5966..c0552a0 100644 --- a/src/policy/provider.zig +++ b/src/policy/provider.zig @@ -34,6 +34,33 @@ pub const StatsCollector = struct { } }; +/// Extension sync plumbing (spec v1.6.0), implemented outside policy_zig by +/// the extensions module: capability advertisement for sync requests and +/// routing of broadcast extension configs from responses. A fn-pointer seam +/// like `StatsCollector` — not a vtable of behaviors, just the two crossings. +/// +/// Handed to a provider by the registry at subscribe time, the same way +/// `StatsCollector` is. Providers with no control plane (file, testing) accept +/// the call and ignore it: there is nobody to advertise capabilities to and no +/// broadcast channel to receive from. +pub const ExtensionSyncHooks = struct { + ctx: *anyopaque, + /// Build ClientMetadata.supported_extensions. Allocate from `arena`. + capabilities: *const fn ( + io: std.Io, + ctx: *anyopaque, + arena: std.mem.Allocator, + ) anyerror![]proto.policy.ExtensionCapability, + /// Receive SyncResponse.extension_configs. Called before policies are + /// handed to the registry, so broadcast targets exist by the time the + /// snapshot compiles extension bindings against them. + apply_configs: *const fn ( + io: std.Io, + ctx: *anyopaque, + configs: []const proto.policy.ExtensionConfig, + ) void, +}; + /// Update notification sent by providers to subscribers pub const PolicyUpdate = struct { policies: []const Policy, diff --git a/src/policy/provider_http.zig b/src/policy/provider_http.zig index e7e872f..a335c68 100644 --- a/src/policy/provider_http.zig +++ b/src/policy/provider_http.zig @@ -42,27 +42,11 @@ const HttpSyncStatusReported = struct { id: []const u8, match_hits: i64, match_m const HttpFetchStarted = struct {}; const HttpFetchCompleted = struct {}; -/// Extension sync plumbing (spec v1.6.0), implemented outside policy_zig by -/// the extensions module: capability advertisement for sync requests and -/// routing of broadcast extension configs from responses. A fn-pointer seam -/// like StatsCollector — not a vtable of behaviors, just the two crossings. -pub const ExtensionSyncHooks = struct { - ctx: *anyopaque, - /// Build ClientMetadata.supported_extensions. Allocate from `arena`. - capabilities: *const fn ( - io: std.Io, - ctx: *anyopaque, - arena: std.mem.Allocator, - ) anyerror![]proto.policy.ExtensionCapability, - /// Receive SyncResponse.extension_configs. Called before policies are - /// handed to the registry, so broadcast targets exist by the time the - /// snapshot compiles extension bindings against them. - apply_configs: *const fn ( - io: std.Io, - ctx: *anyopaque, - configs: []const proto.policy.ExtensionConfig, - ) void, -}; +/// Extension sync plumbing lives in the generic provider layer now (it's +/// handed to every provider variant by the registry, like `StatsCollector`). +/// Re-exported here so existing `HttpProvider.ExtensionSyncHooks` references +/// keep resolving. +pub const ExtensionSyncHooks = policy_provider.ExtensionSyncHooks; /// HTTP-based policy provider that polls a remote endpoint pub const HttpProvider = struct { diff --git a/src/policy/registry.zig b/src/policy/registry.zig index 99fda78..e59d8be 100644 --- a/src/policy/registry.zig +++ b/src/policy/registry.zig @@ -254,6 +254,10 @@ pub const PolicyRegistry = struct { // time to resolve/validate each policy's extension declarations. extension_resolver: ?policy_types.ExtensionResolver, + // Optional extension sync hooks (v1.6.0), pushed to each provider on + // subscribe for capability advertisement + broadcast-config routing. + extension_sync_hooks: ?policy_provider.ExtensionSyncHooks, + /// Subscription context for provider callbacks. /// Allocated with stable address so the callback pointer remains valid. const Subscription = struct { @@ -289,6 +293,7 @@ pub const PolicyRegistry = struct { .subscriptions = .empty, .bus = bus, .extension_resolver = null, + .extension_sync_hooks = null, }; } @@ -300,10 +305,19 @@ pub const PolicyRegistry = struct { self.extension_resolver = resolver; } + /// Wire extension sync hooks (v1.6.0). Stored here and pushed to each + /// provider on `subscribe`, so the consumer wires them once instead of + /// per-provider. Call before `subscribe`; providers subscribed earlier + /// won't retroactively receive them (matches the stats-collector contract). + pub fn setExtensionSyncHooks(self: *PolicyRegistry, hooks: policy_provider.ExtensionSyncHooks) void { + self.extension_sync_hooks = hooks; + } + /// Subscribe a provider to the registry in one step: hand it our stats - /// collector (so it can pull per-policy hit/miss/error rows before each sync) - /// and subscribe to its policy updates with an internal callback that feeds - /// updates back into the registry. + /// collector (so it can pull per-policy hit/miss/error rows before each sync), + /// hand it the extension sync hooks if any are wired, and subscribe to its + /// policy updates with an internal callback that feeds updates back into the + /// registry. pub fn subscribe(self: *PolicyRegistry, prov: Provider) !void { const sub = try self.allocator.create(Subscription); errdefer self.allocator.destroy(sub); @@ -317,6 +331,10 @@ pub const PolicyRegistry = struct { .collect = collectStatsThunk, }); + if (self.extension_sync_hooks) |hooks| { + prov.setExtensionSyncHooks(hooks); + } + try prov.subscribe(.{ .context = @ptrCast(sub), .onUpdate = Subscription.handleUpdate, diff --git a/src/policy/root.zig b/src/policy/root.zig index 8434fbd..0bc1cb7 100644 --- a/src/policy/root.zig +++ b/src/policy/root.zig @@ -58,6 +58,7 @@ pub const PolicyMetadata = source.PolicyMetadata; pub const provider = @import("./provider.zig"); pub const PolicyCallback = provider.PolicyCallback; pub const PolicyUpdate = provider.PolicyUpdate; +pub const ExtensionSyncHooks = provider.ExtensionSyncHooks; /// Re-export registry pub const registry = @import("./registry.zig"); diff --git a/src/policy/types.zig b/src/policy/types.zig index 25b2894..f2463b9 100644 --- a/src/policy/types.zig +++ b/src/policy/types.zig @@ -675,6 +675,18 @@ pub const Provider = union(enum) { } } + /// Hand the provider extension sync hooks (v1.6.0): capability + /// advertisement in sync requests and routing of broadcast extension + /// configs from responses. Only the HTTP provider has a control plane; + /// file/testing accept the call and ignore it (nowhere to advertise to, no + /// broadcast to receive). Wired uniformly by `registry.subscribe`. + pub fn setExtensionSyncHooks(self: Provider, hooks: policy_provider.ExtensionSyncHooks) void { + switch (self) { + .http => |p| p.setExtensionSyncHooks(hooks), + .file, .testing => {}, + } + } + pub fn sourceType(self: Provider) SourceType { return switch (self) { .file => .file, From 0d2def3fa7a4d43abec101d571c8655aeaea42b4 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Thu, 9 Jul 2026 11:39:16 -0400 Subject: [PATCH 7/8] benchmarks --- src/extensions/s3_bench.zig | 284 +++++++++++++++++++++++++++++++ src/extensions/s3_minio_test.zig | 135 +++++++++++++++ src/extensions/s3_stub_test.zig | 262 ++++++++++++++++++++++++++++ 3 files changed, 681 insertions(+) create mode 100644 src/extensions/s3_bench.zig create mode 100644 src/extensions/s3_minio_test.zig create mode 100644 src/extensions/s3_stub_test.zig diff --git a/src/extensions/s3_bench.zig b/src/extensions/s3_bench.zig new file mode 100644 index 0000000..442847e --- /dev/null +++ b/src/extensions/s3_bench.zig @@ -0,0 +1,284 @@ +//! Scale benchmarks for the s3-dump extension against a real S3-compatible +//! backend (MinIO). Built as the `bench-s3` step (ReleaseFast executable, +//! excluded from `zig build test`) and driven by `task bench:s3`, which +//! starts MinIO in docker, runs this, and tears down. +//! +//! Scenarios, chosen to expose where scale hurts: +//! A. deliver hot path, 1 thread — encode+append+mutex cost per record +//! B. deliver hot path, 4 threads — the single-mutex contention ceiling +//! C. upload throughput vs object size — small-object overhead on flush +//! D. 64-policy fan-out — per-object cost with client reuse +//! E. outage cost per flush — z3 connect-retry sleeps serialize flush + +const std = @import("std"); +const proto = @import("proto"); +const s3 = @import("s3"); +const s3_dump_mod = @import("./s3_dump.zig"); +const S3Dump = s3_dump_mod.S3Dump; + +// Bench allocations are large and churny; the debug testing allocator would +// dominate the measurement. smp_allocator is the ReleaseFast-appropriate +// general-purpose allocator. +const gpa = std.heap.smp_allocator; + +const Env = struct { + access_key: []const u8, + secret_key: []const u8, + endpoint: []const u8, + bucket: []const u8, + + fn load(environ: std.process.Environ) ?Env { + return .{ + .access_key = environ.getPosix("AWS_ACCESS_KEY_ID") orelse return null, + .secret_key = environ.getPosix("AWS_SECRET_ACCESS_KEY") orelse return null, + .endpoint = environ.getPosix("S3_ENDPOINT") orelse "http://127.0.0.1:9000", + .bucket = environ.getPosix("S3_BUCKET") orelse "policy-zig-bench", + }; + } +}; + +/// ~200-byte log-shaped line, the typical record size for a waste dump. +const record_line = "{\"timestamp\":\"2026-07-08T12:00:00Z\",\"severity\":\"DEBUG\",\"body\":\"GET /api/v1/checkout/cart 200 requested by user\",\"service.name\":\"checkout-api\",\"http.request.id\":\"01890a5d-ac96-774b-b84e-fe56\"}"; + +fn encodeRecord(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + _ = record; + try writer.writeAll(record_line); +} + +fn encodeTargetRef(allocator: std.mem.Allocator, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = "s3", .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +fn nowNs(io: std.Io) i128 { + return std.Io.Timestamp.now(io, .awake).nanoseconds; +} + +fn mibPerSec(bytes: usize, elapsed_ns: i128) f64 { + const secs = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; + return @as(f64, @floatFromInt(bytes)) / (1024.0 * 1024.0) / secs; +} + +fn perSec(count: usize, elapsed_ns: i128) f64 { + const secs = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; + return @as(f64, @floatFromInt(count)) / secs; +} + +/// Build a dump with one target and `policies` resolved slots. Caller deinits. +fn makeDump( + io: std.Io, + env: Env, + options: S3Dump.Options, + endpoint_override: ?[]const u8, + policies: u32, +) !struct { dump: S3Dump, slots: []u32 } { + var dump = S3Dump.init(gpa, options, .{ + .access_key_id = env.access_key, + .secret_access_key = env.secret_key, + }); + errdefer dump.deinit(); + + const target_json = try std.fmt.allocPrint( + gpa, + \\{{"endpoint": "{s}", "region": "us-east-1", "bucket": "{s}", "prefix": "bench/", "force_path_style": true}} + , + .{ endpoint_override orelse env.endpoint, env.bucket }, + ); + defer gpa.free(target_json); + try dump.addTarget(io, "bench-target", target_json); + + const ref = try encodeTargetRef(gpa, "bench-target"); + defer gpa.free(ref); + + const slots = try gpa.alloc(u32, policies); + errdefer gpa.free(slots); + for (slots, 0..) |*slot, i| { + var id_buf: [32]u8 = undefined; + const policy_id = try std.fmt.bufPrint(&id_buf, "bench-policy-{d:0>3}", .{i}); + slot.* = dump.resolve(io, .log, policy_id, ref) orelse return error.ResolveFailed; + } + return .{ .dump = dump, .slots = slots }; +} + +fn deliverWorker(dump: *S3Dump, io: std.Io, slot: u32, count: usize) void { + for (0..count) |_| { + dump.deliver(io, slot, @ptrCast(&record_line), encodeRecord); + } +} + +pub fn main(init: std.process.Init) !void { + const env = Env.load(init.minimal.environ) orelse { + std.debug.print( + "bench-s3 needs a running S3-compatible server:\n" ++ + " AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (required)\n" ++ + " S3_ENDPOINT (default http://127.0.0.1:9000)\n" ++ + " S3_BUCKET (default policy-zig-bench)\n" ++ + "or run `task bench:s3` to do all of it via docker.\n", + .{}, + ); + return error.MissingS3Environment; + }; + + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + // Ensure the bucket exists. + var admin = try s3.S3Client.init(gpa, .{ + .access_key_id = env.access_key, + .secret_access_key = env.secret_key, + .endpoint = env.endpoint, + .virtual_host_style = false, + }, .{ .io = io }); + defer admin.deinit(); + var create_resp = try admin.createBucket(env.bucket, .{}); + create_resp.deinit(); + + std.debug.print( + "\ns3-dump scale benchmarks ({s}, record={d}B)\n" ++ + "{s}\n", + .{ env.endpoint, record_line.len, "-" ** 78 }, + ); + + // ------------------------------------------------------------------- + // A + B: deliver hot path, 1 vs 4 threads (no uploads; memory only). + // ------------------------------------------------------------------- + const deliver_records: usize = 200_000; + const deliver_opts: S3Dump.Options = .{ .max_sealed_bytes = 1 << 30 }; + inline for (.{ 1, 4 }) |workers| { + var made = try makeDump(io, env, deliver_opts, null, workers); + defer made.dump.deinit(); + defer gpa.free(made.slots); + + const per_worker = deliver_records / workers; + const start = nowNs(io); + if (workers == 1) { + deliverWorker(&made.dump, io, made.slots[0], per_worker); + } else { + var group: std.Io.Group = .init; + var spawn_failed = false; + for (made.slots) |slot| { + group.concurrent(io, deliverWorker, .{ &made.dump, io, slot, per_worker }) catch { + spawn_failed = true; + break; + }; + } + // Await before any early return so no worker outlives the dump. + group.await(io) catch {}; + if (spawn_failed) return error.ConcurrencyUnavailable; + } + const elapsed = nowNs(io) - start; + + const dropped = made.dump.records_dropped.load(.monotonic); + std.debug.print( + "deliver {d} thread(s) {d:>8} recs {d:>11.0} recs/s {d:>8.1} MiB/s drops={d}\n", + .{ + workers, + deliver_records, + perSec(deliver_records, elapsed), + mibPerSec(deliver_records * (record_line.len + 1), elapsed), + dropped, + }, + ); + } + + // ------------------------------------------------------------------- + // C: upload throughput vs object size (~32MiB total per size). + // ------------------------------------------------------------------- + const total_upload_bytes: usize = 32 << 20; + inline for (.{ 64 << 10, 1 << 20, 4 << 20 }) |object_bytes| { + var made = try makeDump(io, env, .{ + .max_batch_bytes = object_bytes, + .max_sealed_bytes = 1 << 30, + .max_attempts = 1, + }, null, 1); + defer made.dump.deinit(); + defer gpa.free(made.slots); + + const records = total_upload_bytes / (record_line.len + 1); + deliverWorker(&made.dump, io, made.slots[0], records); + + const start = nowNs(io); + const result = made.dump.flush(io, .{ .force = true }); + const elapsed = nowNs(io) - start; + + std.debug.print( + "flush {d:>5} KiB objects {d:>8} objs {d:>11.1} objs/s {d:>8.1} MiB/s failed={d}\n", + .{ + object_bytes / 1024, + result.objects_uploaded, + perSec(result.objects_uploaded, elapsed), + mibPerSec(total_upload_bytes, elapsed), + result.objects_failed, + }, + ); + } + + // ------------------------------------------------------------------- + // D: 64-policy fan-out — one flush uploading 64 small objects. + // ------------------------------------------------------------------- + { + const fanout_policies: u32 = 64; + const per_policy_bytes: usize = 128 << 10; + var made = try makeDump(io, env, .{ .max_sealed_bytes = 1 << 30, .max_attempts = 1 }, null, fanout_policies); + defer made.dump.deinit(); + defer gpa.free(made.slots); + + const records_per_policy = per_policy_bytes / (record_line.len + 1); + for (made.slots) |slot| { + deliverWorker(&made.dump, io, slot, records_per_policy); + } + + const start = nowNs(io); + const result = made.dump.flush(io, .{ .force = true }); + const elapsed = nowNs(io) - start; + + std.debug.print( + "fan-out 64 policies {d:>8} objs {d:>11.1} objs/s {d:>8.1} MiB/s failed={d}\n", + .{ + result.objects_uploaded, + perSec(result.objects_uploaded, elapsed), + mibPerSec(fanout_policies * per_policy_bytes, elapsed), + result.objects_failed, + }, + ); + } + + // ------------------------------------------------------------------- + // E: outage cost — dead endpoint, 8 batches, per-flush wall time. + // z3 sleeps ~100ms between connect retries, serialized per object, so + // max_attempts multiplies directly into flush latency during an outage. + // ------------------------------------------------------------------- + inline for (.{ 1, 3 }) |attempts| { + var made = try makeDump(io, env, .{ + .max_sealed_bytes = 1 << 30, + .max_attempts = attempts, + }, "http://127.0.0.1:9", 8); + defer made.dump.deinit(); + defer gpa.free(made.slots); + + for (made.slots) |slot| { + deliverWorker(&made.dump, io, slot, 16); + } + + const start = nowNs(io); + const result = made.dump.flush(io, .{ .force = true }); + const elapsed = nowNs(io) - start; + + std.debug.print( + "outage flush (attempts={d}) {d:>7} objs {d:>10.1} ms/flush requeued={d}\n", + .{ + attempts, + result.objects_failed, + @as(f64, @floatFromInt(elapsed)) / std.time.ns_per_ms, + result.objects_requeued, + }, + ); + } + + std.debug.print("{s}\n", .{"-" ** 78}); +} diff --git a/src/extensions/s3_minio_test.zig b/src/extensions/s3_minio_test.zig new file mode 100644 index 0000000..bf55fab --- /dev/null +++ b/src/extensions/s3_minio_test.zig @@ -0,0 +1,135 @@ +//! Real-storage smoke test for the s3-dump extension, run against a local +//! MinIO container. Not part of `zig build test` — it needs a running +//! backend, so it's a separate build step (`zig build test-s3-e2e`) driven by +//! `task test:s3-e2e`, which starts MinIO via docker, runs this, and tears +//! down. See `src/extensions/s3_stub_test.zig` for the hermetic equivalent +//! that runs in every `zig build test`. +//! +//! Verifies the whole path against a real implementation of the S3 API that +//! the stub only approximates: bucket creation, SigV4 accepted by a real +//! server, and the object actually readable back afterward with +//! byte-identical content. The stub test (s3_stub_test.zig) covers retry +//! mechanics under scripted failures; this test covers "does a real +//! S3-compatible server accept what we send and return what we sent". + +const std = @import("std"); +const proto = @import("proto"); +const s3 = @import("s3"); +const s3_dump_mod = @import("./s3_dump.zig"); +const S3Dump = s3_dump_mod.S3Dump; + +const testing = std.testing; + +fn requiredEnv(environ: std.process.Environ, name: []const u8) ?[]const u8 { + return environ.getPosix(name); +} + +const Env = struct { + access_key: []const u8, + secret_key: []const u8, + endpoint: []const u8, + bucket: []const u8, + + fn load() ?Env { + const environ = std.testing.environ; + return .{ + .access_key = requiredEnv(environ, "AWS_ACCESS_KEY_ID") orelse return null, + .secret_key = requiredEnv(environ, "AWS_SECRET_ACCESS_KEY") orelse return null, + .endpoint = requiredEnv(environ, "S3_ENDPOINT") orelse "http://127.0.0.1:9000", + .bucket = requiredEnv(environ, "S3_BUCKET") orelse "policy-zig-e2e", + }; + } +}; + +fn encodeTargetRef(allocator: std.mem.Allocator, kind: []const u8, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = kind, .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +fn encodeRecord(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + const msg: *const []const u8 = @ptrCast(@alignCast(record)); + try writer.writeAll(msg.*); +} + +test "e2e minio: real upload round-trips through a real S3-compatible server" { + const env = Env.load() orelse return error.SkipZigTest; + const allocator = testing.allocator; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + // Ensure the bucket exists (idempotent: MinIO returns 409 if it's already + // there, which z3's own test suite treats as success too). + var admin = try s3.S3Client.init(allocator, .{ + .access_key_id = env.access_key, + .secret_access_key = env.secret_key, + .endpoint = env.endpoint, + .virtual_host_style = false, + }, .{ .io = io }); + defer admin.deinit(); + var create_resp = try admin.createBucket(env.bucket, .{}); + defer create_resp.deinit(); + try testing.expect(create_resp.http_head.status == .ok or create_resp.http_head.status == .conflict); + + // Drive the extension exactly as the engine would: resolve a target, + // deliver two records, flush for real. + var dump = S3Dump.init(allocator, .{ .max_attempts = 1 }, .{ + .access_key_id = env.access_key, + .secret_access_key = env.secret_key, + }); + defer dump.deinit(); + + const target_json = try std.fmt.allocPrint( + allocator, + \\{{"endpoint": "{s}", "region": "us-east-1", "bucket": "{s}", "prefix": "e2e/", "force_path_style": true}} + , + .{ env.endpoint, env.bucket }, + ); + defer allocator.free(target_json); + try dump.addTarget(io, "minio", target_json); + + const ref = try encodeTargetRef(allocator, "s3", "minio"); + defer allocator.free(ref); + const slot = dump.resolve(io, .log, "e2e-policy", ref).?; + + const rec1: []const u8 = "{\"body\":\"first record\"}"; + const rec2: []const u8 = "{\"body\":\"second record\"}"; + dump.deliver(io, slot, @ptrCast(&rec1), encodeRecord); + dump.deliver(io, slot, @ptrCast(&rec2), encodeRecord); + + const result = dump.flush(io, .{ .force = true }); + try testing.expectEqual(@as(u32, 1), result.objects_uploaded); + try testing.expectEqual(@as(u32, 0), result.objects_failed); + try testing.expectEqual(@as(u64, 2), result.records_uploaded); + + // Verify against the real backend, not our own bookkeeping: list the + // prefix, fetch the object, and check the bytes really landed. + var listing = try admin.listObjects(env.bucket, .{ .prefix = "e2e/log/" }); + defer listing.deinit(); + // A fresh container (the task flow) holds exactly our object; a + // long-lived server may hold leftovers from direct runs, so find ours + // by key substring rather than assuming it's the only one. + const key = blk: { + for (listing.objects) |obj| { + if (std.mem.indexOf(u8, obj.key, "e2e-policy-") != null) break :blk obj.key; + } + return error.UploadedObjectNotFound; + }; + + var get_resp = try admin.getObject(env.bucket, key, .{}); + defer get_resp.deinit(); + try testing.expectEqual(std.http.Status.ok, get_resp.http_head.status); + try testing.expectEqualStrings( + "{\"body\":\"first record\"}\n{\"body\":\"second record\"}\n", + get_resp.body, + ); + + // Clean up so repeated runs don't accumulate objects. + var del_resp = try admin.deleteObject(env.bucket, key, .{}); + del_resp.deinit(); +} diff --git a/src/extensions/s3_stub_test.zig b/src/extensions/s3_stub_test.zig new file mode 100644 index 0000000..e2165e3 --- /dev/null +++ b/src/extensions/s3_stub_test.zig @@ -0,0 +1,262 @@ +//! End-to-end verification of the s3-dump upload path against an in-process +//! HTTP server that speaks just enough S3 — no docker, no external network, +//! runs in plain `zig build test`. Verifies the bytes on the wire: method, +//! path-style URL and key layout, SigV4 signing, the payload hash S3 uses for +//! integrity verification, and the requeue-then-retry durability path. +//! +//! For a real-storage smoke test against MinIO, see `task test:s3-e2e`. + +const std = @import("std"); +const proto = @import("proto"); +const s3_dump_mod = @import("./s3_dump.zig"); +const S3Dump = s3_dump_mod.S3Dump; + +const testing = std.testing; + +/// Minimal S3 stand-in: accepts connections, records every request (method, +/// target, raw head bytes, body), and answers each with the next scripted +/// status. Runs concurrently with the flush under test via io.concurrent. +const StubS3Server = struct { + const Received = struct { + method: std.http.Method, + target: []u8, + head: []u8, + body: []u8, + }; + + allocator: std.mem.Allocator, + statuses: []const std.http.Status, + listener: std.Io.net.Server, + port: u16, + received: std.ArrayList(Received) = .empty, + + fn start( + allocator: std.mem.Allocator, + io: std.Io, + statuses: []const std.http.Status, + ) !StubS3Server { + // std.Io exposes no getsockname, so probe a small fixed range + // instead of binding port 0. + var attempt: u16 = 0; + while (attempt < 32) : (attempt += 1) { + const port: u16 = 42741 + attempt; + const addr = try std.Io.net.IpAddress.parse("127.0.0.1", port); + const listener = addr.listen(io, .{}) catch |err| switch (err) { + error.AddressInUse => continue, + else => return err, + }; + return .{ + .allocator = allocator, + .statuses = statuses, + .listener = listener, + .port = port, + }; + } + return error.AddressInUse; + } + + fn deinit(self: *StubS3Server, io: std.Io) void { + defer self.* = undefined; + self.listener.deinit(io); + for (self.received.items) |r| { + self.allocator.free(r.target); + self.allocator.free(r.head); + self.allocator.free(r.body); + } + self.received.deinit(self.allocator); + } + + /// Serve until every scripted status has been sent. Handles both + /// keep-alive (several requests on one connection) and reconnects + /// (a new client per flush). + fn serve(self: *StubS3Server, io: std.Io) void { + var served: usize = 0; + outer: while (served < self.statuses.len) { + var stream = self.listener.accept(io) catch return; + defer stream.close(io); + var recv_buf: [64 * 1024]u8 = undefined; + var send_buf: [4 * 1024]u8 = undefined; + var conn_reader = stream.reader(io, &recv_buf); + var conn_writer = stream.writer(io, &send_buf); + var http_server = std.http.Server.init(&conn_reader.interface, &conn_writer.interface); + while (served < self.statuses.len) { + var request = http_server.receiveHead() catch continue :outer; + self.handleOne(&request, self.statuses[served]) catch continue :outer; + served += 1; + } + } + } + + fn handleOne( + self: *StubS3Server, + request: *std.http.Server.Request, + status: std.http.Status, + ) !void { + // Head pointers are invalidated once the body stream starts — copy first. + const method = request.head.method; + const target = try self.allocator.dupe(u8, request.head.target); + errdefer self.allocator.free(target); + const head = try self.allocator.dupe(u8, request.head_buffer); + errdefer self.allocator.free(head); + + var transfer_buf: [1024]u8 = undefined; + const body_reader = try request.readerExpectContinue(&transfer_buf); + const body = try body_reader.allocRemaining(self.allocator, .limited(1 << 20)); + errdefer self.allocator.free(body); + + try self.received.append(self.allocator, .{ + .method = method, + .target = target, + .head = head, + .body = body, + }); + try request.respond("", .{ .status = status }); + } + + fn headContains(head: []const u8, needle: []const u8) bool { + return std.ascii.indexOfIgnoreCase(head, needle) != null; + } +}; + +fn encodeTargetRef(allocator: std.mem.Allocator, kind: []const u8, name: []const u8) ![]u8 { + var aw = std.Io.Writer.Allocating.init(allocator); + errdefer aw.deinit(); + const ref: proto.policy.ExtensionTargetRef = .{ .kind = kind, .name = name }; + try ref.encode(&aw.writer, allocator); + var list = aw.toArrayList(); + return list.toOwnedSlice(allocator); +} + +fn targetJson(allocator: std.mem.Allocator, port: u16) ![]u8 { + return std.fmt.allocPrint( + allocator, + \\{{"endpoint": "http://127.0.0.1:{d}", "region": "us-east-1", "bucket": "waste", "prefix": "dumps/"}} + , + .{port}, + ); +} + +fn encodeRecord(record: *const anyopaque, writer: *std.Io.Writer) anyerror!void { + const msg: *const []const u8 = @ptrCast(@alignCast(record)); + try writer.writeAll(msg.*); +} + +const test_credentials: S3Dump.Credentials = .{ + .access_key_id = "test-access-key", + .secret_access_key = "test-secret-key", +}; + +test "e2e stub: flush PUTs the signed ndjson batch under the sealed key" { + const allocator = testing.allocator; + // The shared test-runner io caps concurrency; the stub server needs a + // real thread, so build our own Threaded io like a consumer binary would. + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var stub = try StubS3Server.start(allocator, io, &.{.ok}); + defer stub.deinit(io); + var server_future = io.concurrent(StubS3Server.serve, .{ &stub, io }) catch + return error.SkipZigTest; + // If an assertion fires before the server saw its scripted requests, + // cancel instead of hanging in accept(); on the happy path (already + // awaited) this is a no-op. + defer server_future.cancel(io); + + var dump = S3Dump.init(allocator, .{ .max_attempts = 1 }, test_credentials); + defer dump.deinit(); + const tj = try targetJson(allocator, stub.port); + defer allocator.free(tj); + try dump.addTarget(io, "stub", tj); + + const ref = try encodeTargetRef(allocator, "s3", "stub"); + defer allocator.free(ref); + const slot = dump.resolve(io, .log, "dump-policy", ref).?; + + const rec1: []const u8 = "{\"body\":\"one\"}"; + const rec2: []const u8 = "{\"body\":\"two\"}"; + dump.deliver(io, slot, @ptrCast(&rec1), encodeRecord); + dump.deliver(io, slot, @ptrCast(&rec2), encodeRecord); + + const result = dump.flush(io, .{ .force = true }); + server_future.await(io); + + try testing.expectEqual(@as(u32, 1), result.objects_uploaded); + try testing.expectEqual(@as(u64, 2), result.records_uploaded); + try testing.expectEqual(@as(u32, 0), result.objects_failed); + // bytes_uploaded matches the exact body size; the successfully uploaded + // batch leaves nothing behind in the backlog gauge. + try testing.expectEqual(@as(u64, "{\"body\":\"one\"}\n{\"body\":\"two\"}\n".len), result.bytes_uploaded); + try testing.expectEqual(@as(usize, 0), result.backlog_bytes); + + try testing.expectEqual(@as(usize, 1), stub.received.items.len); + const req = stub.received.items[0]; + try testing.expectEqual(std.http.Method.PUT, req.method); + // Path-style URL: /{bucket}/{key} with the documented key layout. + try testing.expect(std.mem.startsWith(u8, req.target, "/waste/dumps/log/")); + try testing.expect(std.mem.endsWith(u8, req.target, ".ndjson")); + try testing.expect(std.mem.indexOf(u8, req.target, "dump-policy-") != null); + // The exact batch bytes arrived, one record per line. + try testing.expectEqualStrings("{\"body\":\"one\"}\n{\"body\":\"two\"}\n", req.body); + // SigV4-signed, with the payload hash S3 verifies end-to-end. + try testing.expect(StubS3Server.headContains(req.head, "AWS4-HMAC-SHA256")); + var hash: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(req.body, &hash, .{}); + const hex = std.fmt.bytesToHex(hash, .lower); + try testing.expect(StubS3Server.headContains(req.head, &hex)); + try testing.expect(StubS3Server.headContains(req.head, "application/x-ndjson")); +} + +test "e2e stub: 5xx requeues; the retry succeeds under the same key with the same bytes" { + const allocator = testing.allocator; + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var stub = try StubS3Server.start(allocator, io, &.{ .internal_server_error, .ok }); + defer stub.deinit(io); + var server_future = io.concurrent(StubS3Server.serve, .{ &stub, io }) catch + return error.SkipZigTest; + // If an assertion fires before the server saw its scripted requests, + // cancel instead of hanging in accept(); on the happy path (already + // awaited) this is a no-op. + defer server_future.cancel(io); + + var dump = S3Dump.init(allocator, .{ .max_attempts = 1 }, test_credentials); + defer dump.deinit(); + const tj = try targetJson(allocator, stub.port); + defer allocator.free(tj); + try dump.addTarget(io, "stub", tj); + + const ref = try encodeTargetRef(allocator, "s3", "stub"); + defer allocator.free(ref); + const slot = dump.resolve(io, .log, "dump-policy", ref).?; + + const rec: []const u8 = "{\"body\":\"one\"}"; + dump.deliver(io, slot, @ptrCast(&rec), encodeRecord); + + // First flush: server answers 500 → batch requeued, nothing lost. The + // backlog gauge goes nonzero here — this is the metric an edge alerts on + // to catch a struggling destination before records_dropped climbs. + const r1 = dump.flush(io, .{ .force = true }); + try testing.expectEqual(@as(u32, 0), r1.objects_uploaded); + try testing.expectEqual(@as(u32, 1), r1.objects_failed); + try testing.expectEqual(@as(u32, 1), r1.objects_requeued); + try testing.expectEqual(@as(u64, 0), r1.records_dropped); + try testing.expectEqual(@as(u64, rec.len + 1), r1.bytes_failed); + try testing.expect(r1.backlog_bytes > 0); + + // Second flush: retry succeeds, and the gauge falls back to zero. + const r2 = dump.flush(io, .{ .force = true }); + server_future.await(io); + try testing.expectEqual(@as(u32, 1), r2.objects_uploaded); + try testing.expectEqual(@as(u64, 1), r2.records_uploaded); + try testing.expectEqual(@as(u64, rec.len + 1), r2.bytes_uploaded); + try testing.expectEqual(@as(usize, 0), r2.backlog_bytes); + + // Both attempts used the SAME key and carried identical bytes — the + // retry is an idempotent overwrite, never a duplicate object. + try testing.expectEqual(@as(usize, 2), stub.received.items.len); + try testing.expectEqualStrings(stub.received.items[0].target, stub.received.items[1].target); + try testing.expectEqualStrings(stub.received.items[0].body, stub.received.items[1].body); +} From d66fe64165ca66f9150e5c314a3b37fce6252735 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Thu, 9 Jul 2026 12:13:08 -0400 Subject: [PATCH 8/8] fix: empty checks --- src/policy/matcher_index.zig | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/policy/matcher_index.zig b/src/policy/matcher_index.zig index c109d81..b1658c7 100644 --- a/src/policy/matcher_index.zig +++ b/src/policy/matcher_index.zig @@ -1760,7 +1760,8 @@ pub const LogMatcherIndex = struct { } pub fn isEmpty(self: *const LogMatcherIndex) bool { - return self.matcher_keys.len == 0 and self.typed_checks.len == 0; + return self.matcher_keys.len == 0 and self.typed_checks.len == 0 and + self.extension_bindings.len == 0; } pub fn getDatabaseCount(self: *const LogMatcherIndex) usize { @@ -1921,7 +1922,8 @@ pub const MetricMatcherIndex = struct { } pub fn isEmpty(self: *const MetricMatcherIndex) bool { - return self.matcher_keys.len == 0 and self.typed_checks.len == 0; + return self.matcher_keys.len == 0 and self.typed_checks.len == 0 and + self.extension_bindings.len == 0; } pub fn getDatabaseCount(self: *const MetricMatcherIndex) usize { @@ -2077,7 +2079,8 @@ pub const TraceMatcherIndex = struct { } pub fn isEmpty(self: *const TraceMatcherIndex) bool { - return self.matcher_keys.len == 0 and self.typed_checks.len == 0; + return self.matcher_keys.len == 0 and self.typed_checks.len == 0 and + self.extension_bindings.len == 0; } pub fn getDatabaseCount(self: *const TraceMatcherIndex) usize {