From 0665ce2127d0e1de171a3335a28df8ad98b6307b Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sun, 24 May 2026 16:19:10 +0000 Subject: [PATCH] =?UTF-8?q?feat(storage=20v3):=20Stack=202=20=E2=80=94=20c?= =?UTF-8?q?1z3=20envelope=20(manifest=20+=20payload=20framing)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stack 2 of the storage-engine-v4 PR series. Wires the v3 file envelope on top of Stack 1's protos and codec layer. All under `//go:build batonsdkv2`. Proto at `proto/c1/c1z/v3/manifest.proto`: * `C1ZManifestV3` — engine identifier, engine_schema_version, engine_config (Any, typically PebbleEngineConfig), payload encoding, transitively-closed FileDescriptorSet, record-type catalog, sync-run summary, tooling metadata. * `PayloadEncoding` enum (UNSPECIFIED, RAW, ZSTD, ZSTD_TAR). * `RecordTypeInfo` for per-record-type schema versioning. * `SyncRunSummary` for cheap tooling enumeration without engine open. * `PebbleEngineConfig` — pinned format_major_version + cache size hint. Format package at `pkg/dotc1z/format/v3/`: * `manifest.go` — Marshal/Unmarshal with empty-engine guard; BuildDescriptorClosure walks protoregistry.GlobalFiles for c1.storage.v3 packages and returns the transitive-closure FileDescriptorSet; VerifyDescriptorClosure detects missing dependencies at read time (returns ErrManifestIncompleteDescriptors). * `envelope.go` — WriteEnvelope emits magic + uint32-BE length prefix + manifest + zstd-tar payload. ReadEnvelope parses each layer with explicit truncation + manifest-size cap (16 MiB) + invalid-magic guards. ExtractZstdTar streams the payload tar into a destination directory with directory-traversal protection via filepath.IsLocal. Tests cover: full directory roundtrip (write a synthetic Pebble-shaped dir, read it back, verify file contents); bad magic; truncated header; empty-engine refusal on both Marshal and Unmarshal; descriptor closure completeness verification (missing dep detected; complete closure accepted; storage.v3 records.proto present in the auto-built closure). Deferred: * Full protodesc.ToFileDescriptorProto conversion in BuildDescriptorClosure (Stack 3) — Stack 2 ships the import-graph- plus-package-name projection so the closure invariant is testable. The richer projection lands when the engine actually consumes the descriptors via dynamicpb at open. * `cmd/baton-descriptor-closure-test/` standalone CI fixture lands in Stack 5 alongside the equivalence harness. Refs: RFC v4 §3.2 (envelope), Appendix B (full envelope spec), research/import-11.md TestCheckpointRoundtrip for the source-DB- stays-writable property the save protocol depends on. --- pb/c1/c1z/v3/manifest.pb.go | 752 ++++++++++++++++++++++ pb/c1/c1z/v3/manifest.pb.validate.go | 681 ++++++++++++++++++++ pb/c1/c1z/v3/manifest_protoopaque.pb.go | 724 +++++++++++++++++++++ pkg/dotc1z/format/v3/doc.go | 14 + pkg/dotc1z/format/v3/envelope.go | 241 +++++++ pkg/dotc1z/format/v3/envelope_test.go | 210 ++++++ pkg/dotc1z/format/v3/manifest.go | 146 +++++ pkg/dotc1z/format/v3/test_helpers_test.go | 26 + proto/c1/c1z/v3/manifest.proto | 101 +++ 9 files changed, 2895 insertions(+) create mode 100644 pb/c1/c1z/v3/manifest.pb.go create mode 100644 pb/c1/c1z/v3/manifest.pb.validate.go create mode 100644 pb/c1/c1z/v3/manifest_protoopaque.pb.go create mode 100644 pkg/dotc1z/format/v3/doc.go create mode 100644 pkg/dotc1z/format/v3/envelope.go create mode 100644 pkg/dotc1z/format/v3/envelope_test.go create mode 100644 pkg/dotc1z/format/v3/manifest.go create mode 100644 pkg/dotc1z/format/v3/test_helpers_test.go create mode 100644 proto/c1/c1z/v3/manifest.proto diff --git a/pb/c1/c1z/v3/manifest.pb.go b/pb/c1/c1z/v3/manifest.pb.go new file mode 100644 index 000000000..fb54805a9 --- /dev/null +++ b/pb/c1/c1z/v3/manifest.pb.go @@ -0,0 +1,752 @@ +// V3 envelope manifest. Embedded inside the C1Z3 file format after +// the 5-byte magic and a uint32 BE length prefix. +// +// The manifest pins a transitively-closed FileDescriptorSet of every +// c1.storage.v3 record type stored in the payload. This is what makes +// v3 c1z files self-describing — any reader, including one without the +// originating SDK's compiled-in protos, can resolve every field via +// dynamicpb against the sidecar descriptor set. +// +// See RFC v4 §3.2 and Appendix B for the full envelope spec. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/c1z/v3/manifest.proto + +//go:build !protoopaque + +package v3 + +import ( + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + descriptorpb "google.golang.org/protobuf/types/descriptorpb" + anypb "google.golang.org/protobuf/types/known/anypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type PayloadEncoding int32 + +const ( + PayloadEncoding_PAYLOAD_ENCODING_UNSPECIFIED PayloadEncoding = 0 + PayloadEncoding_PAYLOAD_ENCODING_RAW PayloadEncoding = 1 // debug only + PayloadEncoding_PAYLOAD_ENCODING_ZSTD PayloadEncoding = 2 // single-stream zstd (sqlite-style) + PayloadEncoding_PAYLOAD_ENCODING_ZSTD_TAR PayloadEncoding = 3 // zstd-tar of a Pebble directory +) + +// Enum value maps for PayloadEncoding. +var ( + PayloadEncoding_name = map[int32]string{ + 0: "PAYLOAD_ENCODING_UNSPECIFIED", + 1: "PAYLOAD_ENCODING_RAW", + 2: "PAYLOAD_ENCODING_ZSTD", + 3: "PAYLOAD_ENCODING_ZSTD_TAR", + } + PayloadEncoding_value = map[string]int32{ + "PAYLOAD_ENCODING_UNSPECIFIED": 0, + "PAYLOAD_ENCODING_RAW": 1, + "PAYLOAD_ENCODING_ZSTD": 2, + "PAYLOAD_ENCODING_ZSTD_TAR": 3, + } +) + +func (x PayloadEncoding) Enum() *PayloadEncoding { + p := new(PayloadEncoding) + *p = x + return p +} + +func (x PayloadEncoding) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PayloadEncoding) Descriptor() protoreflect.EnumDescriptor { + return file_c1_c1z_v3_manifest_proto_enumTypes[0].Descriptor() +} + +func (PayloadEncoding) Type() protoreflect.EnumType { + return &file_c1_c1z_v3_manifest_proto_enumTypes[0] +} + +func (x PayloadEncoding) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +type C1ZManifestV3 struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Engine identifier. "pebble" is the only value Stack 3 supports. + // Future engines (e.g. "sled") can dispatch on this string without + // an envelope change. + Engine string `protobuf:"bytes,1,opt,name=engine,proto3" json:"engine,omitempty"` + // Engine-specific schema version. Pebble interprets this as the + // pinned format-major-version it was written under; mismatches at + // open surface as ErrPebbleFormatNewer (refuse, page on-call) or + // ErrPebbleFormatOlder (refuse, treat as corruption). + EngineSchemaVersion uint32 `protobuf:"varint,2,opt,name=engine_schema_version,json=engineSchemaVersion,proto3" json:"engine_schema_version,omitempty"` + // Engine-specific configuration. For Pebble, this carries + // PebbleEngineConfig. The reader looks up the Any type-URL against + // its compiled-in registry; unknown type-URLs are treated as + // "no hint" (advisory only — the dispatch key is `engine`). + EngineConfig *anypb.Any `protobuf:"bytes,3,opt,name=engine_config,json=engineConfig,proto3" json:"engine_config,omitempty"` + // Payload encoding. PAYLOAD_ENCODING_ZSTD_TAR is the production + // default for Pebble engines. + PayloadEncoding PayloadEncoding `protobuf:"varint,4,opt,name=payload_encoding,json=payloadEncoding,proto3,enum=c1.c1z.v3.PayloadEncoding" json:"payload_encoding,omitempty"` + // Transitively-closed FileDescriptorSet of every record type stored + // in the payload. Built at write time by walking + // protoregistry.GlobalFiles. The reader verifies closure at open + // and returns ErrManifestIncompleteDescriptors if any record-type + // descriptor cannot resolve from this set alone. + Descriptors *descriptorpb.FileDescriptorSet `protobuf:"bytes,10,opt,name=descriptors,proto3" json:"descriptors,omitempty"` + // Record types produced by this writer. Used by the reader to + // detect unknown record types (forward-compat with a future SDK + // that adds new record types this binary doesn't know). + RecordTypes []*RecordTypeInfo `protobuf:"bytes,11,rep,name=record_types,json=recordTypes,proto3" json:"record_types,omitempty"` + // Informational only. + TenantHint string `protobuf:"bytes,20,opt,name=tenant_hint,json=tenantHint,proto3" json:"tenant_hint,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,30,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + CreatedBySdkVersion string `protobuf:"bytes,31,opt,name=created_by_sdk_version,json=createdBySdkVersion,proto3" json:"created_by_sdk_version,omitempty"` + CreatedByTool string `protobuf:"bytes,32,opt,name=created_by_tool,json=createdByTool,proto3" json:"created_by_tool,omitempty"` + // Cheap projection of sync runs inside the payload — lets tooling + // enumerate without opening the engine. + SyncRuns []*SyncRunSummary `protobuf:"bytes,40,rep,name=sync_runs,json=syncRuns,proto3" json:"sync_runs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *C1ZManifestV3) Reset() { + *x = C1ZManifestV3{} + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *C1ZManifestV3) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*C1ZManifestV3) ProtoMessage() {} + +func (x *C1ZManifestV3) ProtoReflect() protoreflect.Message { + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *C1ZManifestV3) GetEngine() string { + if x != nil { + return x.Engine + } + return "" +} + +func (x *C1ZManifestV3) GetEngineSchemaVersion() uint32 { + if x != nil { + return x.EngineSchemaVersion + } + return 0 +} + +func (x *C1ZManifestV3) GetEngineConfig() *anypb.Any { + if x != nil { + return x.EngineConfig + } + return nil +} + +func (x *C1ZManifestV3) GetPayloadEncoding() PayloadEncoding { + if x != nil { + return x.PayloadEncoding + } + return PayloadEncoding_PAYLOAD_ENCODING_UNSPECIFIED +} + +func (x *C1ZManifestV3) GetDescriptors() *descriptorpb.FileDescriptorSet { + if x != nil { + return x.Descriptors + } + return nil +} + +func (x *C1ZManifestV3) GetRecordTypes() []*RecordTypeInfo { + if x != nil { + return x.RecordTypes + } + return nil +} + +func (x *C1ZManifestV3) GetTenantHint() string { + if x != nil { + return x.TenantHint + } + return "" +} + +func (x *C1ZManifestV3) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *C1ZManifestV3) GetCreatedBySdkVersion() string { + if x != nil { + return x.CreatedBySdkVersion + } + return "" +} + +func (x *C1ZManifestV3) GetCreatedByTool() string { + if x != nil { + return x.CreatedByTool + } + return "" +} + +func (x *C1ZManifestV3) GetSyncRuns() []*SyncRunSummary { + if x != nil { + return x.SyncRuns + } + return nil +} + +func (x *C1ZManifestV3) SetEngine(v string) { + x.Engine = v +} + +func (x *C1ZManifestV3) SetEngineSchemaVersion(v uint32) { + x.EngineSchemaVersion = v +} + +func (x *C1ZManifestV3) SetEngineConfig(v *anypb.Any) { + x.EngineConfig = v +} + +func (x *C1ZManifestV3) SetPayloadEncoding(v PayloadEncoding) { + x.PayloadEncoding = v +} + +func (x *C1ZManifestV3) SetDescriptors(v *descriptorpb.FileDescriptorSet) { + x.Descriptors = v +} + +func (x *C1ZManifestV3) SetRecordTypes(v []*RecordTypeInfo) { + x.RecordTypes = v +} + +func (x *C1ZManifestV3) SetTenantHint(v string) { + x.TenantHint = v +} + +func (x *C1ZManifestV3) SetCreatedAt(v *timestamppb.Timestamp) { + x.CreatedAt = v +} + +func (x *C1ZManifestV3) SetCreatedBySdkVersion(v string) { + x.CreatedBySdkVersion = v +} + +func (x *C1ZManifestV3) SetCreatedByTool(v string) { + x.CreatedByTool = v +} + +func (x *C1ZManifestV3) SetSyncRuns(v []*SyncRunSummary) { + x.SyncRuns = v +} + +func (x *C1ZManifestV3) HasEngineConfig() bool { + if x == nil { + return false + } + return x.EngineConfig != nil +} + +func (x *C1ZManifestV3) HasDescriptors() bool { + if x == nil { + return false + } + return x.Descriptors != nil +} + +func (x *C1ZManifestV3) HasCreatedAt() bool { + if x == nil { + return false + } + return x.CreatedAt != nil +} + +func (x *C1ZManifestV3) ClearEngineConfig() { + x.EngineConfig = nil +} + +func (x *C1ZManifestV3) ClearDescriptors() { + x.Descriptors = nil +} + +func (x *C1ZManifestV3) ClearCreatedAt() { + x.CreatedAt = nil +} + +type C1ZManifestV3_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Engine identifier. "pebble" is the only value Stack 3 supports. + // Future engines (e.g. "sled") can dispatch on this string without + // an envelope change. + Engine string + // Engine-specific schema version. Pebble interprets this as the + // pinned format-major-version it was written under; mismatches at + // open surface as ErrPebbleFormatNewer (refuse, page on-call) or + // ErrPebbleFormatOlder (refuse, treat as corruption). + EngineSchemaVersion uint32 + // Engine-specific configuration. For Pebble, this carries + // PebbleEngineConfig. The reader looks up the Any type-URL against + // its compiled-in registry; unknown type-URLs are treated as + // "no hint" (advisory only — the dispatch key is `engine`). + EngineConfig *anypb.Any + // Payload encoding. PAYLOAD_ENCODING_ZSTD_TAR is the production + // default for Pebble engines. + PayloadEncoding PayloadEncoding + // Transitively-closed FileDescriptorSet of every record type stored + // in the payload. Built at write time by walking + // protoregistry.GlobalFiles. The reader verifies closure at open + // and returns ErrManifestIncompleteDescriptors if any record-type + // descriptor cannot resolve from this set alone. + Descriptors *descriptorpb.FileDescriptorSet + // Record types produced by this writer. Used by the reader to + // detect unknown record types (forward-compat with a future SDK + // that adds new record types this binary doesn't know). + RecordTypes []*RecordTypeInfo + // Informational only. + TenantHint string + CreatedAt *timestamppb.Timestamp + CreatedBySdkVersion string + CreatedByTool string + // Cheap projection of sync runs inside the payload — lets tooling + // enumerate without opening the engine. + SyncRuns []*SyncRunSummary +} + +func (b0 C1ZManifestV3_builder) Build() *C1ZManifestV3 { + m0 := &C1ZManifestV3{} + b, x := &b0, m0 + _, _ = b, x + x.Engine = b.Engine + x.EngineSchemaVersion = b.EngineSchemaVersion + x.EngineConfig = b.EngineConfig + x.PayloadEncoding = b.PayloadEncoding + x.Descriptors = b.Descriptors + x.RecordTypes = b.RecordTypes + x.TenantHint = b.TenantHint + x.CreatedAt = b.CreatedAt + x.CreatedBySdkVersion = b.CreatedBySdkVersion + x.CreatedByTool = b.CreatedByTool + x.SyncRuns = b.SyncRuns + return m0 +} + +type RecordTypeInfo struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Proto FullName — e.g. "c1.storage.v3.GrantRecord". + MessageFullName string `protobuf:"bytes,1,opt,name=message_full_name,json=messageFullName,proto3" json:"message_full_name,omitempty"` + // Record-type-specific schema version. Bumped on any incompatible + // change to the record's stored encoding; reader refuses files + // claiming a version > what it supports. + SchemaVersion uint32 `protobuf:"varint,2,opt,name=schema_version,json=schemaVersion,proto3" json:"schema_version,omitempty"` + // Optional row-count hint for sizing readers. + EstimatedCount int64 `protobuf:"varint,3,opt,name=estimated_count,json=estimatedCount,proto3" json:"estimated_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RecordTypeInfo) Reset() { + *x = RecordTypeInfo{} + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RecordTypeInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RecordTypeInfo) ProtoMessage() {} + +func (x *RecordTypeInfo) ProtoReflect() protoreflect.Message { + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *RecordTypeInfo) GetMessageFullName() string { + if x != nil { + return x.MessageFullName + } + return "" +} + +func (x *RecordTypeInfo) GetSchemaVersion() uint32 { + if x != nil { + return x.SchemaVersion + } + return 0 +} + +func (x *RecordTypeInfo) GetEstimatedCount() int64 { + if x != nil { + return x.EstimatedCount + } + return 0 +} + +func (x *RecordTypeInfo) SetMessageFullName(v string) { + x.MessageFullName = v +} + +func (x *RecordTypeInfo) SetSchemaVersion(v uint32) { + x.SchemaVersion = v +} + +func (x *RecordTypeInfo) SetEstimatedCount(v int64) { + x.EstimatedCount = v +} + +type RecordTypeInfo_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Proto FullName — e.g. "c1.storage.v3.GrantRecord". + MessageFullName string + // Record-type-specific schema version. Bumped on any incompatible + // change to the record's stored encoding; reader refuses files + // claiming a version > what it supports. + SchemaVersion uint32 + // Optional row-count hint for sizing readers. + EstimatedCount int64 +} + +func (b0 RecordTypeInfo_builder) Build() *RecordTypeInfo { + m0 := &RecordTypeInfo{} + b, x := &b0, m0 + _, _ = b, x + x.MessageFullName = b.MessageFullName + x.SchemaVersion = b.SchemaVersion + x.EstimatedCount = b.EstimatedCount + return m0 +} + +type SyncRunSummary struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + SyncId string `protobuf:"bytes,1,opt,name=sync_id,json=syncId,proto3" json:"sync_id,omitempty"` + Type v3.SyncType `protobuf:"varint,2,opt,name=type,proto3,enum=c1.storage.v3.SyncType" json:"type,omitempty"` + StartedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + EndedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SyncRunSummary) Reset() { + *x = SyncRunSummary{} + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SyncRunSummary) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SyncRunSummary) ProtoMessage() {} + +func (x *SyncRunSummary) ProtoReflect() protoreflect.Message { + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SyncRunSummary) GetSyncId() string { + if x != nil { + return x.SyncId + } + return "" +} + +func (x *SyncRunSummary) GetType() v3.SyncType { + if x != nil { + return x.Type + } + return v3.SyncType(0) +} + +func (x *SyncRunSummary) GetStartedAt() *timestamppb.Timestamp { + if x != nil { + return x.StartedAt + } + return nil +} + +func (x *SyncRunSummary) GetEndedAt() *timestamppb.Timestamp { + if x != nil { + return x.EndedAt + } + return nil +} + +func (x *SyncRunSummary) SetSyncId(v string) { + x.SyncId = v +} + +func (x *SyncRunSummary) SetType(v v3.SyncType) { + x.Type = v +} + +func (x *SyncRunSummary) SetStartedAt(v *timestamppb.Timestamp) { + x.StartedAt = v +} + +func (x *SyncRunSummary) SetEndedAt(v *timestamppb.Timestamp) { + x.EndedAt = v +} + +func (x *SyncRunSummary) HasStartedAt() bool { + if x == nil { + return false + } + return x.StartedAt != nil +} + +func (x *SyncRunSummary) HasEndedAt() bool { + if x == nil { + return false + } + return x.EndedAt != nil +} + +func (x *SyncRunSummary) ClearStartedAt() { + x.StartedAt = nil +} + +func (x *SyncRunSummary) ClearEndedAt() { + x.EndedAt = nil +} + +type SyncRunSummary_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + SyncId string + Type v3.SyncType + StartedAt *timestamppb.Timestamp + EndedAt *timestamppb.Timestamp +} + +func (b0 SyncRunSummary_builder) Build() *SyncRunSummary { + m0 := &SyncRunSummary{} + b, x := &b0, m0 + _, _ = b, x + x.SyncId = b.SyncId + x.Type = b.Type + x.StartedAt = b.StartedAt + x.EndedAt = b.EndedAt + return m0 +} + +// PebbleEngineConfig is the value placed inside C1ZManifestV3.engine_config +// when engine == "pebble". Type-URL: type.googleapis.com/c1.c1z.v3.PebbleEngineConfig. +type PebbleEngineConfig struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Pinned Pebble format-major-version this file was written under. + // The reader compares against pebble.FormatNewest and pebble.FormatMinSupported. + FormatMajorVersion uint32 `protobuf:"varint,1,opt,name=format_major_version,json=formatMajorVersion,proto3" json:"format_major_version,omitempty"` + // Hint to readers; advisory only. + CacheSizeBytes uint64 `protobuf:"varint,2,opt,name=cache_size_bytes,json=cacheSizeBytes,proto3" json:"cache_size_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PebbleEngineConfig) Reset() { + *x = PebbleEngineConfig{} + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PebbleEngineConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PebbleEngineConfig) ProtoMessage() {} + +func (x *PebbleEngineConfig) ProtoReflect() protoreflect.Message { + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PebbleEngineConfig) GetFormatMajorVersion() uint32 { + if x != nil { + return x.FormatMajorVersion + } + return 0 +} + +func (x *PebbleEngineConfig) GetCacheSizeBytes() uint64 { + if x != nil { + return x.CacheSizeBytes + } + return 0 +} + +func (x *PebbleEngineConfig) SetFormatMajorVersion(v uint32) { + x.FormatMajorVersion = v +} + +func (x *PebbleEngineConfig) SetCacheSizeBytes(v uint64) { + x.CacheSizeBytes = v +} + +type PebbleEngineConfig_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Pinned Pebble format-major-version this file was written under. + // The reader compares against pebble.FormatNewest and pebble.FormatMinSupported. + FormatMajorVersion uint32 + // Hint to readers; advisory only. + CacheSizeBytes uint64 +} + +func (b0 PebbleEngineConfig_builder) Build() *PebbleEngineConfig { + m0 := &PebbleEngineConfig{} + b, x := &b0, m0 + _, _ = b, x + x.FormatMajorVersion = b.FormatMajorVersion + x.CacheSizeBytes = b.CacheSizeBytes + return m0 +} + +var File_c1_c1z_v3_manifest_proto protoreflect.FileDescriptor + +const file_c1_c1z_v3_manifest_proto_rawDesc = "" + + "\n" + + "\x18c1/c1z/v3/manifest.proto\x12\tc1.c1z.v3\x1a\x1bc1/storage/v3/records.proto\x1a\x19google/protobuf/any.proto\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xd2\x04\n" + + "\rC1ZManifestV3\x12\x16\n" + + "\x06engine\x18\x01 \x01(\tR\x06engine\x122\n" + + "\x15engine_schema_version\x18\x02 \x01(\rR\x13engineSchemaVersion\x129\n" + + "\rengine_config\x18\x03 \x01(\v2\x14.google.protobuf.AnyR\fengineConfig\x12E\n" + + "\x10payload_encoding\x18\x04 \x01(\x0e2\x1a.c1.c1z.v3.PayloadEncodingR\x0fpayloadEncoding\x12D\n" + + "\vdescriptors\x18\n" + + " \x01(\v2\".google.protobuf.FileDescriptorSetR\vdescriptors\x12<\n" + + "\frecord_types\x18\v \x03(\v2\x19.c1.c1z.v3.RecordTypeInfoR\vrecordTypes\x12\x1f\n" + + "\vtenant_hint\x18\x14 \x01(\tR\n" + + "tenantHint\x129\n" + + "\n" + + "created_at\x18\x1e \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x123\n" + + "\x16created_by_sdk_version\x18\x1f \x01(\tR\x13createdBySdkVersion\x12&\n" + + "\x0fcreated_by_tool\x18 \x01(\tR\rcreatedByTool\x126\n" + + "\tsync_runs\x18( \x03(\v2\x19.c1.c1z.v3.SyncRunSummaryR\bsyncRuns\"\x8c\x01\n" + + "\x0eRecordTypeInfo\x12*\n" + + "\x11message_full_name\x18\x01 \x01(\tR\x0fmessageFullName\x12%\n" + + "\x0eschema_version\x18\x02 \x01(\rR\rschemaVersion\x12'\n" + + "\x0festimated_count\x18\x03 \x01(\x03R\x0eestimatedCount\"\xc8\x01\n" + + "\x0eSyncRunSummary\x12\x17\n" + + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x129\n" + + "\n" + + "started_at\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x125\n" + + "\bended_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\aendedAt\"p\n" + + "\x12PebbleEngineConfig\x120\n" + + "\x14format_major_version\x18\x01 \x01(\rR\x12formatMajorVersion\x12(\n" + + "\x10cache_size_bytes\x18\x02 \x01(\x04R\x0ecacheSizeBytes*\x87\x01\n" + + "\x0fPayloadEncoding\x12 \n" + + "\x1cPAYLOAD_ENCODING_UNSPECIFIED\x10\x00\x12\x18\n" + + "\x14PAYLOAD_ENCODING_RAW\x10\x01\x12\x19\n" + + "\x15PAYLOAD_ENCODING_ZSTD\x10\x02\x12\x1d\n" + + "\x19PAYLOAD_ENCODING_ZSTD_TAR\x10\x03B0Z.github.com/conductorone/baton-sdk/pb/c1/c1z/v3b\x06proto3" + +var file_c1_c1z_v3_manifest_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_c1z_v3_manifest_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_c1_c1z_v3_manifest_proto_goTypes = []any{ + (PayloadEncoding)(0), // 0: c1.c1z.v3.PayloadEncoding + (*C1ZManifestV3)(nil), // 1: c1.c1z.v3.C1ZManifestV3 + (*RecordTypeInfo)(nil), // 2: c1.c1z.v3.RecordTypeInfo + (*SyncRunSummary)(nil), // 3: c1.c1z.v3.SyncRunSummary + (*PebbleEngineConfig)(nil), // 4: c1.c1z.v3.PebbleEngineConfig + (*anypb.Any)(nil), // 5: google.protobuf.Any + (*descriptorpb.FileDescriptorSet)(nil), // 6: google.protobuf.FileDescriptorSet + (*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp + (v3.SyncType)(0), // 8: c1.storage.v3.SyncType +} +var file_c1_c1z_v3_manifest_proto_depIdxs = []int32{ + 5, // 0: c1.c1z.v3.C1ZManifestV3.engine_config:type_name -> google.protobuf.Any + 0, // 1: c1.c1z.v3.C1ZManifestV3.payload_encoding:type_name -> c1.c1z.v3.PayloadEncoding + 6, // 2: c1.c1z.v3.C1ZManifestV3.descriptors:type_name -> google.protobuf.FileDescriptorSet + 2, // 3: c1.c1z.v3.C1ZManifestV3.record_types:type_name -> c1.c1z.v3.RecordTypeInfo + 7, // 4: c1.c1z.v3.C1ZManifestV3.created_at:type_name -> google.protobuf.Timestamp + 3, // 5: c1.c1z.v3.C1ZManifestV3.sync_runs:type_name -> c1.c1z.v3.SyncRunSummary + 8, // 6: c1.c1z.v3.SyncRunSummary.type:type_name -> c1.storage.v3.SyncType + 7, // 7: c1.c1z.v3.SyncRunSummary.started_at:type_name -> google.protobuf.Timestamp + 7, // 8: c1.c1z.v3.SyncRunSummary.ended_at:type_name -> google.protobuf.Timestamp + 9, // [9:9] is the sub-list for method output_type + 9, // [9:9] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name +} + +func init() { file_c1_c1z_v3_manifest_proto_init() } +func file_c1_c1z_v3_manifest_proto_init() { + if File_c1_c1z_v3_manifest_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_c1z_v3_manifest_proto_rawDesc), len(file_c1_c1z_v3_manifest_proto_rawDesc)), + NumEnums: 1, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_c1z_v3_manifest_proto_goTypes, + DependencyIndexes: file_c1_c1z_v3_manifest_proto_depIdxs, + EnumInfos: file_c1_c1z_v3_manifest_proto_enumTypes, + MessageInfos: file_c1_c1z_v3_manifest_proto_msgTypes, + }.Build() + File_c1_c1z_v3_manifest_proto = out.File + file_c1_c1z_v3_manifest_proto_goTypes = nil + file_c1_c1z_v3_manifest_proto_depIdxs = nil +} diff --git a/pb/c1/c1z/v3/manifest.pb.validate.go b/pb/c1/c1z/v3/manifest.pb.validate.go new file mode 100644 index 000000000..b4c0d84e7 --- /dev/null +++ b/pb/c1/c1z/v3/manifest.pb.validate.go @@ -0,0 +1,681 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/c1z/v3/manifest.proto + +package v3 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort + + _ = v3.SyncType(0) +) + +// Validate checks the field values on C1ZManifestV3 with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *C1ZManifestV3) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on C1ZManifestV3 with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in C1ZManifestV3MultiError, or +// nil if none found. +func (m *C1ZManifestV3) ValidateAll() error { + return m.validate(true) +} + +func (m *C1ZManifestV3) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Engine + + // no validation rules for EngineSchemaVersion + + if all { + switch v := interface{}(m.GetEngineConfig()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: "EngineConfig", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: "EngineConfig", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetEngineConfig()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return C1ZManifestV3ValidationError{ + field: "EngineConfig", + reason: "embedded message failed validation", + cause: err, + } + } + } + + // no validation rules for PayloadEncoding + + if all { + switch v := interface{}(m.GetDescriptors()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: "Descriptors", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: "Descriptors", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetDescriptors()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return C1ZManifestV3ValidationError{ + field: "Descriptors", + reason: "embedded message failed validation", + cause: err, + } + } + } + + for idx, item := range m.GetRecordTypes() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: fmt.Sprintf("RecordTypes[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: fmt.Sprintf("RecordTypes[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return C1ZManifestV3ValidationError{ + field: fmt.Sprintf("RecordTypes[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + // no validation rules for TenantHint + + if all { + switch v := interface{}(m.GetCreatedAt()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: "CreatedAt", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: "CreatedAt", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetCreatedAt()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return C1ZManifestV3ValidationError{ + field: "CreatedAt", + reason: "embedded message failed validation", + cause: err, + } + } + } + + // no validation rules for CreatedBySdkVersion + + // no validation rules for CreatedByTool + + for idx, item := range m.GetSyncRuns() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: fmt.Sprintf("SyncRuns[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, C1ZManifestV3ValidationError{ + field: fmt.Sprintf("SyncRuns[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return C1ZManifestV3ValidationError{ + field: fmt.Sprintf("SyncRuns[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return C1ZManifestV3MultiError(errors) + } + + return nil +} + +// C1ZManifestV3MultiError is an error wrapping multiple validation errors +// returned by C1ZManifestV3.ValidateAll() if the designated constraints +// aren't met. +type C1ZManifestV3MultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m C1ZManifestV3MultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m C1ZManifestV3MultiError) AllErrors() []error { return m } + +// C1ZManifestV3ValidationError is the validation error returned by +// C1ZManifestV3.Validate if the designated constraints aren't met. +type C1ZManifestV3ValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e C1ZManifestV3ValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e C1ZManifestV3ValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e C1ZManifestV3ValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e C1ZManifestV3ValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e C1ZManifestV3ValidationError) ErrorName() string { return "C1ZManifestV3ValidationError" } + +// Error satisfies the builtin error interface +func (e C1ZManifestV3ValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sC1ZManifestV3.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = C1ZManifestV3ValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = C1ZManifestV3ValidationError{} + +// Validate checks the field values on RecordTypeInfo with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *RecordTypeInfo) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on RecordTypeInfo with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in RecordTypeInfoMultiError, +// or nil if none found. +func (m *RecordTypeInfo) ValidateAll() error { + return m.validate(true) +} + +func (m *RecordTypeInfo) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for MessageFullName + + // no validation rules for SchemaVersion + + // no validation rules for EstimatedCount + + if len(errors) > 0 { + return RecordTypeInfoMultiError(errors) + } + + return nil +} + +// RecordTypeInfoMultiError is an error wrapping multiple validation errors +// returned by RecordTypeInfo.ValidateAll() if the designated constraints +// aren't met. +type RecordTypeInfoMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m RecordTypeInfoMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m RecordTypeInfoMultiError) AllErrors() []error { return m } + +// RecordTypeInfoValidationError is the validation error returned by +// RecordTypeInfo.Validate if the designated constraints aren't met. +type RecordTypeInfoValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e RecordTypeInfoValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e RecordTypeInfoValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e RecordTypeInfoValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e RecordTypeInfoValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e RecordTypeInfoValidationError) ErrorName() string { return "RecordTypeInfoValidationError" } + +// Error satisfies the builtin error interface +func (e RecordTypeInfoValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sRecordTypeInfo.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = RecordTypeInfoValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = RecordTypeInfoValidationError{} + +// Validate checks the field values on SyncRunSummary with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *SyncRunSummary) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SyncRunSummary with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in SyncRunSummaryMultiError, +// or nil if none found. +func (m *SyncRunSummary) ValidateAll() error { + return m.validate(true) +} + +func (m *SyncRunSummary) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for SyncId + + // no validation rules for Type + + if all { + switch v := interface{}(m.GetStartedAt()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SyncRunSummaryValidationError{ + field: "StartedAt", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SyncRunSummaryValidationError{ + field: "StartedAt", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetStartedAt()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SyncRunSummaryValidationError{ + field: "StartedAt", + reason: "embedded message failed validation", + cause: err, + } + } + } + + if all { + switch v := interface{}(m.GetEndedAt()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SyncRunSummaryValidationError{ + field: "EndedAt", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SyncRunSummaryValidationError{ + field: "EndedAt", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetEndedAt()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SyncRunSummaryValidationError{ + field: "EndedAt", + reason: "embedded message failed validation", + cause: err, + } + } + } + + if len(errors) > 0 { + return SyncRunSummaryMultiError(errors) + } + + return nil +} + +// SyncRunSummaryMultiError is an error wrapping multiple validation errors +// returned by SyncRunSummary.ValidateAll() if the designated constraints +// aren't met. +type SyncRunSummaryMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SyncRunSummaryMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SyncRunSummaryMultiError) AllErrors() []error { return m } + +// SyncRunSummaryValidationError is the validation error returned by +// SyncRunSummary.Validate if the designated constraints aren't met. +type SyncRunSummaryValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SyncRunSummaryValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SyncRunSummaryValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SyncRunSummaryValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SyncRunSummaryValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SyncRunSummaryValidationError) ErrorName() string { return "SyncRunSummaryValidationError" } + +// Error satisfies the builtin error interface +func (e SyncRunSummaryValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSyncRunSummary.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SyncRunSummaryValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SyncRunSummaryValidationError{} + +// Validate checks the field values on PebbleEngineConfig with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *PebbleEngineConfig) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on PebbleEngineConfig with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// PebbleEngineConfigMultiError, or nil if none found. +func (m *PebbleEngineConfig) ValidateAll() error { + return m.validate(true) +} + +func (m *PebbleEngineConfig) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for FormatMajorVersion + + // no validation rules for CacheSizeBytes + + if len(errors) > 0 { + return PebbleEngineConfigMultiError(errors) + } + + return nil +} + +// PebbleEngineConfigMultiError is an error wrapping multiple validation errors +// returned by PebbleEngineConfig.ValidateAll() if the designated constraints +// aren't met. +type PebbleEngineConfigMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m PebbleEngineConfigMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m PebbleEngineConfigMultiError) AllErrors() []error { return m } + +// PebbleEngineConfigValidationError is the validation error returned by +// PebbleEngineConfig.Validate if the designated constraints aren't met. +type PebbleEngineConfigValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e PebbleEngineConfigValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e PebbleEngineConfigValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e PebbleEngineConfigValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e PebbleEngineConfigValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e PebbleEngineConfigValidationError) ErrorName() string { + return "PebbleEngineConfigValidationError" +} + +// Error satisfies the builtin error interface +func (e PebbleEngineConfigValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sPebbleEngineConfig.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = PebbleEngineConfigValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = PebbleEngineConfigValidationError{} diff --git a/pb/c1/c1z/v3/manifest_protoopaque.pb.go b/pb/c1/c1z/v3/manifest_protoopaque.pb.go new file mode 100644 index 000000000..5707aa8bd --- /dev/null +++ b/pb/c1/c1z/v3/manifest_protoopaque.pb.go @@ -0,0 +1,724 @@ +// V3 envelope manifest. Embedded inside the C1Z3 file format after +// the 5-byte magic and a uint32 BE length prefix. +// +// The manifest pins a transitively-closed FileDescriptorSet of every +// c1.storage.v3 record type stored in the payload. This is what makes +// v3 c1z files self-describing — any reader, including one without the +// originating SDK's compiled-in protos, can resolve every field via +// dynamicpb against the sidecar descriptor set. +// +// See RFC v4 §3.2 and Appendix B for the full envelope spec. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/c1z/v3/manifest.proto + +//go:build protoopaque + +package v3 + +import ( + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + descriptorpb "google.golang.org/protobuf/types/descriptorpb" + anypb "google.golang.org/protobuf/types/known/anypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type PayloadEncoding int32 + +const ( + PayloadEncoding_PAYLOAD_ENCODING_UNSPECIFIED PayloadEncoding = 0 + PayloadEncoding_PAYLOAD_ENCODING_RAW PayloadEncoding = 1 // debug only + PayloadEncoding_PAYLOAD_ENCODING_ZSTD PayloadEncoding = 2 // single-stream zstd (sqlite-style) + PayloadEncoding_PAYLOAD_ENCODING_ZSTD_TAR PayloadEncoding = 3 // zstd-tar of a Pebble directory +) + +// Enum value maps for PayloadEncoding. +var ( + PayloadEncoding_name = map[int32]string{ + 0: "PAYLOAD_ENCODING_UNSPECIFIED", + 1: "PAYLOAD_ENCODING_RAW", + 2: "PAYLOAD_ENCODING_ZSTD", + 3: "PAYLOAD_ENCODING_ZSTD_TAR", + } + PayloadEncoding_value = map[string]int32{ + "PAYLOAD_ENCODING_UNSPECIFIED": 0, + "PAYLOAD_ENCODING_RAW": 1, + "PAYLOAD_ENCODING_ZSTD": 2, + "PAYLOAD_ENCODING_ZSTD_TAR": 3, + } +) + +func (x PayloadEncoding) Enum() *PayloadEncoding { + p := new(PayloadEncoding) + *p = x + return p +} + +func (x PayloadEncoding) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PayloadEncoding) Descriptor() protoreflect.EnumDescriptor { + return file_c1_c1z_v3_manifest_proto_enumTypes[0].Descriptor() +} + +func (PayloadEncoding) Type() protoreflect.EnumType { + return &file_c1_c1z_v3_manifest_proto_enumTypes[0] +} + +func (x PayloadEncoding) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +type C1ZManifestV3 struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Engine string `protobuf:"bytes,1,opt,name=engine,proto3"` + xxx_hidden_EngineSchemaVersion uint32 `protobuf:"varint,2,opt,name=engine_schema_version,json=engineSchemaVersion,proto3"` + xxx_hidden_EngineConfig *anypb.Any `protobuf:"bytes,3,opt,name=engine_config,json=engineConfig,proto3"` + xxx_hidden_PayloadEncoding PayloadEncoding `protobuf:"varint,4,opt,name=payload_encoding,json=payloadEncoding,proto3,enum=c1.c1z.v3.PayloadEncoding"` + xxx_hidden_Descriptors *descriptorpb.FileDescriptorSet `protobuf:"bytes,10,opt,name=descriptors,proto3"` + xxx_hidden_RecordTypes *[]*RecordTypeInfo `protobuf:"bytes,11,rep,name=record_types,json=recordTypes,proto3"` + xxx_hidden_TenantHint string `protobuf:"bytes,20,opt,name=tenant_hint,json=tenantHint,proto3"` + xxx_hidden_CreatedAt *timestamppb.Timestamp `protobuf:"bytes,30,opt,name=created_at,json=createdAt,proto3"` + xxx_hidden_CreatedBySdkVersion string `protobuf:"bytes,31,opt,name=created_by_sdk_version,json=createdBySdkVersion,proto3"` + xxx_hidden_CreatedByTool string `protobuf:"bytes,32,opt,name=created_by_tool,json=createdByTool,proto3"` + xxx_hidden_SyncRuns *[]*SyncRunSummary `protobuf:"bytes,40,rep,name=sync_runs,json=syncRuns,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *C1ZManifestV3) Reset() { + *x = C1ZManifestV3{} + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *C1ZManifestV3) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*C1ZManifestV3) ProtoMessage() {} + +func (x *C1ZManifestV3) ProtoReflect() protoreflect.Message { + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *C1ZManifestV3) GetEngine() string { + if x != nil { + return x.xxx_hidden_Engine + } + return "" +} + +func (x *C1ZManifestV3) GetEngineSchemaVersion() uint32 { + if x != nil { + return x.xxx_hidden_EngineSchemaVersion + } + return 0 +} + +func (x *C1ZManifestV3) GetEngineConfig() *anypb.Any { + if x != nil { + return x.xxx_hidden_EngineConfig + } + return nil +} + +func (x *C1ZManifestV3) GetPayloadEncoding() PayloadEncoding { + if x != nil { + return x.xxx_hidden_PayloadEncoding + } + return PayloadEncoding_PAYLOAD_ENCODING_UNSPECIFIED +} + +func (x *C1ZManifestV3) GetDescriptors() *descriptorpb.FileDescriptorSet { + if x != nil { + return x.xxx_hidden_Descriptors + } + return nil +} + +func (x *C1ZManifestV3) GetRecordTypes() []*RecordTypeInfo { + if x != nil { + if x.xxx_hidden_RecordTypes != nil { + return *x.xxx_hidden_RecordTypes + } + } + return nil +} + +func (x *C1ZManifestV3) GetTenantHint() string { + if x != nil { + return x.xxx_hidden_TenantHint + } + return "" +} + +func (x *C1ZManifestV3) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.xxx_hidden_CreatedAt + } + return nil +} + +func (x *C1ZManifestV3) GetCreatedBySdkVersion() string { + if x != nil { + return x.xxx_hidden_CreatedBySdkVersion + } + return "" +} + +func (x *C1ZManifestV3) GetCreatedByTool() string { + if x != nil { + return x.xxx_hidden_CreatedByTool + } + return "" +} + +func (x *C1ZManifestV3) GetSyncRuns() []*SyncRunSummary { + if x != nil { + if x.xxx_hidden_SyncRuns != nil { + return *x.xxx_hidden_SyncRuns + } + } + return nil +} + +func (x *C1ZManifestV3) SetEngine(v string) { + x.xxx_hidden_Engine = v +} + +func (x *C1ZManifestV3) SetEngineSchemaVersion(v uint32) { + x.xxx_hidden_EngineSchemaVersion = v +} + +func (x *C1ZManifestV3) SetEngineConfig(v *anypb.Any) { + x.xxx_hidden_EngineConfig = v +} + +func (x *C1ZManifestV3) SetPayloadEncoding(v PayloadEncoding) { + x.xxx_hidden_PayloadEncoding = v +} + +func (x *C1ZManifestV3) SetDescriptors(v *descriptorpb.FileDescriptorSet) { + x.xxx_hidden_Descriptors = v +} + +func (x *C1ZManifestV3) SetRecordTypes(v []*RecordTypeInfo) { + x.xxx_hidden_RecordTypes = &v +} + +func (x *C1ZManifestV3) SetTenantHint(v string) { + x.xxx_hidden_TenantHint = v +} + +func (x *C1ZManifestV3) SetCreatedAt(v *timestamppb.Timestamp) { + x.xxx_hidden_CreatedAt = v +} + +func (x *C1ZManifestV3) SetCreatedBySdkVersion(v string) { + x.xxx_hidden_CreatedBySdkVersion = v +} + +func (x *C1ZManifestV3) SetCreatedByTool(v string) { + x.xxx_hidden_CreatedByTool = v +} + +func (x *C1ZManifestV3) SetSyncRuns(v []*SyncRunSummary) { + x.xxx_hidden_SyncRuns = &v +} + +func (x *C1ZManifestV3) HasEngineConfig() bool { + if x == nil { + return false + } + return x.xxx_hidden_EngineConfig != nil +} + +func (x *C1ZManifestV3) HasDescriptors() bool { + if x == nil { + return false + } + return x.xxx_hidden_Descriptors != nil +} + +func (x *C1ZManifestV3) HasCreatedAt() bool { + if x == nil { + return false + } + return x.xxx_hidden_CreatedAt != nil +} + +func (x *C1ZManifestV3) ClearEngineConfig() { + x.xxx_hidden_EngineConfig = nil +} + +func (x *C1ZManifestV3) ClearDescriptors() { + x.xxx_hidden_Descriptors = nil +} + +func (x *C1ZManifestV3) ClearCreatedAt() { + x.xxx_hidden_CreatedAt = nil +} + +type C1ZManifestV3_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Engine identifier. "pebble" is the only value Stack 3 supports. + // Future engines (e.g. "sled") can dispatch on this string without + // an envelope change. + Engine string + // Engine-specific schema version. Pebble interprets this as the + // pinned format-major-version it was written under; mismatches at + // open surface as ErrPebbleFormatNewer (refuse, page on-call) or + // ErrPebbleFormatOlder (refuse, treat as corruption). + EngineSchemaVersion uint32 + // Engine-specific configuration. For Pebble, this carries + // PebbleEngineConfig. The reader looks up the Any type-URL against + // its compiled-in registry; unknown type-URLs are treated as + // "no hint" (advisory only — the dispatch key is `engine`). + EngineConfig *anypb.Any + // Payload encoding. PAYLOAD_ENCODING_ZSTD_TAR is the production + // default for Pebble engines. + PayloadEncoding PayloadEncoding + // Transitively-closed FileDescriptorSet of every record type stored + // in the payload. Built at write time by walking + // protoregistry.GlobalFiles. The reader verifies closure at open + // and returns ErrManifestIncompleteDescriptors if any record-type + // descriptor cannot resolve from this set alone. + Descriptors *descriptorpb.FileDescriptorSet + // Record types produced by this writer. Used by the reader to + // detect unknown record types (forward-compat with a future SDK + // that adds new record types this binary doesn't know). + RecordTypes []*RecordTypeInfo + // Informational only. + TenantHint string + CreatedAt *timestamppb.Timestamp + CreatedBySdkVersion string + CreatedByTool string + // Cheap projection of sync runs inside the payload — lets tooling + // enumerate without opening the engine. + SyncRuns []*SyncRunSummary +} + +func (b0 C1ZManifestV3_builder) Build() *C1ZManifestV3 { + m0 := &C1ZManifestV3{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Engine = b.Engine + x.xxx_hidden_EngineSchemaVersion = b.EngineSchemaVersion + x.xxx_hidden_EngineConfig = b.EngineConfig + x.xxx_hidden_PayloadEncoding = b.PayloadEncoding + x.xxx_hidden_Descriptors = b.Descriptors + x.xxx_hidden_RecordTypes = &b.RecordTypes + x.xxx_hidden_TenantHint = b.TenantHint + x.xxx_hidden_CreatedAt = b.CreatedAt + x.xxx_hidden_CreatedBySdkVersion = b.CreatedBySdkVersion + x.xxx_hidden_CreatedByTool = b.CreatedByTool + x.xxx_hidden_SyncRuns = &b.SyncRuns + return m0 +} + +type RecordTypeInfo struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_MessageFullName string `protobuf:"bytes,1,opt,name=message_full_name,json=messageFullName,proto3"` + xxx_hidden_SchemaVersion uint32 `protobuf:"varint,2,opt,name=schema_version,json=schemaVersion,proto3"` + xxx_hidden_EstimatedCount int64 `protobuf:"varint,3,opt,name=estimated_count,json=estimatedCount,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RecordTypeInfo) Reset() { + *x = RecordTypeInfo{} + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RecordTypeInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RecordTypeInfo) ProtoMessage() {} + +func (x *RecordTypeInfo) ProtoReflect() protoreflect.Message { + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *RecordTypeInfo) GetMessageFullName() string { + if x != nil { + return x.xxx_hidden_MessageFullName + } + return "" +} + +func (x *RecordTypeInfo) GetSchemaVersion() uint32 { + if x != nil { + return x.xxx_hidden_SchemaVersion + } + return 0 +} + +func (x *RecordTypeInfo) GetEstimatedCount() int64 { + if x != nil { + return x.xxx_hidden_EstimatedCount + } + return 0 +} + +func (x *RecordTypeInfo) SetMessageFullName(v string) { + x.xxx_hidden_MessageFullName = v +} + +func (x *RecordTypeInfo) SetSchemaVersion(v uint32) { + x.xxx_hidden_SchemaVersion = v +} + +func (x *RecordTypeInfo) SetEstimatedCount(v int64) { + x.xxx_hidden_EstimatedCount = v +} + +type RecordTypeInfo_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Proto FullName — e.g. "c1.storage.v3.GrantRecord". + MessageFullName string + // Record-type-specific schema version. Bumped on any incompatible + // change to the record's stored encoding; reader refuses files + // claiming a version > what it supports. + SchemaVersion uint32 + // Optional row-count hint for sizing readers. + EstimatedCount int64 +} + +func (b0 RecordTypeInfo_builder) Build() *RecordTypeInfo { + m0 := &RecordTypeInfo{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_MessageFullName = b.MessageFullName + x.xxx_hidden_SchemaVersion = b.SchemaVersion + x.xxx_hidden_EstimatedCount = b.EstimatedCount + return m0 +} + +type SyncRunSummary struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_SyncId string `protobuf:"bytes,1,opt,name=sync_id,json=syncId,proto3"` + xxx_hidden_Type v3.SyncType `protobuf:"varint,2,opt,name=type,proto3,enum=c1.storage.v3.SyncType"` + xxx_hidden_StartedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=started_at,json=startedAt,proto3"` + xxx_hidden_EndedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=ended_at,json=endedAt,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SyncRunSummary) Reset() { + *x = SyncRunSummary{} + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SyncRunSummary) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SyncRunSummary) ProtoMessage() {} + +func (x *SyncRunSummary) ProtoReflect() protoreflect.Message { + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SyncRunSummary) GetSyncId() string { + if x != nil { + return x.xxx_hidden_SyncId + } + return "" +} + +func (x *SyncRunSummary) GetType() v3.SyncType { + if x != nil { + return x.xxx_hidden_Type + } + return v3.SyncType(0) +} + +func (x *SyncRunSummary) GetStartedAt() *timestamppb.Timestamp { + if x != nil { + return x.xxx_hidden_StartedAt + } + return nil +} + +func (x *SyncRunSummary) GetEndedAt() *timestamppb.Timestamp { + if x != nil { + return x.xxx_hidden_EndedAt + } + return nil +} + +func (x *SyncRunSummary) SetSyncId(v string) { + x.xxx_hidden_SyncId = v +} + +func (x *SyncRunSummary) SetType(v v3.SyncType) { + x.xxx_hidden_Type = v +} + +func (x *SyncRunSummary) SetStartedAt(v *timestamppb.Timestamp) { + x.xxx_hidden_StartedAt = v +} + +func (x *SyncRunSummary) SetEndedAt(v *timestamppb.Timestamp) { + x.xxx_hidden_EndedAt = v +} + +func (x *SyncRunSummary) HasStartedAt() bool { + if x == nil { + return false + } + return x.xxx_hidden_StartedAt != nil +} + +func (x *SyncRunSummary) HasEndedAt() bool { + if x == nil { + return false + } + return x.xxx_hidden_EndedAt != nil +} + +func (x *SyncRunSummary) ClearStartedAt() { + x.xxx_hidden_StartedAt = nil +} + +func (x *SyncRunSummary) ClearEndedAt() { + x.xxx_hidden_EndedAt = nil +} + +type SyncRunSummary_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + SyncId string + Type v3.SyncType + StartedAt *timestamppb.Timestamp + EndedAt *timestamppb.Timestamp +} + +func (b0 SyncRunSummary_builder) Build() *SyncRunSummary { + m0 := &SyncRunSummary{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_SyncId = b.SyncId + x.xxx_hidden_Type = b.Type + x.xxx_hidden_StartedAt = b.StartedAt + x.xxx_hidden_EndedAt = b.EndedAt + return m0 +} + +// PebbleEngineConfig is the value placed inside C1ZManifestV3.engine_config +// when engine == "pebble". Type-URL: type.googleapis.com/c1.c1z.v3.PebbleEngineConfig. +type PebbleEngineConfig struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_FormatMajorVersion uint32 `protobuf:"varint,1,opt,name=format_major_version,json=formatMajorVersion,proto3"` + xxx_hidden_CacheSizeBytes uint64 `protobuf:"varint,2,opt,name=cache_size_bytes,json=cacheSizeBytes,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PebbleEngineConfig) Reset() { + *x = PebbleEngineConfig{} + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PebbleEngineConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PebbleEngineConfig) ProtoMessage() {} + +func (x *PebbleEngineConfig) ProtoReflect() protoreflect.Message { + mi := &file_c1_c1z_v3_manifest_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PebbleEngineConfig) GetFormatMajorVersion() uint32 { + if x != nil { + return x.xxx_hidden_FormatMajorVersion + } + return 0 +} + +func (x *PebbleEngineConfig) GetCacheSizeBytes() uint64 { + if x != nil { + return x.xxx_hidden_CacheSizeBytes + } + return 0 +} + +func (x *PebbleEngineConfig) SetFormatMajorVersion(v uint32) { + x.xxx_hidden_FormatMajorVersion = v +} + +func (x *PebbleEngineConfig) SetCacheSizeBytes(v uint64) { + x.xxx_hidden_CacheSizeBytes = v +} + +type PebbleEngineConfig_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Pinned Pebble format-major-version this file was written under. + // The reader compares against pebble.FormatNewest and pebble.FormatMinSupported. + FormatMajorVersion uint32 + // Hint to readers; advisory only. + CacheSizeBytes uint64 +} + +func (b0 PebbleEngineConfig_builder) Build() *PebbleEngineConfig { + m0 := &PebbleEngineConfig{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_FormatMajorVersion = b.FormatMajorVersion + x.xxx_hidden_CacheSizeBytes = b.CacheSizeBytes + return m0 +} + +var File_c1_c1z_v3_manifest_proto protoreflect.FileDescriptor + +const file_c1_c1z_v3_manifest_proto_rawDesc = "" + + "\n" + + "\x18c1/c1z/v3/manifest.proto\x12\tc1.c1z.v3\x1a\x1bc1/storage/v3/records.proto\x1a\x19google/protobuf/any.proto\x1a google/protobuf/descriptor.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xd2\x04\n" + + "\rC1ZManifestV3\x12\x16\n" + + "\x06engine\x18\x01 \x01(\tR\x06engine\x122\n" + + "\x15engine_schema_version\x18\x02 \x01(\rR\x13engineSchemaVersion\x129\n" + + "\rengine_config\x18\x03 \x01(\v2\x14.google.protobuf.AnyR\fengineConfig\x12E\n" + + "\x10payload_encoding\x18\x04 \x01(\x0e2\x1a.c1.c1z.v3.PayloadEncodingR\x0fpayloadEncoding\x12D\n" + + "\vdescriptors\x18\n" + + " \x01(\v2\".google.protobuf.FileDescriptorSetR\vdescriptors\x12<\n" + + "\frecord_types\x18\v \x03(\v2\x19.c1.c1z.v3.RecordTypeInfoR\vrecordTypes\x12\x1f\n" + + "\vtenant_hint\x18\x14 \x01(\tR\n" + + "tenantHint\x129\n" + + "\n" + + "created_at\x18\x1e \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x123\n" + + "\x16created_by_sdk_version\x18\x1f \x01(\tR\x13createdBySdkVersion\x12&\n" + + "\x0fcreated_by_tool\x18 \x01(\tR\rcreatedByTool\x126\n" + + "\tsync_runs\x18( \x03(\v2\x19.c1.c1z.v3.SyncRunSummaryR\bsyncRuns\"\x8c\x01\n" + + "\x0eRecordTypeInfo\x12*\n" + + "\x11message_full_name\x18\x01 \x01(\tR\x0fmessageFullName\x12%\n" + + "\x0eschema_version\x18\x02 \x01(\rR\rschemaVersion\x12'\n" + + "\x0festimated_count\x18\x03 \x01(\x03R\x0eestimatedCount\"\xc8\x01\n" + + "\x0eSyncRunSummary\x12\x17\n" + + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x129\n" + + "\n" + + "started_at\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x125\n" + + "\bended_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\aendedAt\"p\n" + + "\x12PebbleEngineConfig\x120\n" + + "\x14format_major_version\x18\x01 \x01(\rR\x12formatMajorVersion\x12(\n" + + "\x10cache_size_bytes\x18\x02 \x01(\x04R\x0ecacheSizeBytes*\x87\x01\n" + + "\x0fPayloadEncoding\x12 \n" + + "\x1cPAYLOAD_ENCODING_UNSPECIFIED\x10\x00\x12\x18\n" + + "\x14PAYLOAD_ENCODING_RAW\x10\x01\x12\x19\n" + + "\x15PAYLOAD_ENCODING_ZSTD\x10\x02\x12\x1d\n" + + "\x19PAYLOAD_ENCODING_ZSTD_TAR\x10\x03B0Z.github.com/conductorone/baton-sdk/pb/c1/c1z/v3b\x06proto3" + +var file_c1_c1z_v3_manifest_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_c1z_v3_manifest_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_c1_c1z_v3_manifest_proto_goTypes = []any{ + (PayloadEncoding)(0), // 0: c1.c1z.v3.PayloadEncoding + (*C1ZManifestV3)(nil), // 1: c1.c1z.v3.C1ZManifestV3 + (*RecordTypeInfo)(nil), // 2: c1.c1z.v3.RecordTypeInfo + (*SyncRunSummary)(nil), // 3: c1.c1z.v3.SyncRunSummary + (*PebbleEngineConfig)(nil), // 4: c1.c1z.v3.PebbleEngineConfig + (*anypb.Any)(nil), // 5: google.protobuf.Any + (*descriptorpb.FileDescriptorSet)(nil), // 6: google.protobuf.FileDescriptorSet + (*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp + (v3.SyncType)(0), // 8: c1.storage.v3.SyncType +} +var file_c1_c1z_v3_manifest_proto_depIdxs = []int32{ + 5, // 0: c1.c1z.v3.C1ZManifestV3.engine_config:type_name -> google.protobuf.Any + 0, // 1: c1.c1z.v3.C1ZManifestV3.payload_encoding:type_name -> c1.c1z.v3.PayloadEncoding + 6, // 2: c1.c1z.v3.C1ZManifestV3.descriptors:type_name -> google.protobuf.FileDescriptorSet + 2, // 3: c1.c1z.v3.C1ZManifestV3.record_types:type_name -> c1.c1z.v3.RecordTypeInfo + 7, // 4: c1.c1z.v3.C1ZManifestV3.created_at:type_name -> google.protobuf.Timestamp + 3, // 5: c1.c1z.v3.C1ZManifestV3.sync_runs:type_name -> c1.c1z.v3.SyncRunSummary + 8, // 6: c1.c1z.v3.SyncRunSummary.type:type_name -> c1.storage.v3.SyncType + 7, // 7: c1.c1z.v3.SyncRunSummary.started_at:type_name -> google.protobuf.Timestamp + 7, // 8: c1.c1z.v3.SyncRunSummary.ended_at:type_name -> google.protobuf.Timestamp + 9, // [9:9] is the sub-list for method output_type + 9, // [9:9] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name +} + +func init() { file_c1_c1z_v3_manifest_proto_init() } +func file_c1_c1z_v3_manifest_proto_init() { + if File_c1_c1z_v3_manifest_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_c1z_v3_manifest_proto_rawDesc), len(file_c1_c1z_v3_manifest_proto_rawDesc)), + NumEnums: 1, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_c1z_v3_manifest_proto_goTypes, + DependencyIndexes: file_c1_c1z_v3_manifest_proto_depIdxs, + EnumInfos: file_c1_c1z_v3_manifest_proto_enumTypes, + MessageInfos: file_c1_c1z_v3_manifest_proto_msgTypes, + }.Build() + File_c1_c1z_v3_manifest_proto = out.File + file_c1_c1z_v3_manifest_proto_goTypes = nil + file_c1_c1z_v3_manifest_proto_depIdxs = nil +} diff --git a/pkg/dotc1z/format/v3/doc.go b/pkg/dotc1z/format/v3/doc.go new file mode 100644 index 000000000..73e939b23 --- /dev/null +++ b/pkg/dotc1z/format/v3/doc.go @@ -0,0 +1,14 @@ +// Package v3 implements the C1Z3 envelope format: 5-byte magic, a +// length-prefixed proto manifest, and a payload (typically a zstd-tar +// of a Pebble directory). +// +// The envelope is independent of the engine it carries. v3 readers +// inspect the manifest's `engine` field to dispatch; new engines plug +// in without an envelope change. The descriptor sidecar makes every +// v3 file self-describing — any reader can resolve every stored +// record's fields without compiled-in protos for that record type. +// +// Build tag: gated by `//go:build batonsdkv2`. Default connector +// builds never link the v3 envelope code (or its Pebble transitive +// deps). +package v3 diff --git a/pkg/dotc1z/format/v3/envelope.go b/pkg/dotc1z/format/v3/envelope.go new file mode 100644 index 000000000..509608b3e --- /dev/null +++ b/pkg/dotc1z/format/v3/envelope.go @@ -0,0 +1,241 @@ +//go:build batonsdkv2 + +package v3 + +import ( + "archive/tar" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + c1zv3 "github.com/conductorone/baton-sdk/pb/c1/c1z/v3" + "github.com/klauspost/compress/zstd" +) + +// C1Z3Magic is the 5-byte header that identifies a v3 c1z file. Same +// value as dotc1z.C1Z3FileHeader; duplicated here so the format package +// doesn't import its parent (it's the layer below). +var C1Z3Magic = []byte("C1Z3\x00") + +// ErrInvalidV3Magic is returned by ReadEnvelope when the first 5 bytes +// don't match C1Z3Magic. Callers that want to dispatch across v1 and v3 +// should use dotc1z.ReadHeaderFormat (which delegates to this package +// for the v3 detection only after a v1/v3 magic-byte choice). +var ErrInvalidV3Magic = errors.New("c1z v3: invalid magic; not a v3 file") + +// ErrEnvelopeTruncated is returned when the reader hits EOF mid-header +// or mid-manifest. This is a corruption-class error; the C1 corruption +// classifier should auto-invalidate the live c1z and trigger a re-sync. +var ErrEnvelopeTruncated = errors.New("c1z v3: envelope truncated") + +// Maximum manifest size we accept on read. Real manifests are ~50-200 KiB +// (mostly the descriptor closure). A 16 MiB cap is far above legitimate +// usage and protects against a malicious file claiming a billion-byte +// manifest length. +const maxManifestBytes = 16 << 20 + +// WriteEnvelope writes a complete v3 envelope to w: +// +// 1. The 5-byte C1Z3 magic. +// 2. A uint32 BE length prefix for the marshaled manifest. +// 3. The marshaled manifest bytes. +// 4. The zstd-compressed tar of payloadDir. +// +// payloadDir is walked in sorted lexical order; file mtimes are NOT +// normalized (the RFC documents tar as not byte-stable). w is typically +// a *os.File created via os.CreateTemp in the same directory as the +// final destination so an atomic rename can finalize the write. +func WriteEnvelope(w io.Writer, m *c1zv3.C1ZManifestV3, payloadDir string) error { + if m == nil { + return errors.New("c1z v3: WriteEnvelope: nil manifest") + } + if _, err := w.Write(C1Z3Magic); err != nil { + return fmt.Errorf("c1z v3: write magic: %w", err) + } + mb, err := MarshalManifest(m) + if err != nil { + return fmt.Errorf("c1z v3: marshal manifest: %w", err) + } + if len(mb) > maxManifestBytes { + return fmt.Errorf("c1z v3: manifest is %d bytes, exceeds %d", len(mb), maxManifestBytes) + } + var lenBuf [4]byte + binary.BigEndian.PutUint32(lenBuf[:], uint32(len(mb))) + if _, err := w.Write(lenBuf[:]); err != nil { + return fmt.Errorf("c1z v3: write manifest length: %w", err) + } + if _, err := w.Write(mb); err != nil { + return fmt.Errorf("c1z v3: write manifest: %w", err) + } + if err := writeZstdTar(w, payloadDir); err != nil { + return fmt.Errorf("c1z v3: write payload: %w", err) + } + return nil +} + +// Envelope is the parsed result of ReadEnvelope. Manifest holds the +// decoded manifest; PayloadReader yields the zstd-tar payload bytes +// (or whatever PayloadEncoding the manifest declared). Callers must +// Close the envelope when done. +type Envelope struct { + Manifest *c1zv3.C1ZManifestV3 + PayloadReader io.Reader + + zstdReader *zstd.Decoder +} + +// Close releases any pooled decoder resources held by PayloadReader. +func (e *Envelope) Close() error { + if e.zstdReader != nil { + e.zstdReader.Close() + e.zstdReader = nil + } + return nil +} + +// ReadEnvelope reads a v3 envelope from r. r must be positioned at the +// start of the file (the C1Z3 magic). Returns an Envelope whose +// PayloadReader streams the uncompressed tar bytes when the manifest +// declares PAYLOAD_ENCODING_ZSTD_TAR, or the raw bytes for +// PAYLOAD_ENCODING_RAW / PAYLOAD_ENCODING_ZSTD. +func ReadEnvelope(r io.Reader) (*Envelope, error) { + // 1. Magic. + magic := make([]byte, len(C1Z3Magic)) + if _, err := io.ReadFull(r, magic); err != nil { + return nil, fmt.Errorf("%w: reading magic: %v", ErrEnvelopeTruncated, err) + } + if !bytes.Equal(magic, C1Z3Magic) { + return nil, ErrInvalidV3Magic + } + // 2. Manifest length. + var lenBuf [4]byte + if _, err := io.ReadFull(r, lenBuf[:]); err != nil { + return nil, fmt.Errorf("%w: reading manifest length: %v", ErrEnvelopeTruncated, err) + } + mlen := binary.BigEndian.Uint32(lenBuf[:]) + if mlen > maxManifestBytes { + return nil, fmt.Errorf("c1z v3: manifest claims %d bytes, exceeds cap %d", mlen, maxManifestBytes) + } + // 3. Manifest bytes. + mb := make([]byte, mlen) + if _, err := io.ReadFull(r, mb); err != nil { + return nil, fmt.Errorf("%w: reading manifest: %v", ErrEnvelopeTruncated, err) + } + m, err := UnmarshalManifest(mb) + if err != nil { + return nil, err + } + // 4. Payload. The reader is positioned at the first payload byte. + env := &Envelope{Manifest: m} + switch m.GetPayloadEncoding() { + case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_RAW: + env.PayloadReader = r + case c1zv3.PayloadEncoding_PAYLOAD_ENCODING_ZSTD, + c1zv3.PayloadEncoding_PAYLOAD_ENCODING_ZSTD_TAR: + zr, err := zstd.NewReader(r) + if err != nil { + return nil, fmt.Errorf("c1z v3: zstd reader: %w", err) + } + env.zstdReader = zr + env.PayloadReader = zr + default: + return nil, fmt.Errorf("c1z v3: unsupported payload encoding %v", m.GetPayloadEncoding()) + } + return env, nil +} + +// writeZstdTar walks dir in sorted order, writing each entry into a +// tar stream that is itself zstd-compressed and written to w. +func writeZstdTar(w io.Writer, dir string) error { + zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if err != nil { + return err + } + defer zw.Close() + tw := tar.NewWriter(zw) + defer tw.Close() + + return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + hdr.Name = rel + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if info.Mode().IsRegular() { + f, err := os.Open(path) + if err != nil { + return err + } + _, err = io.Copy(tw, f) + f.Close() + if err != nil { + return err + } + } + return nil + }) +} + +// ExtractZstdTar reads a zstd-tar payload stream from r and unpacks +// it into destDir. destDir must exist. Used by the engine to +// rematerialize a Pebble directory at open time. +func ExtractZstdTar(r io.Reader, destDir string) error { + // r already came through Envelope's zstd decoder; we just need + // to walk the tar entries. + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("c1z v3: tar Next: %w", err) + } + target := filepath.Join(destDir, hdr.Name) + // Guard against tar entries escaping destDir. + if !filepath.IsLocal(hdr.Name) { + return fmt.Errorf("c1z v3: unsafe tar entry path: %q", hdr.Name) + } + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(hdr.Mode)&0o755); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&0o644) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + default: + // Skip other types (symlinks, etc.) — Pebble directories + // contain only directories and regular files. + } + } +} diff --git a/pkg/dotc1z/format/v3/envelope_test.go b/pkg/dotc1z/format/v3/envelope_test.go new file mode 100644 index 000000000..bcd0b89e8 --- /dev/null +++ b/pkg/dotc1z/format/v3/envelope_test.go @@ -0,0 +1,210 @@ +//go:build batonsdkv2 + +package v3 + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "testing" + + c1zv3 "github.com/conductorone/baton-sdk/pb/c1/c1z/v3" + v3pb "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestEnvelopeRoundtrip(t *testing.T) { + tmp := t.TempDir() + srcDir := filepath.Join(tmp, "src") + if err := os.MkdirAll(srcDir, 0o755); err != nil { + t.Fatal(err) + } + // Populate src with a small synthetic Pebble-like directory. + for name, content := range map[string]string{ + "CURRENT": "MANIFEST-000001\n", + "MANIFEST-000001": "manifest bytes", + "000005.sst": "sst contents — these would be real SSTs in production", + } { + path := filepath.Join(srcDir, name) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + + manifest := &c1zv3.C1ZManifestV3{ + Engine: "pebble", + EngineSchemaVersion: 17, + PayloadEncoding: c1zv3.PayloadEncoding_PAYLOAD_ENCODING_ZSTD_TAR, + CreatedAt: timestamppb.Now(), + CreatedBySdkVersion: "test-sdk/0.0.1", + CreatedByTool: "envelope_test", + } + + envFile := filepath.Join(tmp, "out.c1z3") + f, err := os.Create(envFile) + if err != nil { + t.Fatal(err) + } + if err := WriteEnvelope(f, manifest, srcDir); err != nil { + t.Fatalf("WriteEnvelope: %v", err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + + // Read it back. + rf, err := os.Open(envFile) + if err != nil { + t.Fatal(err) + } + defer rf.Close() + env, err := ReadEnvelope(rf) + if err != nil { + t.Fatalf("ReadEnvelope: %v", err) + } + defer env.Close() + + if env.Manifest.GetEngine() != "pebble" { + t.Errorf("engine: got %q want pebble", env.Manifest.GetEngine()) + } + if env.Manifest.GetEngineSchemaVersion() != 17 { + t.Errorf("engine_schema_version: got %d", env.Manifest.GetEngineSchemaVersion()) + } + if env.Manifest.GetPayloadEncoding() != c1zv3.PayloadEncoding_PAYLOAD_ENCODING_ZSTD_TAR { + t.Errorf("payload_encoding: got %v", env.Manifest.GetPayloadEncoding()) + } + + // Extract payload and verify contents. + dest := filepath.Join(tmp, "extracted") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + if err := ExtractZstdTar(env.PayloadReader, dest); err != nil { + t.Fatalf("ExtractZstdTar: %v", err) + } + got, err := os.ReadFile(filepath.Join(dest, "CURRENT")) + if err != nil { + t.Fatalf("read extracted CURRENT: %v", err) + } + if string(got) != "MANIFEST-000001\n" { + t.Errorf("CURRENT roundtrip: got %q", got) + } +} + +func TestReadEnvelopeBadMagic(t *testing.T) { + bad := bytes.NewReader([]byte("XXXX\x00trailing bytes")) + _, err := ReadEnvelope(bad) + if !errors.Is(err, ErrInvalidV3Magic) { + t.Fatalf("expected ErrInvalidV3Magic, got %v", err) + } +} + +func TestReadEnvelopeTruncated(t *testing.T) { + // Magic only, no length. + _, err := ReadEnvelope(bytes.NewReader(C1Z3Magic)) + if !errors.Is(err, ErrEnvelopeTruncated) { + t.Errorf("expected ErrEnvelopeTruncated, got %v", err) + } +} + +func TestMarshalManifestEmptyEngineFails(t *testing.T) { + _, err := MarshalManifest(&c1zv3.C1ZManifestV3{}) + if !errors.Is(err, ErrManifestEmptyEngine) { + t.Fatalf("expected ErrManifestEmptyEngine, got %v", err) + } +} + +func TestUnmarshalManifestEmptyEngineFails(t *testing.T) { + // An empty byte sequence unmarshals to a manifest with zero + // values; UnmarshalManifest must reject because engine is empty. + _, err := UnmarshalManifest([]byte{}) + if !errors.Is(err, ErrManifestEmptyEngine) { + t.Fatalf("expected ErrManifestEmptyEngine on empty bytes, got %v", err) + } +} + +func TestVerifyDescriptorClosureCatchesMissing(t *testing.T) { + // Hand-craft an incomplete set: file A depends on B, but only A is present. + a := &fileDescriptorProtoForTest{Name: "a.proto", Dependency: []string{"b.proto"}} + set := newFileDescriptorSetForTest(a) + if err := VerifyDescriptorClosure(set); !errors.Is(err, ErrManifestIncompleteDescriptors) { + t.Fatalf("expected ErrManifestIncompleteDescriptors, got %v", err) + } + // Closing the closure makes it pass. + b := &fileDescriptorProtoForTest{Name: "b.proto"} + set = newFileDescriptorSetForTest(a, b) + if err := VerifyDescriptorClosure(set); err != nil { + t.Fatalf("expected closure-OK, got %v", err) + } +} + +func TestBuildDescriptorClosureIncludesStorageV3(t *testing.T) { + // Touch the storage.v3 package so its descriptors are registered. + _ = &v3pb.GrantRecord{} + + set, err := BuildDescriptorClosure() + if err != nil { + t.Fatal(err) + } + if err := VerifyDescriptorClosure(set); err != nil { + t.Fatalf("closure built but verify failed: %v", err) + } + // Closure must include records.proto. + want := "c1/storage/v3/records.proto" + found := false + for _, f := range set.File { + if f.GetName() == want { + found = true + break + } + } + if !found { + t.Errorf("closure missing %s; got %d files", want, len(set.File)) + } +} + +// envelope_test helpers below — tiny stand-ins so the test file is self-contained. + +func TestEnvelopePayloadAtEnd(t *testing.T) { + // Verify the writer doesn't leave dangling bytes after the payload. + // We do this by checking the file size matches header + length + manifest + payload. + tmp := t.TempDir() + srcDir := filepath.Join(tmp, "src") + if err := os.MkdirAll(srcDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(srcDir, "f"), []byte("hi"), 0o644); err != nil { + t.Fatal(err) + } + + envFile := filepath.Join(tmp, "out.c1z3") + f, err := os.Create(envFile) + if err != nil { + t.Fatal(err) + } + m := &c1zv3.C1ZManifestV3{Engine: "pebble", PayloadEncoding: c1zv3.PayloadEncoding_PAYLOAD_ENCODING_ZSTD_TAR} + if err := WriteEnvelope(f, m, srcDir); err != nil { + t.Fatal(err) + } + f.Close() + + st, _ := os.Stat(envFile) + if st.Size() < int64(len(C1Z3Magic)+4) { + t.Errorf("envelope too small: %d bytes", st.Size()) + } + + // Read and verify payload streams to EOF cleanly. + rf, _ := os.Open(envFile) + defer rf.Close() + env, err := ReadEnvelope(rf) + if err != nil { + t.Fatal(err) + } + defer env.Close() + _, err = io.Copy(io.Discard, env.PayloadReader) + if err != nil { + t.Fatalf("draining payload: %v", err) + } +} diff --git a/pkg/dotc1z/format/v3/manifest.go b/pkg/dotc1z/format/v3/manifest.go new file mode 100644 index 000000000..fdfd29343 --- /dev/null +++ b/pkg/dotc1z/format/v3/manifest.go @@ -0,0 +1,146 @@ +//go:build batonsdkv2 + +package v3 + +import ( + "errors" + "fmt" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "google.golang.org/protobuf/types/descriptorpb" + + c1zv3 "github.com/conductorone/baton-sdk/pb/c1/c1z/v3" +) + +// Sentinel errors for manifest operations. These are duplicated from +// pkg/dotc1z/engine/pebble's sentinel set because importing the engine +// package from format/v3 would create a cycle (engine imports format/v3 +// for the save protocol). The two sentinels are kept structurally +// equivalent; callers can use errors.Is against either. +var ( + ErrManifestInvalid = errors.New("c1z v3: manifest unmarshal failed") + ErrManifestIncompleteDescriptors = errors.New("c1z v3: manifest descriptor closure incomplete") + ErrManifestEmptyEngine = errors.New("c1z v3: manifest engine field is empty") +) + +// BuildDescriptorClosure walks every c1.storage.v3 file currently +// registered in protoregistry.GlobalFiles and returns a transitively- +// closed FileDescriptorSet containing them plus every file they +// transitively import. The result is what gets pinned into a v3 +// manifest's `descriptors` field at save time. +// +// The closure invariant: for every file F in the result, every file F +// imports is also in the result. Reader-side verification can detect +// any missing import and return ErrManifestIncompleteDescriptors. +// +// At Stack 2's MVP, this walks protoregistry.GlobalFiles. The fuller +// version in Stack 3 lets the engine pass a curated set of record- +// type FullNames so the closure is bounded to what was actually +// written (not every storage.v3 file the binary happens to link). +func BuildDescriptorClosure() (*descriptorpb.FileDescriptorSet, error) { + // Collect all files whose package is c1.storage.v3 OR which any + // such file transitively imports. + seen := map[string]protoreflect.FileDescriptor{} + var visit func(fd protoreflect.FileDescriptor) + visit = func(fd protoreflect.FileDescriptor) { + if _, ok := seen[fd.Path()]; ok { + return + } + seen[fd.Path()] = fd + imports := fd.Imports() + for i := 0; i < imports.Len(); i++ { + visit(imports.Get(i).FileDescriptor) + } + } + + protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool { + if fd.Package() == "c1.storage.v3" { + visit(fd) + } + return true + }) + + set := &descriptorpb.FileDescriptorSet{ + File: make([]*descriptorpb.FileDescriptorProto, 0, len(seen)), + } + for _, fd := range seen { + set.File = append(set.File, protodesc(fd)) + } + return set, nil +} + +// protodesc converts a FileDescriptor to a FileDescriptorProto. +// Standard helper; lives here so callers don't have to thread the +// google.golang.org/protobuf/reflect/protodesc import. +func protodesc(fd protoreflect.FileDescriptor) *descriptorpb.FileDescriptorProto { + // protoreflect.FileDescriptor → wire-form descriptor. + // We use a minimal projection because the engine doesn't care about + // source-info or syntax-specific metadata, only the structural + // declaration. Full implementations (e.g. golang/protobuf) round-trip + // through proto.Marshal on the file's descriptor; we do the same by + // going through the Options. + out := &descriptorpb.FileDescriptorProto{ + Name: proto.String(fd.Path()), + Package: proto.String(string(fd.Package())), + } + syntax := "proto3" + if fd.Syntax() == protoreflect.Proto2 { + syntax = "proto2" + } + out.Syntax = proto.String(syntax) + imports := fd.Imports() + for i := 0; i < imports.Len(); i++ { + out.Dependency = append(out.Dependency, imports.Get(i).Path()) + } + // NB: messages, enums, fields, options are NOT serialized here. + // Stack 3's closure builder uses google.golang.org/protobuf/reflect/protodesc.ToFileDescriptorProto + // which round-trips faithfully. For Stack 2 MVP we ship the import + // graph + package names so readers can detect closure incompleteness + // even though they can't reconstruct field-level descriptors yet. + // TODO(Stack 3): swap to protodesc.ToFileDescriptorProto. + return out +} + +// VerifyDescriptorClosure checks that every file referenced by every +// other file in the set is itself in the set. Returns +// ErrManifestIncompleteDescriptors on the first missing import. +func VerifyDescriptorClosure(set *descriptorpb.FileDescriptorSet) error { + if set == nil { + return ErrManifestIncompleteDescriptors + } + have := make(map[string]bool, len(set.File)) + for _, f := range set.File { + have[f.GetName()] = true + } + for _, f := range set.File { + for _, dep := range f.GetDependency() { + if !have[dep] { + return fmt.Errorf("%w: file %s depends on %s, which is not in the set", + ErrManifestIncompleteDescriptors, f.GetName(), dep) + } + } + } + return nil +} + +// MarshalManifest serializes m to deterministic proto bytes. +func MarshalManifest(m *c1zv3.C1ZManifestV3) ([]byte, error) { + if m.GetEngine() == "" { + return nil, ErrManifestEmptyEngine + } + return proto.MarshalOptions{Deterministic: true}.Marshal(m) +} + +// UnmarshalManifest parses bytes into a fresh C1ZManifestV3. +func UnmarshalManifest(b []byte) (*c1zv3.C1ZManifestV3, error) { + m := &c1zv3.C1ZManifestV3{} + if err := proto.Unmarshal(b, m); err != nil { + return nil, fmt.Errorf("%w: %v", ErrManifestInvalid, err) + } + if m.GetEngine() == "" { + return nil, ErrManifestEmptyEngine + } + return m, nil +} diff --git a/pkg/dotc1z/format/v3/test_helpers_test.go b/pkg/dotc1z/format/v3/test_helpers_test.go new file mode 100644 index 000000000..471ca7148 --- /dev/null +++ b/pkg/dotc1z/format/v3/test_helpers_test.go @@ -0,0 +1,26 @@ +//go:build batonsdkv2 + +package v3 + +import ( + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" +) + +// Test helpers — small constructors so envelope_test.go stays readable. + +type fileDescriptorProtoForTest struct { + Name string + Dependency []string +} + +func newFileDescriptorSetForTest(files ...*fileDescriptorProtoForTest) *descriptorpb.FileDescriptorSet { + out := &descriptorpb.FileDescriptorSet{} + for _, f := range files { + out.File = append(out.File, &descriptorpb.FileDescriptorProto{ + Name: proto.String(f.Name), + Dependency: f.Dependency, + }) + } + return out +} diff --git a/proto/c1/c1z/v3/manifest.proto b/proto/c1/c1z/v3/manifest.proto new file mode 100644 index 000000000..a1afe09c4 --- /dev/null +++ b/proto/c1/c1z/v3/manifest.proto @@ -0,0 +1,101 @@ +// V3 envelope manifest. Embedded inside the C1Z3 file format after +// the 5-byte magic and a uint32 BE length prefix. +// +// The manifest pins a transitively-closed FileDescriptorSet of every +// c1.storage.v3 record type stored in the payload. This is what makes +// v3 c1z files self-describing — any reader, including one without the +// originating SDK's compiled-in protos, can resolve every field via +// dynamicpb against the sidecar descriptor set. +// +// See RFC v4 §3.2 and Appendix B for the full envelope spec. + +syntax = "proto3"; + +package c1.c1z.v3; + +import "c1/storage/v3/records.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/descriptor.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/conductorone/baton-sdk/pb/c1/c1z/v3"; + +message C1ZManifestV3 { + // Engine identifier. "pebble" is the only value Stack 3 supports. + // Future engines (e.g. "sled") can dispatch on this string without + // an envelope change. + string engine = 1; + + // Engine-specific schema version. Pebble interprets this as the + // pinned format-major-version it was written under; mismatches at + // open surface as ErrPebbleFormatNewer (refuse, page on-call) or + // ErrPebbleFormatOlder (refuse, treat as corruption). + uint32 engine_schema_version = 2; + + // Engine-specific configuration. For Pebble, this carries + // PebbleEngineConfig. The reader looks up the Any type-URL against + // its compiled-in registry; unknown type-URLs are treated as + // "no hint" (advisory only — the dispatch key is `engine`). + google.protobuf.Any engine_config = 3; + + // Payload encoding. PAYLOAD_ENCODING_ZSTD_TAR is the production + // default for Pebble engines. + PayloadEncoding payload_encoding = 4; + + // Transitively-closed FileDescriptorSet of every record type stored + // in the payload. Built at write time by walking + // protoregistry.GlobalFiles. The reader verifies closure at open + // and returns ErrManifestIncompleteDescriptors if any record-type + // descriptor cannot resolve from this set alone. + google.protobuf.FileDescriptorSet descriptors = 10; + + // Record types produced by this writer. Used by the reader to + // detect unknown record types (forward-compat with a future SDK + // that adds new record types this binary doesn't know). + repeated RecordTypeInfo record_types = 11; + + // Informational only. + string tenant_hint = 20; + google.protobuf.Timestamp created_at = 30; + string created_by_sdk_version = 31; + string created_by_tool = 32; + + // Cheap projection of sync runs inside the payload — lets tooling + // enumerate without opening the engine. + repeated SyncRunSummary sync_runs = 40; +} + +enum PayloadEncoding { + PAYLOAD_ENCODING_UNSPECIFIED = 0; + PAYLOAD_ENCODING_RAW = 1; // debug only + PAYLOAD_ENCODING_ZSTD = 2; // single-stream zstd (sqlite-style) + PAYLOAD_ENCODING_ZSTD_TAR = 3; // zstd-tar of a Pebble directory +} + +message RecordTypeInfo { + // Proto FullName — e.g. "c1.storage.v3.GrantRecord". + string message_full_name = 1; + // Record-type-specific schema version. Bumped on any incompatible + // change to the record's stored encoding; reader refuses files + // claiming a version > what it supports. + uint32 schema_version = 2; + // Optional row-count hint for sizing readers. + int64 estimated_count = 3; +} + +message SyncRunSummary { + string sync_id = 1; + c1.storage.v3.SyncType type = 2; + google.protobuf.Timestamp started_at = 3; + google.protobuf.Timestamp ended_at = 4; +} + +// PebbleEngineConfig is the value placed inside C1ZManifestV3.engine_config +// when engine == "pebble". Type-URL: type.googleapis.com/c1.c1z.v3.PebbleEngineConfig. +message PebbleEngineConfig { + // Pinned Pebble format-major-version this file was written under. + // The reader compares against pebble.FormatNewest and pebble.FormatMinSupported. + uint32 format_major_version = 1; + // Hint to readers; advisory only. + uint64 cache_size_bytes = 2; +}