From a6705915c17e63d513b3047a315ef0b568602085 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 11 Jul 2026 06:42:14 +0900 Subject: [PATCH 01/16] encryption: project sidecar writer registry --- adapter/encryption_admin.go | 90 ++++++++--- adapter/encryption_admin_test.go | 142 +++++++++++++++++- ...6_04_29_partial_data_at_rest_encryption.md | 4 +- ...emented_7c_confchange_time_registration.md | 6 +- ...lemented_7d_sidecar_registry_projection.md | 52 +++++++ main.go | 2 + multiraft_runtime.go | 6 +- 7 files changed, 269 insertions(+), 33 deletions(-) create mode 100644 docs/design/2026_07_11_implemented_7d_sidecar_registry_projection.md diff --git a/adapter/encryption_admin.go b/adapter/encryption_admin.go index c35ccc894..d16e7bfda 100644 --- a/adapter/encryption_admin.go +++ b/adapter/encryption_admin.go @@ -30,6 +30,7 @@ import ( type EncryptionAdminServer struct { sidecarPath string fullNodeID uint64 + writerRegistry encryption.WriterRegistryStore buildSHA string latestAppliedIndex func() uint64 // proposer is the raw raft proposer used for cleartext-only @@ -182,6 +183,19 @@ func WithEncryptionAdminFullNodeID(id uint64) EncryptionAdminServerOption { } } +// WithEncryptionAdminWriterRegistry wires the §4.1 writer-registry +// store that read-only sidecar recovery RPCs use to project +// writer_registry_for_caller. A nil argument is a no-op so tests and +// encryption-disabled nodes keep the pre-Stage-7 empty-map posture. +func WithEncryptionAdminWriterRegistry(reg encryption.WriterRegistryStore) EncryptionAdminServerOption { + return func(s *EncryptionAdminServer) { + if reg == nil { + return + } + s.writerRegistry = reg + } +} + // WithEncryptionAdminBuildSHA overrides the auto-detected // runtime/debug build SHA. Tests use this to pin a deterministic // value; production wiring leaves it empty. @@ -374,10 +388,10 @@ func (s *EncryptionAdminServer) GetCapability(_ context.Context, _ *pb.Empty) (* // pointers; the wrapped material is leakage-safe because it is // KEK-wrapped, which is the same property the on-disk sidecar has. // -// The writer_registry_for_caller map is empty until Stage 7 wires -// the registry. Callers in the §7.1 cutover path tolerate an empty -// map because the §5.6 step 1a batch is sourced from the -// GetCapability fan-out, not from this RPC. +// When the writer registry is wired, writer_registry_for_caller +// carries this node's recorded last_seen_local_epoch per sidecar DEK. +// Unwired tests and encryption-disabled nodes keep the historical +// empty non-nil map. func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) (*pb.SidecarStateReport, error) { if s.sidecarPath == "" { return nil, grpcStatusError(codes.FailedPrecondition, "encryption: sidecar path is not configured on this node") @@ -386,6 +400,10 @@ func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) if err != nil { return nil, statusFromSidecarErr(err) } + writerRegistry, err := s.writerRegistryForCaller(sc, s.fullNodeID) + if err != nil { + return nil, err + } resp := &pb.SidecarStateReport{ ActiveStorageId: sc.Active.Storage, ActiveRaftId: sc.Active.Raft, @@ -393,7 +411,7 @@ func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) RaftEnvelopeCutoverIndex: sc.RaftEnvelopeCutoverIndex, LatestAppliedIndex: s.appliedIndex(sc.RaftAppliedIndex), WrappedDeksById: wrappedDEKMap(sc), - WriterRegistryForCaller: map[uint32]uint32{}, + WriterRegistryForCaller: writerRegistry, } return resp, nil } @@ -410,14 +428,6 @@ func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) // state to a recovering follower and silently overwrite recent // rotations. func (s *EncryptionAdminServer) ResyncSidecar(ctx context.Context, req *pb.ResyncSidecarRequest) (*pb.ResyncSidecarResponse, error) { - // req.CallerFullNodeId is intentionally unused for the - // recovery payload itself in PR-B; Stage 7 will use it to - // scope the writer-registry projection to that specific - // caller per §5.5. Recording it here keeps the field on the - // hot path so a future leader-side audit log can correlate - // resyncs to the requesting member without a wire-format - // change. - _ = req if err := s.requireLeader(ctx); err != nil { return nil, err } @@ -428,20 +438,62 @@ func (s *EncryptionAdminServer) ResyncSidecar(ctx context.Context, req *pb.Resyn if err != nil { return nil, statusFromSidecarErr(err) } + var callerFullNodeID uint64 + if req != nil { + callerFullNodeID = req.GetCallerFullNodeId() + } + writerRegistry, err := s.writerRegistryForCaller(sc, callerFullNodeID) + if err != nil { + return nil, err + } return &pb.ResyncSidecarResponse{ WrappedDeksById: wrappedDEKMap(sc), ActiveStorageId: sc.Active.Storage, ActiveRaftId: sc.Active.Raft, LeaderLatestAppliedIndex: s.appliedIndex(sc.RaftAppliedIndex), - // §5.5 follower-repair: leader's recorded - // last_seen_local_epoch per (dek_id, caller). Stage 7 - // fills this from the writer registry. PR-A returns an - // empty non-nil map because a node recovering before - // the registry exists has nothing to re-derive. - WriterRegistryForCaller: map[uint32]uint32{}, + WriterRegistryForCaller: writerRegistry, }, nil } +func (s *EncryptionAdminServer) writerRegistryForCaller(sc *encryption.Sidecar, fullNodeID uint64) (map[uint32]uint32, error) { + out := map[uint32]uint32{} + if s.writerRegistry == nil { + return out, nil + } + if fullNodeID == 0 { + return nil, grpcStatusError(codes.InvalidArgument, + "encryption: full_node_id is required to project writer_registry_for_caller") + } + nodeID16 := encryption.NodeID16(fullNodeID) + for idStr := range sc.Keys { + dekID, err := parseSidecarKeyID(idStr) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, + "encryption: sidecar key id %q could not be projected into writer registry: %v", idStr, err) + } + raw, ok, err := s.writerRegistry.GetRegistryRow(encryption.RegistryKey(dekID, nodeID16)) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, + "encryption: read writer registry row for dek_id=%d full_node_id=%#x: %v", dekID, fullNodeID, err) + } + if !ok { + continue + } + row, err := encryption.DecodeRegistryValue(raw) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, + "encryption: decode writer registry row for dek_id=%d full_node_id=%#x: %v", dekID, fullNodeID, err) + } + if row.FullNodeID != fullNodeID { + return nil, grpcStatusErrorf(codes.Internal, + "encryption: writer registry node_id collision for dek_id=%d caller_full_node_id=%#x registry_full_node_id=%#x", + dekID, fullNodeID, row.FullNodeID) + } + out[dekID] = uint32(row.LastSeenLocalEpoch) + } + return out, nil +} + func (s *EncryptionAdminServer) appliedIndex(sidecarValue uint64) uint64 { if s.latestAppliedIndex == nil { return sidecarValue diff --git a/adapter/encryption_admin_test.go b/adapter/encryption_admin_test.go index 659fb06eb..6680c326b 100644 --- a/adapter/encryption_admin_test.go +++ b/adapter/encryption_admin_test.go @@ -159,6 +159,39 @@ func TestEncryptionAdmin_GetSidecarState_ShipsWrappedDEKs(t *testing.T) { assertSidecarWrappedDEKs(t, got) } +func TestEncryptionAdmin_GetSidecarState_ProjectsWriterRegistryForLocalNode(t *testing.T) { + t.Parallel() + const localFullNodeID uint64 = 0xCAFE_BABE_0000_1234 + path := writeSidecarFixture(t, &encryption.Sidecar{ + RaftAppliedIndex: 42, + Active: encryption.ActiveKeys{Storage: 1, Raft: 2}, + Keys: map[string]encryption.SidecarKey{ + "1": {Purpose: "storage", Wrapped: []byte("wrapped-1")}, + "2": {Purpose: "raft", Wrapped: []byte("wrapped-2")}, + "3": {Purpose: "storage", Wrapped: []byte("wrapped-old-storage")}, + }, + }) + reg := newTestWriterRegistry() + reg.seed(t, 1, localFullNodeID, 4, 9) + reg.seed(t, 2, localFullNodeID, 5, 10) + // No row for DEK 3: the projection omits absent rows instead of + // inventing a zero epoch that could be mistaken for a real record. + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminFullNodeID(localFullNodeID), + WithEncryptionAdminWriterRegistry(reg), + ) + + got, err := srv.GetSidecarState(context.Background(), &pb.Empty{}) + if err != nil { + t.Fatalf("GetSidecarState: %v", err) + } + want := map[uint32]uint32{1: 9, 2: 10} + if fmt.Sprint(got.WriterRegistryForCaller) != fmt.Sprint(want) { + t.Fatalf("WriterRegistryForCaller=%v, want %v", got.WriterRegistryForCaller, want) + } +} + func assertSidecarHeader(t *testing.T, got *pb.SidecarStateReport) { t.Helper() if got.ActiveStorageId != 1 || got.ActiveRaftId != 2 { @@ -184,10 +217,10 @@ func assertSidecarWrappedDEKs(t *testing.T, got *pb.SidecarStateReport) { t.Errorf("wrapped[2]=%q, want %q", got.WrappedDeksById[2], "wrapped-2") } if got.WriterRegistryForCaller == nil { - t.Errorf("WriterRegistryForCaller=nil, want empty non-nil map (PR-A contract)") + t.Errorf("WriterRegistryForCaller=nil, want empty non-nil map when registry is unwired") } if len(got.WriterRegistryForCaller) != 0 { - t.Errorf("WriterRegistryForCaller=%v, want empty in PR-A", got.WriterRegistryForCaller) + t.Errorf("WriterRegistryForCaller=%v, want empty when registry is unwired", got.WriterRegistryForCaller) } } @@ -245,14 +278,67 @@ func TestEncryptionAdmin_ResyncSidecar_ShipsWrappedDEKs(t *testing.T) { if string(got.WrappedDeksById[3]) != "ws" || string(got.WrappedDeksById[4]) != "wr" { t.Errorf("wrapped=%v, want id3=ws id4=wr", got.WrappedDeksById) } - // Mirror the GetSidecarState contract: non-nil empty map until - // Stage 7 wires the writer registry. Locks in the §5.5 promise - // so a future change to the field cannot silently degrade to nil. if got.WriterRegistryForCaller == nil { - t.Errorf("WriterRegistryForCaller=nil, want empty non-nil map (PR-A contract)") + t.Errorf("WriterRegistryForCaller=nil, want empty non-nil map when registry is unwired") } if len(got.WriterRegistryForCaller) != 0 { - t.Errorf("WriterRegistryForCaller=%v, want empty in PR-A", got.WriterRegistryForCaller) + t.Errorf("WriterRegistryForCaller=%v, want empty when registry is unwired", got.WriterRegistryForCaller) + } +} + +func TestEncryptionAdmin_ResyncSidecar_ProjectsWriterRegistryForCaller(t *testing.T) { + t.Parallel() + const callerFullNodeID uint64 = 0x1111_2222_3333_4444 + path := writeSidecarFixture(t, &encryption.Sidecar{ + RaftAppliedIndex: 17, + Active: encryption.ActiveKeys{Storage: 3, Raft: 4}, + Keys: map[string]encryption.SidecarKey{ + "3": {Purpose: "storage", Wrapped: []byte("ws")}, + "4": {Purpose: "raft", Wrapped: []byte("wr")}, + "5": {Purpose: "storage", Wrapped: []byte("old")}, + }, + }) + reg := newTestWriterRegistry() + reg.seed(t, 3, callerFullNodeID, 1, 6) + reg.seed(t, 4, callerFullNodeID, 2, 8) + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminLeaderView(stubLeaderView{state: raftengine.StateLeader}), + WithEncryptionAdminWriterRegistry(reg), + ) + + got, err := srv.ResyncSidecar(context.Background(), &pb.ResyncSidecarRequest{CallerFullNodeId: callerFullNodeID}) + if err != nil { + t.Fatalf("ResyncSidecar: %v", err) + } + want := map[uint32]uint32{3: 6, 4: 8} + if fmt.Sprint(got.WriterRegistryForCaller) != fmt.Sprint(want) { + t.Fatalf("WriterRegistryForCaller=%v, want %v", got.WriterRegistryForCaller, want) + } +} + +func TestEncryptionAdmin_ResyncSidecar_RejectsWriterRegistryNodeCollision(t *testing.T) { + t.Parallel() + const callerFullNodeID uint64 = 0x10001 + const collidingFullNodeID uint64 = 0x20001 + path := writeSidecarFixture(t, &encryption.Sidecar{ + Active: encryption.ActiveKeys{Storage: 7, Raft: 8}, + Keys: map[string]encryption.SidecarKey{ + "7": {Purpose: "storage", Wrapped: []byte("ws")}, + "8": {Purpose: "raft", Wrapped: []byte("wr")}, + }, + }) + reg := newTestWriterRegistry() + reg.seed(t, 7, collidingFullNodeID, 1, 2) + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminLeaderView(stubLeaderView{state: raftengine.StateLeader}), + WithEncryptionAdminWriterRegistry(reg), + ) + + _, err := srv.ResyncSidecar(context.Background(), &pb.ResyncSidecarRequest{CallerFullNodeId: callerFullNodeID}) + if status.Code(err) != codes.Internal { + t.Fatalf("ResyncSidecar status=%v, want Internal for writer-registry node collision (err=%v)", status.Code(err), err) } } @@ -1115,6 +1201,48 @@ func writeSidecarFixture(t *testing.T, sc *encryption.Sidecar) string { return path } +type testWriterRegistry struct { + rows map[string][]byte + getErr error +} + +func newTestWriterRegistry() *testWriterRegistry { + return &testWriterRegistry{rows: map[string][]byte{}} +} + +func (r *testWriterRegistry) GetRegistryRow(key []byte) ([]byte, bool, error) { + if r.getErr != nil { + return nil, false, r.getErr + } + raw, ok := r.rows[string(key)] + if !ok { + return nil, false, nil + } + out := append([]byte(nil), raw...) + return out, true, nil +} + +func (r *testWriterRegistry) SetRegistryRow(key, value []byte) error { + if r.rows == nil { + r.rows = map[string][]byte{} + } + r.rows[string(key)] = append([]byte(nil), value...) + return nil +} + +func (r *testWriterRegistry) seed(t *testing.T, dekID uint32, fullNodeID uint64, firstSeen, lastSeen uint16) { + t.Helper() + key := encryption.RegistryKey(dekID, encryption.NodeID16(fullNodeID)) + val := encryption.EncodeRegistryValue(encryption.RegistryValue{ + FullNodeID: fullNodeID, + FirstSeenLocalEpoch: firstSeen, + LastSeenLocalEpoch: lastSeen, + }) + if err := r.SetRegistryRow(key, val); err != nil { + t.Fatalf("seed registry row: %v", err) + } +} + // fixedCapabilityFanout returns a closure that yields the supplied // result regardless of context — lets tests drive the §4 fan-out // branches deterministically without spinning real clients. A diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 0d77a8784..2c53e0626 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -1,6 +1,6 @@ # Data-at-rest encryption for elastickv -Status: Partial — Stages 0–6 and Stage 8 shipped (5E deferred); Stage 7 partially shipped; Stage 9 open +Status: Partial — Stages 0–8 shipped (5E deferred); Stage 9 open Author: bootjp Date: 2026-04-29 @@ -30,7 +30,7 @@ Date: 2026-04-29 | 6D | §6.6 `enable-storage-envelope` admin RPC + §7.1 Phase-1 storage cutover (§6.2 toggle ON) + Voters ∪ Learners capability gate + production storage-envelope write-path wiring | shipped | `2026_05_18_implemented_6d_enable_storage_envelope.md` + `2026_05_25_implemented_6d6c2_production_storage_envelope_wiring.md` | | 6E | §6.6 `enable-raft-envelope` admin RPC + §7.1 Phase-2 raft cutover + `raft_envelope_cutover_index` sidecar record + `internal/raftengine/etcd/engine.go` `applyNormalEntry` unwrap hook activation + `ErrRaftUnwrapFailed` HaltApply path + `kv/coordinator.go` / `kv/sharded_coordinator.go` wrap-on-propose switch (Phase-2 leader-side §6.3 proposal-payload wrap) + §7.1 steps 1–6 proposal quiescence barrier (block new user proposal intake, drain in-flight queue, source-tag exemption for the cutover entry itself) + production runtime wiring for startup/apply-time wrap install and post-cutover admin proposals + 6C-4 fail-closed guards. | shipped | 2026_05_31_implemented_6e_enable_raft_envelope.md | | 6F | §6.5 `--encryption-rotate-on-startup` request flag + leader-elected rotation proposal on the default encryption Raft group. The leader rotates every active DEK purpose once before listeners open; followers keep an in-memory pending request and fire only if they acquire leadership in the same process uptime. | shipped | this PR | -| 7 | Writer registry + deterministic nonce (§4.1). Implemented slices: process-start registration, storage-layer registration gate, runtime re-registration for cutover and rotation, and conf-change-time pre-registration. Still open: §5.5 registry projection in `GetSidecarState` / `ResyncSidecar` (`WriterRegistryForCaller`). | partial | `2026_05_26_implemented_7a_process_start_registration.md` + `2026_05_26_implemented_7a2_storage_layer_registration_enforcement.md` + `2026_05_28_implemented_7b_runtime_reregistration.md` + `2026_05_28_implemented_7b_prime_runtime_reregistration_rotation.md` + `2026_05_29_implemented_7c_confchange_time_registration.md` | +| 7 | Writer registry + deterministic nonce (§4.1). Covers process-start registration, storage-layer registration gate, runtime re-registration for cutover and rotation, conf-change-time pre-registration, and §5.5 registry projection in `GetSidecarState` / `ResyncSidecar` (`WriterRegistryForCaller`). | shipped | `2026_05_26_implemented_7a_process_start_registration.md` + `2026_05_26_implemented_7a2_storage_layer_registration_enforcement.md` + `2026_05_28_implemented_7b_runtime_reregistration.md` + `2026_05_28_implemented_7b_prime_runtime_reregistration_rotation.md` + `2026_05_29_implemented_7c_confchange_time_registration.md` + `2026_07_11_implemented_7d_sidecar_registry_projection.md` | | 8 | Snapshot header v2 (§4.4); WAL coverage closure (§4.3 / §4.6) | shipped | [`2026_05_29_implemented_8a_snapshot_header_v2.md`](2026_05_29_implemented_8a_snapshot_header_v2.md) + [`2026_06_01_implemented_8b_wal_coverage_closure.md`](2026_06_01_implemented_8b_wal_coverage_closure.md) | | 9 | KMS-backed wrappers, compression, rotation/retire/rewrite, Jepsen (§5.2, §5.4, §6.4, §8) | open | — | diff --git a/docs/design/2026_05_29_implemented_7c_confchange_time_registration.md b/docs/design/2026_05_29_implemented_7c_confchange_time_registration.md index 6e13370f9..84113a53e 100644 --- a/docs/design/2026_05_29_implemented_7c_confchange_time_registration.md +++ b/docs/design/2026_05_29_implemented_7c_confchange_time_registration.md @@ -463,8 +463,8 @@ that bypass the standard workflow) must rely on the `ErrNodeIDCollision` startup membership pre-check before issuing the ConfChange. -This closes the 7c ConfChange-time registration slice. The parent -Stage 7 remains partial until the §5.5 `WriterRegistryForCaller` -projection is wired into `GetSidecarState` / `ResyncSidecar`. +This closes the 7c ConfChange-time registration slice. The remaining +Stage 7 §5.5 `WriterRegistryForCaller` projection is tracked by +`2026_07_11_implemented_7d_sidecar_registry_projection.md`. Stage 8 (snapshot header v2) and Stage 9 (KMS + compress + rotation/retire/rewrite + Jepsen) follow. diff --git a/docs/design/2026_07_11_implemented_7d_sidecar_registry_projection.md b/docs/design/2026_07_11_implemented_7d_sidecar_registry_projection.md new file mode 100644 index 000000000..b84f51a9a --- /dev/null +++ b/docs/design/2026_07_11_implemented_7d_sidecar_registry_projection.md @@ -0,0 +1,52 @@ +# Implemented 7d: sidecar registry projection + +Status: Implemented +Author: bootjp +Date: 2026-07-11 + +## Scope + +This closes the remaining Stage 7 §5.5 recovery surface for +`WriterRegistryForCaller`. + +The prior Stage 7 slices already registered writers at process start, +storage-layer admission, runtime cutover/rotation, and membership +change time. The open gap was the compaction-fallback recovery RPC: +a node whose sidecar fell behind a compacted Raft log needs the +leader's per-DEK `last_seen_local_epoch` record for that specific +caller so it can choose a strictly monotonic replacement +`local_epoch`. + +## Implementation + +`EncryptionAdminServer` now accepts the production +`encryption.WriterRegistryStore` through +`WithEncryptionAdminWriterRegistry`. + +`GetSidecarState` projects registry rows for the local full node ID. +`ResyncSidecar` projects rows for `ResyncSidecarRequest.caller_full_node_id`. +Both responses keep returning a non-nil empty map when the registry is +not wired, preserving the encryption-disabled and test posture. + +For each DEK present in the sidecar, the server reads +`RegistryKey(dek_id, NodeID16(full_node_id))` and returns the decoded +`LastSeenLocalEpoch`. Missing rows are omitted instead of zero-filled, +because zero could be mistaken for a real registry observation. A +decoded row whose stored `FullNodeID` does not match the requested +caller is rejected as an internal node-ID collision. + +Production wiring stores the writer registry on each +`raftGroupRuntime` as the group is built, then passes it into the +per-shard `EncryptionAdmin` registration in `startRaftServers`. + +## Validation + +- `go test ./adapter -run 'TestEncryptionAdmin_(GetSidecarState|ResyncSidecar)' -count=1 -timeout=240s` +- `go test ./adapter -run TestEncryptionAdmin -count=1 -timeout=300s` +- `go test ./store ./internal/encryption . -count=1 -timeout=180s` +- `golangci-lint run ./adapter ./store ./internal/encryption . --timeout=5m` + +The broader `go test ./adapter ./store ./internal/encryption . -count=1 +-timeout=300s` run timed out in the full adapter package. The targeted +encryption admin tests and the directly affected non-adapter packages +passed. diff --git a/main.go b/main.go index 5bc27a830..52be20f72 100644 --- a/main.go +++ b/main.go @@ -1468,6 +1468,7 @@ func buildShardGroups( _ = st.Close() return nil, nil, errors.Wrapf(err, "failed to start raft group %d", g.id) } + runtime.writerRegistry = reg runtimes = append(runtimes, runtime) // Stage 6E-2c: route every shard group's TransactionManager // through NewLeaderProxyForShardGroup so the proposer chain @@ -2683,6 +2684,7 @@ func startRaftServers( enableMutators, rt.engine, encryptionCapabilityFanout, + adapter.WithEncryptionAdminWriterRegistry(rt.writerRegistry), adapter.WithEncryptionAdminLatestAppliedIndex(appliedIndexForEngine(rt.engine)), adapter.WithEncryptionAdminPostCutoverProposer(proposerForGroup(rt, shardGroups)), adapter.WithEncryptionAdminCutoverBarrier(encWiring.raftEnvelope.barrier()), diff --git a/multiraft_runtime.go b/multiraft_runtime.go index c46daacfe..e5695ec3e 100644 --- a/multiraft_runtime.go +++ b/multiraft_runtime.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" @@ -27,8 +28,9 @@ type raftGroupRuntime struct { engineMu sync.RWMutex engine raftengine.Engine - store store.MVCCStore - stateMachine raftengine.StateMachine + store store.MVCCStore + writerRegistry encryption.WriterRegistryStore + stateMachine raftengine.StateMachine registerTransport func(grpc.ServiceRegistrar) closeFactory func() error // releases factory-created resources (transport, stores) From 9630091fa8891f4b5e93e13906d2003e26c86eef Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 11 Jul 2026 06:49:52 +0900 Subject: [PATCH 02/16] encryption: classify missing registry node id --- adapter/encryption_admin.go | 8 +++---- adapter/encryption_admin_test.go | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/adapter/encryption_admin.go b/adapter/encryption_admin.go index d16e7bfda..3927c63b8 100644 --- a/adapter/encryption_admin.go +++ b/adapter/encryption_admin.go @@ -400,7 +400,7 @@ func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) if err != nil { return nil, statusFromSidecarErr(err) } - writerRegistry, err := s.writerRegistryForCaller(sc, s.fullNodeID) + writerRegistry, err := s.writerRegistryForCaller(sc, s.fullNodeID, codes.Internal) if err != nil { return nil, err } @@ -442,7 +442,7 @@ func (s *EncryptionAdminServer) ResyncSidecar(ctx context.Context, req *pb.Resyn if req != nil { callerFullNodeID = req.GetCallerFullNodeId() } - writerRegistry, err := s.writerRegistryForCaller(sc, callerFullNodeID) + writerRegistry, err := s.writerRegistryForCaller(sc, callerFullNodeID, codes.InvalidArgument) if err != nil { return nil, err } @@ -455,13 +455,13 @@ func (s *EncryptionAdminServer) ResyncSidecar(ctx context.Context, req *pb.Resyn }, nil } -func (s *EncryptionAdminServer) writerRegistryForCaller(sc *encryption.Sidecar, fullNodeID uint64) (map[uint32]uint32, error) { +func (s *EncryptionAdminServer) writerRegistryForCaller(sc *encryption.Sidecar, fullNodeID uint64, missingIDCode codes.Code) (map[uint32]uint32, error) { out := map[uint32]uint32{} if s.writerRegistry == nil { return out, nil } if fullNodeID == 0 { - return nil, grpcStatusError(codes.InvalidArgument, + return nil, grpcStatusError(missingIDCode, "encryption: full_node_id is required to project writer_registry_for_caller") } nodeID16 := encryption.NodeID16(fullNodeID) diff --git a/adapter/encryption_admin_test.go b/adapter/encryption_admin_test.go index 6680c326b..e7e097f2a 100644 --- a/adapter/encryption_admin_test.go +++ b/adapter/encryption_admin_test.go @@ -192,6 +192,26 @@ func TestEncryptionAdmin_GetSidecarState_ProjectsWriterRegistryForLocalNode(t *t } } +func TestEncryptionAdmin_GetSidecarState_RejectsMissingLocalNodeIDAsInternal(t *testing.T) { + t.Parallel() + path := writeSidecarFixture(t, &encryption.Sidecar{ + Active: encryption.ActiveKeys{Storage: 1, Raft: 2}, + Keys: map[string]encryption.SidecarKey{ + "1": {Purpose: "storage", Wrapped: []byte("wrapped-1")}, + "2": {Purpose: "raft", Wrapped: []byte("wrapped-2")}, + }, + }) + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminWriterRegistry(newTestWriterRegistry()), + ) + + _, err := srv.GetSidecarState(context.Background(), &pb.Empty{}) + if status.Code(err) != codes.Internal { + t.Fatalf("GetSidecarState status=%v, want Internal for missing local full node id (err=%v)", status.Code(err), err) + } +} + func assertSidecarHeader(t *testing.T, got *pb.SidecarStateReport) { t.Helper() if got.ActiveStorageId != 1 || got.ActiveRaftId != 2 { @@ -317,6 +337,27 @@ func TestEncryptionAdmin_ResyncSidecar_ProjectsWriterRegistryForCaller(t *testin } } +func TestEncryptionAdmin_ResyncSidecar_RejectsMissingCallerNodeIDAsInvalidArgument(t *testing.T) { + t.Parallel() + path := writeSidecarFixture(t, &encryption.Sidecar{ + Active: encryption.ActiveKeys{Storage: 3, Raft: 4}, + Keys: map[string]encryption.SidecarKey{ + "3": {Purpose: "storage", Wrapped: []byte("ws")}, + "4": {Purpose: "raft", Wrapped: []byte("wr")}, + }, + }) + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminLeaderView(stubLeaderView{state: raftengine.StateLeader}), + WithEncryptionAdminWriterRegistry(newTestWriterRegistry()), + ) + + _, err := srv.ResyncSidecar(context.Background(), &pb.ResyncSidecarRequest{}) + if status.Code(err) != codes.InvalidArgument { + t.Fatalf("ResyncSidecar status=%v, want InvalidArgument for missing caller full node id (err=%v)", status.Code(err), err) + } +} + func TestEncryptionAdmin_ResyncSidecar_RejectsWriterRegistryNodeCollision(t *testing.T) { t.Parallel() const callerFullNodeID uint64 = 0x10001 From 4e1931bf35ef7a31d0eb079c0a14a36071ee27b6 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 11 Jul 2026 07:01:22 +0900 Subject: [PATCH 03/16] encryption: use default registry for sidecar projection --- main.go | 15 +++++++++++++- main_encryption_admin_test.go | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 52be20f72..9159a1ed1 100644 --- a/main.go +++ b/main.go @@ -1546,6 +1546,14 @@ func postCutoverProposerForRuntime(rt *raftGroupRuntime, shardGroups map[uint64] return proposerForGroup(rt, shardGroups) } +func writerRegistryForEncryptionAdmin(runtimes []*raftGroupRuntime, defaultGroup uint64) encryption.WriterRegistryStore { + defaultRuntime := findDefaultGroupRuntime(runtimes, defaultGroup) + if defaultRuntime == nil { + return nil + } + return defaultRuntime.writerRegistry +} + func appliedIndexForEngine(engine raftengine.Engine) func() uint64 { applied, ok := engine.(interface{ AppliedIndex() uint64 }) if !ok { @@ -1859,6 +1867,7 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, encWiring: in.encWiring, redisApplyObserver: in.redisApplyObserver, dynamoAddress: *dynamoAddr, + defaultGroup: in.cfg.defaultGroup, leaderDynamo: in.cfg.leaderDynamo, s3Address: *s3Addr, leaderS3: in.cfg.leaderS3, @@ -2612,6 +2621,7 @@ func startRaftServers( forwardDeps adminForwardServerDeps, confChangeInterceptor internalraftadmin.MembershipChangeInterceptor, encWiring encryptionWriteWiring, + defaultGroup uint64, s3BlobObserver adapter.S3BlobOffloadObserver, s3BlobPushBlocked func() bool, ) error { @@ -2622,6 +2632,7 @@ func startRaftServers( const extraOptsCap = 2 enableMutators := encryptionMutatorsEnabled() encryptionCapabilityFanout := buildEncryptionCapabilityFanout(ctx, eg, runtimes, enableMutators) + adminWriterRegistry := writerRegistryForEncryptionAdmin(runtimes, defaultGroup) for _, rt := range runtimes { baseOpts := internalutil.GRPCServerOptions() opts := make([]grpc.ServerOption, 0, len(baseOpts)+extraOptsCap) @@ -2684,7 +2695,7 @@ func startRaftServers( enableMutators, rt.engine, encryptionCapabilityFanout, - adapter.WithEncryptionAdminWriterRegistry(rt.writerRegistry), + adapter.WithEncryptionAdminWriterRegistry(adminWriterRegistry), adapter.WithEncryptionAdminLatestAppliedIndex(appliedIndexForEngine(rt.engine)), adapter.WithEncryptionAdminPostCutoverProposer(proposerForGroup(rt, shardGroups)), adapter.WithEncryptionAdminCutoverBarrier(encWiring.raftEnvelope.barrier()), @@ -3014,6 +3025,7 @@ type runtimeServerRunner struct { redisApplyObserver *adapter.RedisApplyObserver encWiring encryptionWriteWiring dynamoAddress string + defaultGroup uint64 leaderDynamo map[string]string s3Address string leaderS3 map[string]string @@ -3132,6 +3144,7 @@ func (r *runtimeServerRunner) startRaftTransport() error { forwardDeps, r.encryptionConfChangeInterceptor, r.encWiring, + r.defaultGroup, r.metricsRegistry.S3BlobOffloadObserver(), s3BlobPushBlocked, ); err != nil { diff --git a/main_encryption_admin_test.go b/main_encryption_admin_test.go index df98b0d1c..96a7743be 100644 --- a/main_encryption_admin_test.go +++ b/main_encryption_admin_test.go @@ -5,6 +5,7 @@ import ( "net" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/raftengine" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" pb "github.com/bootjp/elastickv/proto" @@ -41,6 +42,20 @@ func (stubEncryptionAdminEngine) LinearizableRead(context.Context) (uint64, erro return 0, nil } +type stubEncryptionAdminRegistry struct { + name string +} + +func (stubEncryptionAdminRegistry) GetRegistryRow([]byte) ([]byte, bool, error) { + return nil, false, nil +} + +func (stubEncryptionAdminRegistry) SetRegistryRow(_, _ []byte) error { + return nil +} + +var _ encryption.WriterRegistryStore = stubEncryptionAdminRegistry{} + // TestEncryptionAdminFullNodeID_DistinctPerRaftId pins the // PR760 r1 Codex P1 regression: full_node_id must be derived from // --raftId (per-node-stable) and NOT from the Raft group id @@ -65,6 +80,29 @@ func TestEncryptionAdminFullNodeID_DistinctPerRaftId(t *testing.T) { } } +func TestWriterRegistryForEncryptionAdminUsesDefaultGroup(t *testing.T) { + defaultRegistry := stubEncryptionAdminRegistry{name: "default"} + otherRegistry := stubEncryptionAdminRegistry{name: "other"} + runtimes := []*raftGroupRuntime{ + {spec: groupSpec{id: 2}, writerRegistry: otherRegistry}, + {spec: groupSpec{id: 1}, writerRegistry: defaultRegistry}, + } + + got := writerRegistryForEncryptionAdmin(runtimes, 1) + if got != defaultRegistry { + t.Fatalf("writerRegistryForEncryptionAdmin selected %#v, want default group registry %#v", got, defaultRegistry) + } +} + +func TestWriterRegistryForEncryptionAdminMissingDefaultGroup(t *testing.T) { + got := writerRegistryForEncryptionAdmin([]*raftGroupRuntime{ + {spec: groupSpec{id: 2}, writerRegistry: stubEncryptionAdminRegistry{name: "other"}}, + }, 1) + if got != nil { + t.Fatalf("writerRegistryForEncryptionAdmin selected %#v, want nil when default group runtime is missing", got) + } +} + // TestEncryptionAdmin_MutatingRPCRefusedWhenGateOff pins the // Stage 6B-2 double-gate. With either condition of the gate false // (enableMutators=false OR engine=nil), Proposer + LeaderView From 304df036cb27798071f5441da622e7604d1d1511 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 20:21:05 +0900 Subject: [PATCH 04/16] encryption: add compress-then-encrypt storage path --- ...6_04_29_partial_data_at_rest_encryption.md | 5 +- ...8_implemented_9a_encryption_compression.md | 39 +++ go.mod | 2 +- internal/encryption/envelope.go | 8 + internal/encryption/envelope_test.go | 27 +- internal/encryption/errors.go | 6 + internal/encryption/raft_envelope.go | 5 + internal/encryption/raft_envelope_test.go | 27 ++ store/encryption_compression_test.go | 240 ++++++++++++++++++ store/encryption_glue.go | 51 +++- store/lsm_store.go | 28 +- store/lsm_store_env_test.go | 8 +- store/lsm_store_sync_mode_benchmark_test.go | 70 +++++ store/lsm_store_test.go | 2 +- store/snapshot_pebble_sst.go | 4 +- 15 files changed, 490 insertions(+), 32 deletions(-) create mode 100644 docs/design/2026_07_18_implemented_9a_encryption_compression.md create mode 100644 store/encryption_compression_test.go diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 2c53e0626..25703d586 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -1,6 +1,6 @@ # Data-at-rest encryption for elastickv -Status: Partial — Stages 0–8 shipped (5E deferred); Stage 9 open +Status: Partial — Stages 0–8 and 9A shipped (5E deferred); remaining Stage 9 work open Author: bootjp Date: 2026-04-29 @@ -32,7 +32,8 @@ Date: 2026-04-29 | 6F | §6.5 `--encryption-rotate-on-startup` request flag + leader-elected rotation proposal on the default encryption Raft group. The leader rotates every active DEK purpose once before listeners open; followers keep an in-memory pending request and fire only if they acquire leadership in the same process uptime. | shipped | this PR | | 7 | Writer registry + deterministic nonce (§4.1). Covers process-start registration, storage-layer registration gate, runtime re-registration for cutover and rotation, conf-change-time pre-registration, and §5.5 registry projection in `GetSidecarState` / `ResyncSidecar` (`WriterRegistryForCaller`). | shipped | `2026_05_26_implemented_7a_process_start_registration.md` + `2026_05_26_implemented_7a2_storage_layer_registration_enforcement.md` + `2026_05_28_implemented_7b_runtime_reregistration.md` + `2026_05_28_implemented_7b_prime_runtime_reregistration_rotation.md` + `2026_05_29_implemented_7c_confchange_time_registration.md` + `2026_07_11_implemented_7d_sidecar_registry_projection.md` | | 8 | Snapshot header v2 (§4.4); WAL coverage closure (§4.3 / §4.6) | shipped | [`2026_05_29_implemented_8a_snapshot_header_v2.md`](2026_05_29_implemented_8a_snapshot_header_v2.md) + [`2026_06_01_implemented_8b_wal_coverage_closure.md`](2026_06_01_implemented_8b_wal_coverage_closure.md) | -| 9 | KMS-backed wrappers, compression, rotation/retire/rewrite, Jepsen (§5.2, §5.4, §6.4, §8) | open | — | +| 9A | Compress-then-encrypt, authenticated compression flag, encrypted-store Pebble compression policy, storage benchmark (§6.4, §8.3) | shipped | `2026_07_18_implemented_9a_encryption_compression.md` | +| 9B+ | KMS-backed wrappers, rotation/retire/rewrite, remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8) | open | — | Stages 0–4 ship the entire byte-tag pipeline (storage envelope, raft envelope, FSM dispatch, halt-on-error) but leave it **production diff --git a/docs/design/2026_07_18_implemented_9a_encryption_compression.md b/docs/design/2026_07_18_implemented_9a_encryption_compression.md new file mode 100644 index 000000000..3d97b31dd --- /dev/null +++ b/docs/design/2026_07_18_implemented_9a_encryption_compression.md @@ -0,0 +1,39 @@ +# Stage 9A data-at-rest compression + +Status: Implemented +Author: bootjp +Date: 2026-07-18 + +## Scope + +This milestone implements the compress-then-encrypt storage path from +`2026_04_29_partial_data_at_rest_encryption.md` §6.4. + +- Snappy runs on plaintext before AES-256-GCM. +- The compressed representation is selected only when it is strictly smaller. +- `FlagCompressed` is authenticated as part of the envelope AAD. +- Reads decompress only after GCM authentication succeeds. +- Unknown v1 flag bits fail closed. +- The cleartext-rebadge guard verifies both valid compression-flag variants. +- Pebble block compression is disabled for encryption-wired stores, including + database reopen and snapshot-restore temporary databases. + +Raft proposal envelopes remain uncompressed. Their apply path is latency +sensitive and the storage layer already performs the only value-compression +pass after proposal decryption. + +## Compatibility + +Existing v1 envelopes have `flag=0` and remain readable. New readers accept +both `flag=0` and `FlagCompressed`; legacy cleartext operation and stores with +no encryption wiring retain Pebble's default block-compression policy. + +## Verification + +- Storage round trips for compressed, uncompressed, empty, and already + compressed values. +- Authenticated compression-flag tamper rejection. +- Fail-closed handling for an authenticated malformed Snappy payload. +- Property testing over arbitrary byte slices for compression framing. +- Pebble compression-policy tests for encrypted and legacy stores. +- Paired cleartext/encrypted 1 KiB storage benchmark for `benchstat` review. diff --git a/go.mod b/go.mod index b9c9df389..02cfe9faf 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/getsentry/sentry-go v0.47.0 github.com/goccy/go-json v0.10.6 + github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e github.com/hanwen/go-fuse/v2 v2.10.1 github.com/klauspost/compress v1.19.0 github.com/pkg/errors v0.9.1 @@ -69,7 +70,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect diff --git a/internal/encryption/envelope.go b/internal/encryption/envelope.go index 0d221d3c3..52ff934fb 100644 --- a/internal/encryption/envelope.go +++ b/internal/encryption/envelope.go @@ -90,6 +90,10 @@ func (e *Envelope) Encode() ([]byte, error) { return nil, errors.Wrapf(ErrEnvelopeVersion, "encode: got 0x%02x, want 0x%02x", e.Version, EnvelopeVersionV1) } + if e.Flag&^FlagCompressed != 0 { + return nil, errors.Wrapf(ErrEnvelopeFlag, + "encode: got 0x%02x, allowed mask 0x%02x", e.Flag, FlagCompressed) + } if len(e.Body) < TagSize { return nil, errors.Wrapf(ErrEnvelopeShort, "encode: body %d bytes, want >= %d", len(e.Body), TagSize) @@ -116,6 +120,10 @@ func DecodeEnvelope(src []byte) (*Envelope, error) { return nil, errors.Wrapf(ErrEnvelopeVersion, "got 0x%02x, want 0x%02x", src[versionOffset], EnvelopeVersionV1) } + if src[flagOffset]&^FlagCompressed != 0 { + return nil, errors.Wrapf(ErrEnvelopeFlag, + "got 0x%02x, allowed mask 0x%02x", src[flagOffset], FlagCompressed) + } e := &Envelope{ Version: src[versionOffset], Flag: src[flagOffset], diff --git a/internal/encryption/envelope_test.go b/internal/encryption/envelope_test.go index 74b89552f..2d041a59b 100644 --- a/internal/encryption/envelope_test.go +++ b/internal/encryption/envelope_test.go @@ -2,6 +2,7 @@ package encryption_test import ( "bytes" + "fmt" "testing" "github.com/bootjp/elastickv/internal/encryption" @@ -35,7 +36,7 @@ func TestEnvelope_RoundTrip(t *testing.T) { name: "max key_id", env: encryption.Envelope{ Version: encryption.EnvelopeVersionV1, - Flag: 0xff, + Flag: encryption.FlagCompressed, KeyID: 0xFFFF_FFFF, Body: bytes.Repeat([]byte{0x77}, 1024+encryption.TagSize), }, @@ -129,6 +130,30 @@ func TestEnvelope_Decode_RejectsUnknownVersion(t *testing.T) { } } +func TestEnvelope_RejectsUnknownFlagBits(t *testing.T) { + t.Parallel() + for _, flag := range []byte{0x02, 0x80, 0xff} { + t.Run(fmt.Sprintf("flag_%02x", flag), func(t *testing.T) { + env := encryption.Envelope{ + Version: encryption.EnvelopeVersionV1, + Flag: flag, + KeyID: 1, + Body: bytes.Repeat([]byte{0x55}, encryption.TagSize), + } + if _, err := env.Encode(); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("Encode flag=%#x: expected ErrEnvelopeFlag, got %v", flag, err) + } + + buf := make([]byte, encryption.HeaderSize+encryption.TagSize) + buf[0] = encryption.EnvelopeVersionV1 + buf[1] = flag + if _, err := encryption.DecodeEnvelope(buf); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("DecodeEnvelope flag=%#x: expected ErrEnvelopeFlag, got %v", flag, err) + } + }) + } +} + func TestEnvelope_Decode_DoesNotAliasInput(t *testing.T) { src := make([]byte, encryption.HeaderSize+encryption.TagSize+8) src[0] = encryption.EnvelopeVersionV1 diff --git a/internal/encryption/errors.go b/internal/encryption/errors.go index 80bfeb8de..836247cdb 100644 --- a/internal/encryption/errors.go +++ b/internal/encryption/errors.go @@ -35,6 +35,12 @@ var ( // current build does not know how to parse. Reserved values per §11.3. ErrEnvelopeVersion = errors.New("encryption: unknown envelope version") + // ErrEnvelopeFlag indicates an envelope set a flag bit that the current + // format does not define. V1 only permits FlagCompressed; accepting other + // bits would make future format extensions ambiguous and could route + // authenticated plaintext through the wrong post-decrypt transform. + ErrEnvelopeFlag = errors.New("encryption: unknown envelope flag bits") + // ErrNilKeystore indicates NewCipher was called with a nil Keystore. // Surfaced at construction time so a wiring mistake is caught // before the first Encrypt/Decrypt would otherwise nil-deref panic. diff --git a/internal/encryption/raft_envelope.go b/internal/encryption/raft_envelope.go index bf317c744..16df2aa75 100644 --- a/internal/encryption/raft_envelope.go +++ b/internal/encryption/raft_envelope.go @@ -88,6 +88,7 @@ func WrapRaftPayload(c *Cipher, keyID uint32, nonce, payload []byte) ([]byte, er // // - ErrEnvelopeShort: encoded shorter than HeaderSize+TagSize // - ErrEnvelopeVersion: unknown version byte +// - ErrEnvelopeFlag: a storage-only or unknown flag bit is present // - ErrUnknownKeyID: DEK is not loaded (retired or sidecar missing) // - ErrIntegrity: GCM tag mismatch (tampered envelope, wrong DEK, // or layer confusion with a storage envelope) @@ -104,6 +105,10 @@ func UnwrapRaftPayload(c *Cipher, encoded []byte) ([]byte, error) { if err != nil { return nil, errors.Wrap(err, "encryption: raft envelope decode") } + if env.Flag != 0 { + return nil, errors.Wrapf(ErrEnvelopeFlag, + "encryption: raft envelope flag must be 0x00, got 0x%02x", env.Flag) + } aad := BuildRaftAAD(env.Version, env.KeyID) plain, err := c.Decrypt(env.Body, aad, env.KeyID, env.Nonce[:]) if err != nil { diff --git a/internal/encryption/raft_envelope_test.go b/internal/encryption/raft_envelope_test.go index 12915a321..23f837455 100644 --- a/internal/encryption/raft_envelope_test.go +++ b/internal/encryption/raft_envelope_test.go @@ -172,6 +172,33 @@ func TestRaftEnvelope_RejectsRetiredKey(t *testing.T) { } } +func TestRaftEnvelope_RejectsStorageCompressionFlag(t *testing.T) { + t.Parallel() + c, keyID := raftFixture(t) + nonce := newRandomNonce(t) + payload := []byte("raft payload") + aad := encryption.BuildRaftAAD(encryption.EnvelopeVersionV1, keyID) + body, err := c.Encrypt(payload, aad, keyID, nonce) + if err != nil { + t.Fatalf("Encrypt: %v", err) + } + var nonceArray [encryption.NonceSize]byte + copy(nonceArray[:], nonce) + encoded, err := (&encryption.Envelope{ + Version: encryption.EnvelopeVersionV1, + Flag: encryption.FlagCompressed, + KeyID: keyID, + Nonce: nonceArray, + Body: body, + }).Encode() + if err != nil { + t.Fatalf("Envelope.Encode: %v", err) + } + if _, err := encryption.UnwrapRaftPayload(c, encoded); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("expected ErrEnvelopeFlag, got %v", err) + } +} + // TestRaftEnvelope_ShortInputRejected covers DecodeEnvelope's // length precondition (HeaderSize + TagSize = 34 bytes minimum). func TestRaftEnvelope_ShortInputRejected(t *testing.T) { diff --git a/store/encryption_compression_test.go b/store/encryption_compression_test.go new file mode 100644 index 000000000..a83ec0d21 --- /dev/null +++ b/store/encryption_compression_test.go @@ -0,0 +1,240 @@ +package store + +import ( + "bytes" + "context" + "crypto/rand" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" + "github.com/golang/snappy" + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +func TestEncryption_CompressesOnlyWhenSmaller(t *testing.T) { + t.Parallel() + incompressible := make([]byte, 4096) + if _, err := rand.Read(incompressible); err != nil { + t.Fatalf("rand.Read incompressible value: %v", err) + } + cases := []struct { + name string + value []byte + compressed bool + }{ + {name: "compressible", value: bytes.Repeat([]byte("json-field:"), 512), compressed: true}, + {name: "empty", value: []byte{}, compressed: false}, + {name: "small", value: []byte("value"), compressed: false}, + {name: "high entropy", value: incompressible, compressed: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newEncryptedStoreFixture(t, 101) + key := []byte("compression/" + tc.name) + if err := f.mvcc.PutAt(context.Background(), key, tc.value, 100, 0); err != nil { + t.Fatalf("PutAt: %v", err) + } + + var flag byte + f.tamperPebbleValue(t, key, 100, func(raw []byte) []byte { + sv, err := decodeValue(raw) + if err != nil { + t.Fatalf("decodeValue: %v", err) + } + env, err := encryption.DecodeEnvelope(sv.Value) + if err != nil { + t.Fatalf("DecodeEnvelope: %v", err) + } + flag = env.Flag + return raw + }) + require.Equal(t, tc.compressed, flag&encryption.FlagCompressed != 0) + + got, err := f.mvcc.GetAt(context.Background(), key, 100) + require.NoError(t, err) + require.Equal(t, tc.value, got) + }) + } +} + +func TestEncryption_CompressionFlagTamperRejected(t *testing.T) { + t.Parallel() + f := newEncryptedStoreFixture(t, 102) + key := []byte("compressed-tamper") + value := bytes.Repeat([]byte("compress-me"), 512) + if err := f.mvcc.PutAt(context.Background(), key, value, 100, 0); err != nil { + t.Fatalf("PutAt: %v", err) + } + f.tamperPebbleValue(t, key, 100, func(raw []byte) []byte { + // value header (9) + envelope version (1) precede the flag. + raw[valueHeaderSize+1] &^= encryption.FlagCompressed + return raw + }) + if _, err := f.mvcc.GetAt(context.Background(), key, 100); !errors.Is(err, ErrEncryptedReadIntegrity) { + t.Fatalf("compression flag tamper: expected ErrEncryptedReadIntegrity, got %v", err) + } +} + +func TestEncryption_CompressedEnvelopeRebadgeRejected(t *testing.T) { + t.Parallel() + f := newEncryptedStoreFixture(t, 104) + key := []byte("compressed-rebadge") + value := bytes.Repeat([]byte("sensitive-json-field"), 512) + if err := f.mvcc.PutAt(context.Background(), key, value, 100, 0); err != nil { + t.Fatalf("PutAt: %v", err) + } + f.tamperPebbleValue(t, key, 100, func(raw []byte) []byte { + raw[0] &^= encStateMask + return raw + }) + if _, err := f.mvcc.GetAt(context.Background(), key, 100); !errors.Is(err, ErrEncryptedReadIntegrity) { + t.Fatalf("compressed rebadge: expected ErrEncryptedReadIntegrity, got %v", err) + } +} + +func TestEncryption_RejectsAuthenticatedMalformedSnappy(t *testing.T) { + t.Parallel() + f := newEncryptedStoreFixture(t, 103) + key := []byte("malformed-snappy") + const commitTS uint64 = 100 + pebbleKey := encodeKey(key, commitTS) + var header [valueHeaderSize]byte + writeValueHeaderBytes(header[:], false, 0, encStateEncrypted) + aad := buildStorageAAD(encryption.EnvelopeVersionV1, encryption.FlagCompressed, f.keyID, header[:], pebbleKey) + var nonce [encryption.NonceSize]byte + if _, err := rand.Read(nonce[:]); err != nil { + t.Fatalf("rand.Read nonce: %v", err) + } + body, err := f.cipher.Encrypt([]byte{0xff}, aad, f.keyID, nonce[:]) + if err != nil { + t.Fatalf("Encrypt malformed payload: %v", err) + } + envelopeBytes, err := (&encryption.Envelope{ + Version: encryption.EnvelopeVersionV1, + Flag: encryption.FlagCompressed, + KeyID: f.keyID, + Nonce: nonce, + Body: body, + }).Encode() + if err != nil { + t.Fatalf("Envelope.Encode: %v", err) + } + + f.closeIfOpen(t) + pdb, err := pebble.Open(f.dir, &pebble.Options{}) + if err != nil { + t.Fatalf("pebble.Open: %v", err) + } + if err := pdb.Set(pebbleKey, encodeValue(envelopeBytes, false, 0, encStateEncrypted), pebble.Sync); err != nil { + _ = pdb.Close() + t.Fatalf("pdb.Set: %v", err) + } + if err := pdb.Close(); err != nil { + t.Fatalf("pdb.Close: %v", err) + } + f.reopen(t) + + if _, err := f.mvcc.GetAt(context.Background(), key, commitTS); !errors.Is(err, ErrEncryptedReadCompression) { + t.Fatalf("malformed authenticated Snappy: expected ErrEncryptedReadCompression, got %v", err) + } +} + +func TestCompressionRoundTripProperty(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + plaintext := rapid.SliceOfN(rapid.Byte(), 0, 16<<10).Draw(t, "plaintext") + payload, flag := compressForEncryption(plaintext) + if flag == 0 { + if !bytes.Equal(payload, plaintext) { + t.Fatalf("uncompressed payload mismatch") + } + return + } + if flag != encryption.FlagCompressed { + t.Fatalf("unexpected compression flag %#x", flag) + } + if len(payload) >= len(plaintext) { + t.Fatalf("compressed payload grew: got=%d original=%d", len(payload), len(plaintext)) + } + got, err := snappy.Decode(nil, payload) + if err != nil { + t.Fatalf("snappy.Decode: %v", err) + } + if !bytes.Equal(got, plaintext) { + t.Fatalf("round-trip mismatch") + } + }) +} + +func TestEncryptedStoreCompressionRoundTripProperty(t *testing.T) { + t.Parallel() + ks := encryption.NewKeystore() + dek := make([]byte, encryption.KeySize) + if _, err := rand.Read(dek); err != nil { + t.Fatalf("rand.Read DEK: %v", err) + } + const keyID uint32 = 105 + if err := ks.Set(keyID, dek); err != nil { + t.Fatalf("Keystore.Set: %v", err) + } + cipher, err := encryption.NewCipher(ks) + if err != nil { + t.Fatalf("NewCipher: %v", err) + } + s := &pebbleStore{ + cipher: cipher, + nonceFactory: NewCounterNonceFactory(0x0102, 0x0304), + activeStorageKeyID: func() (uint32, bool) { return keyID, true }, + } + rapid.Check(t, func(t *rapid.T) { + key := rapid.SliceOfN(rapid.Byte(), 0, 256).Draw(t, "key") + plaintext := rapid.SliceOfN(rapid.Byte(), 0, 16<<10).Draw(t, "plaintext") + expireAt := rapid.Uint64().Draw(t, "expireAt") + pebbleKey := encodeKey(key, 100) + body, encState, err := s.encryptForKey(pebbleKey, plaintext, expireAt, false) + if err != nil { + t.Fatalf("encryptForKey: %v", err) + } + if encState != encStateEncrypted { + t.Fatalf("encryption state=%#x, want encrypted", encState) + } + got, err := s.decryptForKey(pebbleKey, storedValue{ + Value: body, + EncState: encState, + ExpireAt: expireAt, + }, body) + if err != nil { + t.Fatalf("decryptForKey: %v", err) + } + if !bytes.Equal(got, plaintext) { + t.Fatalf("round-trip mismatch") + } + }) +} + +func TestPebbleCompressionPolicy(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + disable bool + wantName string + }{ + {name: "legacy defaults", disable: false, wantName: "Snappy"}, + {name: "encrypted store", disable: true, wantName: "NoCompression"}, + } { + t.Run(tc.name, func(t *testing.T) { + opts, cache := defaultPebbleOptionsWithCache(tc.disable) + t.Cleanup(cache.Unref) + for level, levelOpts := range opts.Levels { + profile := levelOpts.Compression() + if tc.disable { + require.Equalf(t, tc.wantName, profile.Name, "level %d", level) + continue + } + require.NotEqualf(t, "NoCompression", profile.Name, "level %d", level) + } + }) + } +} diff --git a/store/encryption_glue.go b/store/encryption_glue.go index a51740c96..1590b93cc 100644 --- a/store/encryption_glue.go +++ b/store/encryption_glue.go @@ -4,6 +4,7 @@ import ( "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/v2" + "github.com/golang/snappy" ) // ErrUnsupportedStoreForWriterRegistry is returned by @@ -103,6 +104,13 @@ func WriterRegistryFor(s MVCCStore) (encryption.WriterRegistryStore, error) { // Callers can disambiguate it from any other read error with errors.Is. var ErrEncryptedReadIntegrity = errors.New("store: encrypted value failed integrity check (GCM tag mismatch); refusing to surface plaintext") +// ErrEncryptedReadCompression is returned when an authenticated envelope is +// marked as Snappy-compressed but its decrypted body is not a valid Snappy +// stream. Disk corruption normally fails GCM first; reaching this sentinel +// means a writer produced a malformed authenticated payload, so reads fail +// closed instead of surfacing the compressed bytes as user data. +var ErrEncryptedReadCompression = errors.New("store: authenticated encrypted value contains an invalid Snappy payload") + // NonceFactory produces unique 12-byte AES-GCM nonces for the storage // envelope (§4.1). The factory is responsible for the cluster-wide // uniqueness invariant across `(node_id, local_epoch, write_count)` — @@ -345,12 +353,11 @@ func (s *pebbleStore) encryptForKey(pebbleKey, plaintext []byte, expireAt uint64 return nil, 0, errors.Wrap(err, "store: nonce factory") } nonce := nonceArr[:] - // flag = 0: Snappy compression deferred to Stage 9 per design §4.1. - const envelopeFlag byte = 0 + payload, envelopeFlag := compressForEncryption(plaintext) var hdr [valueHeaderSize]byte writeValueHeaderBytes(hdr[:], false /*tombstone*/, expireAt, encStateEncrypted) aad := buildStorageAAD(encryption.EnvelopeVersionV1, envelopeFlag, keyID, hdr[:], pebbleKey) - ciphertextAndTag, err := s.cipher.Encrypt(plaintext, aad, keyID, nonce) + ciphertextAndTag, err := s.cipher.Encrypt(payload, aad, keyID, nonce) if err != nil { return nil, 0, errors.Wrap(err, "store: encrypt value") } @@ -413,6 +420,14 @@ func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byt } return nil, errors.Wrap(err, "store: decrypt value") } + if env.Flag&encryption.FlagCompressed != 0 { + plain, err = snappy.Decode(nil, plain) + if err != nil { + return nil, errors.Wrap( + errors.WithSecondaryError(ErrEncryptedReadCompression, err), + "store: decompress encrypted value") + } + } // AES-GCM Open returns a nil dst slice for an empty plaintext; // upstream callers (notably ExistsAt) distinguish "key absent" // from "key present with empty value" via val != nil. Normalize @@ -424,6 +439,18 @@ func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byt return plain, nil } +// compressForEncryption applies §6.4's compress-then-encrypt policy. Snappy +// output is used only when it is strictly smaller than the original bytes; +// already-compressed and small payloads stay uncompressed so encryption never +// increases CPU and storage cost for a larger intermediate representation. +func compressForEncryption(plaintext []byte) ([]byte, byte) { + compressed := snappy.Encode(nil, plaintext) + if len(compressed) >= len(plaintext) { + return plaintext, 0 + } + return compressed, encryption.FlagCompressed +} + // rejectRebadgedEnvelope is the cleartext-branch guard for the §4.1 // encryption-state rebadge attack. The on-disk encryption_state bit // is not itself authenticated, so a disk attacker who flips it from @@ -447,8 +474,8 @@ func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byt // - envelope_version = EnvelopeVersionV1 (the encrypt path's // fixed value; trusting on-disk would let a corrupted version // byte force the body through DecodeEnvelope's error path) -// - flag = 0 (Snappy compression is deferred; the -// encrypt path's fixed value) +// - flag: both supported values (uncompressed and Snappy-compressed), +// because the attacker can rewrite the on-disk flag byte // - tombstone = false (the encrypt path never wraps // tombstones, so any on-disk tombstone bit on an encrypted // entry is necessarily attacker-supplied) @@ -496,12 +523,14 @@ func (s *pebbleStore) rejectRebadgedEnvelope(pebbleKey []byte, sv storedValue, b } for _, kid := range s.cipher.LoadedKeyIDs() { for _, candidateExpire := range candidateExpireAts { - var hdr [valueHeaderSize]byte - writeValueHeaderBytes(hdr[:], false /*canonical*/, candidateExpire, encStateEncrypted) - aad := buildStorageAAD(encryption.EnvelopeVersionV1, 0 /*flag canonical*/, kid, hdr[:], pebbleKey) - if _, err := s.cipher.Decrypt(ct, aad, kid, nonce); err == nil { - return errors.Wrap(ErrEncryptedReadIntegrity, - "store: cleartext-labelled value verifies as a relabeled envelope under a loaded DEK") + for _, candidateFlag := range []byte{0, encryption.FlagCompressed} { + var hdr [valueHeaderSize]byte + writeValueHeaderBytes(hdr[:], false /*canonical*/, candidateExpire, encStateEncrypted) + aad := buildStorageAAD(encryption.EnvelopeVersionV1, candidateFlag, kid, hdr[:], pebbleKey) + if _, err := s.cipher.Decrypt(ct, aad, kid, nonce); err == nil { + return errors.Wrap(ErrEncryptedReadIntegrity, + "store: cleartext-labelled value verifies as a relabeled envelope under a loaded DEK") + } } } } diff --git a/store/lsm_store.go b/store/lsm_store.go index f2e04e6c0..6eaba5aa0 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -338,7 +338,7 @@ func WithSSTIngestSnapshots(enabled bool) PebbleStoreOption { // store/open reference and returns it to the caller. The caller MUST Unref the // returned cache after the DB is closed (or after pebble.Open fails), matching // the lifetime rules used by NewPebbleStore, Restore, and temp restore DBs. -func defaultPebbleOptionsWithCache() (*pebble.Options, *pebble.Cache) { +func defaultPebbleOptionsWithCache(disableCompression bool) (*pebble.Options, *pebble.Cache) { cache := processPebbleCacheRef() opts := &pebble.Options{ FS: vfs.Default, @@ -348,6 +348,14 @@ func defaultPebbleOptionsWithCache() (*pebble.Options, *pebble.Cache) { // Enable automatic compactions and apply all other Pebble defaults. // EnsureDefaults leaves Cache alone because we already set it. opts.EnsureDefaults() + if disableCompression { + // Storage-envelope payloads are compressed before encryption. The + // resulting ciphertext is high entropy, so Pebble block compression + // only burns CPU and cannot recover additional space. + opts.ApplyCompressionSettings(func() pebble.DBCompressionSettings { + return pebble.DBCompressionNone + }) + } return opts, cache } @@ -388,7 +396,7 @@ func NewPebbleStore(dir string, opts ...PebbleStoreOption) (MVCCStore, error) { return nil, err } - pebbleOpts, cache := defaultPebbleOptionsWithCache() + pebbleOpts, cache := defaultPebbleOptionsWithCache(s.cipher != nil) db, err := pebble.Open(dir, pebbleOpts) if err != nil { cache.Unref() @@ -3111,7 +3119,7 @@ func (s *pebbleStore) reopenFreshDB() error { if err := os.MkdirAll(s.dir, dirPerms); err != nil { return errors.WithStack(err) } - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(s.cipher != nil) db, err := pebble.Open(s.dir, opts) if err != nil { cache.Unref() @@ -3182,7 +3190,7 @@ func (s *pebbleStore) restorePebbleNativeAtomic(r io.Reader) error { return err } - if err := writeNativeSnapshotToTempDir(r, tmpDir, ts); err != nil { + if err := writeNativeSnapshotToTempDir(r, tmpDir, ts, s.cipher != nil); err != nil { _ = os.RemoveAll(tmpDir) return err } @@ -3224,8 +3232,8 @@ func makeSiblingTempDir(dir, tag string) (string, error) { // writeNativeSnapshotToTempDir writes batch-loop entries from r into a fresh // Pebble database at tmpDir, persists ts as the lastCommitTS meta-key, then // closes the database. -func writeNativeSnapshotToTempDir(r io.Reader, tmpDir string, ts uint64) error { - opts, cache := defaultPebbleOptionsWithCache() +func writeNativeSnapshotToTempDir(r io.Reader, tmpDir string, ts uint64, disableCompression bool) error { + opts, cache := defaultPebbleOptionsWithCache(disableCompression) tmpDB, err := pebble.Open(tmpDir, opts) if err != nil { cache.Unref() @@ -3271,12 +3279,12 @@ func readStreamingMVCCRestoreHeader(r io.Reader) (io.Reader, hash.Hash32, uint32 return body, hash, expectedChecksum, lastCommitTS, minRetainedTS, nil } -func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash32, expectedChecksum uint32, lastCommitTS uint64, minRetainedTS uint64) (string, error) { +func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash32, expectedChecksum uint32, lastCommitTS uint64, minRetainedTS uint64, disableCompression bool) (string, error) { tmpDir := filepath.Clean(dir) + ".restore-tmp" if err := os.RemoveAll(tmpDir); err != nil { return "", errors.WithStack(err) } - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(disableCompression) tmpDB, err := pebble.Open(tmpDir, opts) if err != nil { cache.Unref() @@ -3316,7 +3324,7 @@ func (s *pebbleStore) restoreFromStreamingMVCC(r io.Reader) error { return err } - tmpDir, err := writeStreamingMVCCRestoreTempDB(s.dir, body, hash, expectedChecksum, lastCommitTS, minRetainedTS) + tmpDir, err := writeStreamingMVCCRestoreTempDB(s.dir, body, hash, expectedChecksum, lastCommitTS, minRetainedTS, s.cipher != nil) if err != nil { return err } @@ -3400,7 +3408,7 @@ func (s *pebbleStore) swapInTempDBWithMetadata(tmpDir string, expected *pebbleSn } func (s *pebbleStore) reopenStoreDB(dir string) error { - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(s.cipher != nil) db, err := pebble.Open(dir, opts) if err != nil { cache.Unref() diff --git a/store/lsm_store_env_test.go b/store/lsm_store_env_test.go index 0fa262f36..132b98b29 100644 --- a/store/lsm_store_env_test.go +++ b/store/lsm_store_env_test.go @@ -144,7 +144,7 @@ func TestSetSmallPebbleCacheForTestRestores(t *testing.T) { // and that Unref is safe for each borrowed store/open reference. func TestDefaultPebbleOptionsCarriesCache(t *testing.T) { setSmallPebbleCacheForTest(t) - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(false) require.NotNil(t, cache) require.Same(t, cache, opts.Cache) require.Equal(t, int64(16)<<20, cache.MaxSize()) @@ -153,9 +153,9 @@ func TestDefaultPebbleOptionsCarriesCache(t *testing.T) { func TestDefaultPebbleOptionsSharesProcessCache(t *testing.T) { setSmallPebbleCacheForTest(t) - opts1, cache1 := defaultPebbleOptionsWithCache() + opts1, cache1 := defaultPebbleOptionsWithCache(false) defer cache1.Unref() - opts2, cache2 := defaultPebbleOptionsWithCache() + opts2, cache2 := defaultPebbleOptionsWithCache(false) defer cache2.Unref() require.Same(t, cache1, cache2) @@ -173,7 +173,7 @@ func TestDefaultPebbleOptionsSharesProcessCacheConcurrently(t *testing.T) { for range borrowers { go func() { defer wg.Done() - _, cache := defaultPebbleOptionsWithCache() + _, cache := defaultPebbleOptionsWithCache(false) caches <- cache }() } diff --git a/store/lsm_store_sync_mode_benchmark_test.go b/store/lsm_store_sync_mode_benchmark_test.go index bb1f02662..a06a5ff58 100644 --- a/store/lsm_store_sync_mode_benchmark_test.go +++ b/store/lsm_store_sync_mode_benchmark_test.go @@ -2,9 +2,11 @@ package store import ( "context" + "crypto/rand" "fmt" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/pebble/v2" ) @@ -74,3 +76,71 @@ func BenchmarkApplyMutationsRaft_SyncMode(b *testing.B) { }) } } + +// BenchmarkApplyMutationsRaft_Encryption compares the Stage 9 +// compress-then-encrypt path against the legacy cleartext path with 1 KiB +// values. NoSync isolates encryption/compression CPU from fsync latency; the +// merge gate is evaluated with benchstat on the paired results. +func BenchmarkApplyMutationsRaft_Encryption(b *testing.B) { + value := make([]byte, 1024) + if _, err := rand.Read(value); err != nil { + b.Fatalf("rand.Read value: %v", err) + } + for _, tc := range []struct { + name string + encrypted bool + }{ + {name: "cleartext", encrypted: false}, + {name: "encrypted", encrypted: true}, + } { + b.Run(tc.name, func(b *testing.B) { + s, err := NewPebbleStore(b.TempDir(), benchmarkEncryptionOptions(b, tc.encrypted)...) + if err != nil { + b.Fatalf("NewPebbleStore: %v", err) + } + defer s.Close() + ps, ok := s.(*pebbleStore) + if !ok { + b.Fatalf("NewPebbleStore returned non-*pebbleStore type: %T", s) + } + ps.fsmApplyWriteOpts = pebble.NoSync + ps.fsmApplySyncModeLabel = fsmSyncModeNoSync + b.SetBytes(int64(len(value))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + key := []byte(fmt.Sprintf("enc-bench-%010d", i)) + startTS := uint64(i) * 2 //nolint:gosec // benchmark loop index is non-negative. + muts := []*KVPairMutation{{Op: OpTypePut, Key: key, Value: value}} + if err := s.ApplyMutationsRaft(context.Background(), muts, nil, startTS, startTS+1); err != nil { + b.Fatalf("ApplyMutationsRaft: %v", err) + } + } + }) + } +} + +func benchmarkEncryptionOptions(b *testing.B, enabled bool) []PebbleStoreOption { + b.Helper() + if !enabled { + return nil + } + ks := encryption.NewKeystore() + dek := make([]byte, encryption.KeySize) + if _, err := rand.Read(dek); err != nil { + b.Fatalf("rand.Read DEK: %v", err) + } + const keyID uint32 = 1 + if err := ks.Set(keyID, dek); err != nil { + b.Fatalf("Keystore.Set: %v", err) + } + cipher, err := encryption.NewCipher(ks) + if err != nil { + b.Fatalf("NewCipher: %v", err) + } + return []PebbleStoreOption{WithEncryption( + cipher, + NewCounterNonceFactory(1, 1), + func() (uint32, bool) { return keyID, true }, + )} +} diff --git a/store/lsm_store_test.go b/store/lsm_store_test.go index ecea4526a..83a1d6f7c 100644 --- a/store/lsm_store_test.go +++ b/store/lsm_store_test.go @@ -817,7 +817,7 @@ func TestSnapshotBatchShouldFlushOnByteLimit(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(false) db, err := pebble.Open(dir, opts) require.NoError(t, err) defer func() { diff --git a/store/snapshot_pebble_sst.go b/store/snapshot_pebble_sst.go index c23c972af..1c495a38d 100644 --- a/store/snapshot_pebble_sst.go +++ b/store/snapshot_pebble_sst.go @@ -327,7 +327,7 @@ func (s *pebbleSSTIngestSnapshot) Close() error { } func buildPebbleSSTIngestBundle(checkpointDir string, metadata pebbleSnapshotMetadata, targetFileBytes uint64) (*pebbleSSTIngestBundle, error) { - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(false) opts.ReadOnly = true db, err := pebble.Open(checkpointDir, opts) if err != nil { @@ -761,7 +761,7 @@ func metadataFromSSTIngestManifest(manifest pebbleSSTIngestManifest) pebbleSnaps } func ingestSSTSnapshotIntoTempDB(tmpDir string, paths []string, metadata pebbleSnapshotMetadata) error { - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(false) db, err := pebble.Open(tmpDir, opts) if err != nil { cache.Unref() From 2daeafa3985272ac99b8b9d2519449c28e86b9c1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 20:37:14 +0900 Subject: [PATCH 05/16] encryption: bound compressed read allocation --- store/encryption_compression_test.go | 53 ++++++++++++++++++++++++++++ store/encryption_glue.go | 44 +++++++++++++++++++---- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/store/encryption_compression_test.go b/store/encryption_compression_test.go index a83ec0d21..3509bd788 100644 --- a/store/encryption_compression_test.go +++ b/store/encryption_compression_test.go @@ -26,6 +26,8 @@ func TestEncryption_CompressesOnlyWhenSmaller(t *testing.T) { compressed bool }{ {name: "compressible", value: bytes.Repeat([]byte("json-field:"), 512), compressed: true}, + {name: "below threshold", value: bytes.Repeat([]byte("a"), minEncryptionCompressionSize-1), compressed: false}, + {name: "at threshold", value: bytes.Repeat([]byte("a"), minEncryptionCompressionSize), compressed: true}, {name: "empty", value: []byte{}, compressed: false}, {name: "small", value: []byte("value"), compressed: false}, {name: "high entropy", value: incompressible, compressed: false}, @@ -142,6 +144,57 @@ func TestEncryption_RejectsAuthenticatedMalformedSnappy(t *testing.T) { } } +func TestEncryption_RejectsAuthenticatedOversizedSnappyBeforeDecode(t *testing.T) { + previousMax := maxSnapshotValueSize + maxSnapshotValueSize = 64 + t.Cleanup(func() { maxSnapshotValueSize = previousMax }) + + f := newEncryptedStoreFixture(t, 106) + key := []byte("oversized-snappy") + const commitTS uint64 = 100 + pebbleKey := encodeKey(key, commitTS) + var header [valueHeaderSize]byte + writeValueHeaderBytes(header[:], false, 0, encStateEncrypted) + aad := buildStorageAAD(encryption.EnvelopeVersionV1, encryption.FlagCompressed, f.keyID, header[:], pebbleKey) + compressed := snappy.Encode(nil, bytes.Repeat([]byte("a"), maxSnapshotValueSize+1)) + var nonce [encryption.NonceSize]byte + if _, err := rand.Read(nonce[:]); err != nil { + t.Fatalf("rand.Read nonce: %v", err) + } + body, err := f.cipher.Encrypt(compressed, aad, f.keyID, nonce[:]) + if err != nil { + t.Fatalf("Encrypt oversized payload: %v", err) + } + envelopeBytes, err := (&encryption.Envelope{ + Version: encryption.EnvelopeVersionV1, + Flag: encryption.FlagCompressed, + KeyID: f.keyID, + Nonce: nonce, + Body: body, + }).Encode() + if err != nil { + t.Fatalf("Envelope.Encode: %v", err) + } + + f.closeIfOpen(t) + pdb, err := pebble.Open(f.dir, &pebble.Options{}) + if err != nil { + t.Fatalf("pebble.Open: %v", err) + } + if err := pdb.Set(pebbleKey, encodeValue(envelopeBytes, false, 0, encStateEncrypted), pebble.Sync); err != nil { + _ = pdb.Close() + t.Fatalf("pdb.Set: %v", err) + } + if err := pdb.Close(); err != nil { + t.Fatalf("pdb.Close: %v", err) + } + f.reopen(t) + + if _, err := f.mvcc.GetAt(context.Background(), key, commitTS); !errors.Is(err, ErrEncryptedReadCompression) { + t.Fatalf("oversized authenticated Snappy: expected ErrEncryptedReadCompression, got %v", err) + } +} + func TestCompressionRoundTripProperty(t *testing.T) { rapid.Check(t, func(t *rapid.T) { plaintext := rapid.SliceOfN(rapid.Byte(), 0, 16<<10).Draw(t, "plaintext") diff --git a/store/encryption_glue.go b/store/encryption_glue.go index 1590b93cc..43492566b 100644 --- a/store/encryption_glue.go +++ b/store/encryption_glue.go @@ -111,6 +111,8 @@ var ErrEncryptedReadIntegrity = errors.New("store: encrypted value failed integr // closed instead of surfacing the compressed bytes as user data. var ErrEncryptedReadCompression = errors.New("store: authenticated encrypted value contains an invalid Snappy payload") +const minEncryptionCompressionSize = 32 + // NonceFactory produces unique 12-byte AES-GCM nonces for the storage // envelope (§4.1). The factory is responsible for the cluster-wide // uniqueness invariant across `(node_id, local_epoch, write_count)` — @@ -420,13 +422,9 @@ func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byt } return nil, errors.Wrap(err, "store: decrypt value") } - if env.Flag&encryption.FlagCompressed != 0 { - plain, err = snappy.Decode(nil, plain) - if err != nil { - return nil, errors.Wrap( - errors.WithSecondaryError(ErrEncryptedReadCompression, err), - "store: decompress encrypted value") - } + plain, err = decompressAuthenticatedValue(plain, env.Flag) + if err != nil { + return nil, err } // AES-GCM Open returns a nil dst slice for an empty plaintext; // upstream callers (notably ExistsAt) distinguish "key absent" @@ -439,11 +437,43 @@ func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byt return plain, nil } +// decompressAuthenticatedValue inspects the Snappy length prefix before any +// output allocation. Callers must authenticate the envelope before reaching +// this helper; the compressed bytes are otherwise attacker-controlled. +func decompressAuthenticatedValue(plain []byte, flag byte) ([]byte, error) { + if flag&encryption.FlagCompressed == 0 { + return plain, nil + } + decodedLen, err := snappy.DecodedLen(plain) + if err != nil { + return nil, errors.Wrap( + errors.WithSecondaryError(ErrEncryptedReadCompression, err), + "store: inspect compressed encrypted value") + } + if decodedLen > maxSnapshotValueSize { + return nil, errors.Wrapf( + ErrEncryptedReadCompression, + "store: compressed encrypted value expands to %d bytes, maximum is %d", + decodedLen, + maxSnapshotValueSize) + } + decoded, err := snappy.Decode(nil, plain) + if err != nil { + return nil, errors.Wrap( + errors.WithSecondaryError(ErrEncryptedReadCompression, err), + "store: decompress encrypted value") + } + return decoded, nil +} + // compressForEncryption applies §6.4's compress-then-encrypt policy. Snappy // output is used only when it is strictly smaller than the original bytes; // already-compressed and small payloads stay uncompressed so encryption never // increases CPU and storage cost for a larger intermediate representation. func compressForEncryption(plaintext []byte) ([]byte, byte) { + if len(plaintext) < minEncryptionCompressionSize { + return plaintext, 0 + } compressed := snappy.Encode(nil, plaintext) if len(compressed) >= len(plaintext) { return plaintext, 0 From a6bfa76f47cd747a05d0b2298a8458872ecca84e Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 20:42:09 +0900 Subject: [PATCH 06/16] store: remove benchmark lint suppression --- store/lsm_store_sync_mode_benchmark_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/store/lsm_store_sync_mode_benchmark_test.go b/store/lsm_store_sync_mode_benchmark_test.go index a06a5ff58..bab9be116 100644 --- a/store/lsm_store_sync_mode_benchmark_test.go +++ b/store/lsm_store_sync_mode_benchmark_test.go @@ -110,7 +110,10 @@ func BenchmarkApplyMutationsRaft_Encryption(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { key := []byte(fmt.Sprintf("enc-bench-%010d", i)) - startTS := uint64(i) * 2 //nolint:gosec // benchmark loop index is non-negative. + if i < 0 { + b.Fatalf("unexpected negative iteration counter: %d", i) + } + startTS := uint64(i) * 2 muts := []*KVPairMutation{{Op: OpTypePut, Key: key, Value: value}} if err := s.ApplyMutationsRaft(context.Background(), muts, nil, startTS, startTS+1); err != nil { b.Fatalf("ApplyMutationsRaft: %v", err) From 8bacf37e2200a8bfbdff24b009b6e5a3abf75caa Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 20:57:35 +0900 Subject: [PATCH 07/16] encryption: harden recovery and compression compatibility --- adapter/encryption_admin.go | 37 +++++++++++-- adapter/encryption_admin_test.go | 18 +++++++ ...6_04_29_partial_data_at_rest_encryption.md | 25 ++++++--- ...8_implemented_9a_encryption_compression.md | 14 +++-- internal/encryption/envelope.go | 53 +++++++++++-------- internal/encryption/envelope_test.go | 35 +++++++++--- internal/encryption/errors.go | 6 +-- internal/encryption/raft_envelope.go | 4 ++ internal/encryption/raft_envelope_test.go | 4 +- main.go | 19 +++++-- main_encryption_admin_test.go | 38 +++++++++++++ store/encryption_compression_test.go | 19 +++++-- store/encryption_glue.go | 46 +++++++++++----- store/lsm_store.go | 6 +-- 14 files changed, 249 insertions(+), 75 deletions(-) diff --git a/adapter/encryption_admin.go b/adapter/encryption_admin.go index 3927c63b8..bfbe5f99d 100644 --- a/adapter/encryption_admin.go +++ b/adapter/encryption_admin.go @@ -46,6 +46,10 @@ type EncryptionAdminServer struct { // the raft envelope. postCutoverProposer raftengine.Proposer leaderView raftengine.LeaderView + // recoveryLeaderView is the authority for sidecar/registry recovery. + // In multi-group deployments it points at the default group even when + // this server is registered on another group's listener. + recoveryLeaderView raftengine.LeaderView // capabilityFanout, when wired, runs the §4 Voters ∪ Learners // fan-out before the §7.1 Phase 1 cutover entry is proposed. // A nil value short-circuits EnableStorageEnvelope with @@ -272,6 +276,17 @@ func WithEncryptionAdminLeaderView(v raftengine.LeaderView) EncryptionAdminServe } } +// WithEncryptionAdminRecoveryLeaderView registers the default-group +// leadership oracle used by ResyncSidecar. A nil value preserves the +// single-group fallback to leaderView. +func WithEncryptionAdminRecoveryLeaderView(v raftengine.LeaderView) EncryptionAdminServerOption { + return func(s *EncryptionAdminServer) { + if v != nil { + s.recoveryLeaderView = v + } + } +} + // WithEncryptionAdminCutoverBarrier wires the §7.1 quiescence // barrier controller used by EnableRaftEnvelope. A nil argument is // a no-op (the server stays in the cutover-disabled posture); @@ -428,7 +443,7 @@ func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) // state to a recovering follower and silently overwrite recent // rotations. func (s *EncryptionAdminServer) ResyncSidecar(ctx context.Context, req *pb.ResyncSidecarRequest) (*pb.ResyncSidecarResponse, error) { - if err := s.requireLeader(ctx); err != nil { + if err := s.requireRecoveryLeader(ctx); err != nil { return nil, err } if s.sidecarPath == "" { @@ -1912,11 +1927,23 @@ func proposeErrorToStatus(err error, opcode byte) error { // a follower's recovery flow could pull an outdated DEK // set from a stranded leader and miss recent rotations. func (s *EncryptionAdminServer) requireLeader(ctx context.Context) error { - if s.leaderView == nil { + return requireEncryptionLeader(ctx, s.leaderView) +} + +func (s *EncryptionAdminServer) requireRecoveryLeader(ctx context.Context) error { + view := s.recoveryLeaderView + if view == nil { + view = s.leaderView + } + return requireEncryptionLeader(ctx, view) +} + +func requireEncryptionLeader(ctx context.Context, view raftengine.LeaderView) error { + if view == nil { return nil } - if s.leaderView.State() != raftengine.StateLeader { - leader := s.leaderView.Leader() + if view.State() != raftengine.StateLeader { + leader := view.Leader() if leader.ID == "" && leader.Address == "" { return grpcStatusError(codes.FailedPrecondition, "encryption: not leader (no known leader)") } @@ -1924,7 +1951,7 @@ func (s *EncryptionAdminServer) requireLeader(ctx context.Context) error { "encryption: not leader (current leader id=%q address=%q)", leader.ID, leader.Address) } - if err := s.leaderView.VerifyLeader(ctx); err != nil { + if err := view.VerifyLeader(ctx); err != nil { return verifyLeaderErrorToStatus(err) } return nil diff --git a/adapter/encryption_admin_test.go b/adapter/encryption_admin_test.go index e7e097f2a..21718ff21 100644 --- a/adapter/encryption_admin_test.go +++ b/adapter/encryption_admin_test.go @@ -1000,6 +1000,24 @@ func TestEncryptionAdmin_ResyncSidecar_RejectsStaleLeader(t *testing.T) { } } +func TestEncryptionAdmin_ResyncSidecar_UsesRecoveryLeaderView(t *testing.T) { + t.Parallel() + srv := NewEncryptionAdminServer( + WithEncryptionAdminLeaderView(stubLeaderView{state: raftengine.StateLeader}), + WithEncryptionAdminRecoveryLeaderView(stubLeaderView{ + state: raftengine.StateFollower, + leader: raftengine.LeaderInfo{ID: "default-leader", Address: "n2:50051"}, + }), + ) + _, err := srv.ResyncSidecar(context.Background(), &pb.ResyncSidecarRequest{}) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("ResyncSidecar status=%v, want FailedPrecondition from default-group recovery view", status.Code(err)) + } + if !strings.Contains(err.Error(), "default-leader") { + t.Fatalf("ResyncSidecar error %q does not identify the default-group leader", err) + } +} + // TestEncryptionAdmin_RotateDEK_VerifyLeader_PreservesContextCodes // pins the context-code mapping: when VerifyLeader returns // context.Canceled / context.DeadlineExceeded (the caller's ctx diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 25703d586..fc953defa 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -452,14 +452,16 @@ the way out. Envelope format (single byte stream, stored as the Pebble value): ```text -+--------+------+---------+----------+----------------+--------+ -| 0x01 | flag | key_id | nonce | ciphertext | tag | -| 1 byte | 1 B | 4 bytes | 12 bytes | N bytes | 16 B | -+--------+------+---------+----------+----------------+--------+ ++---------+------+---------+----------+----------------+--------+ +| version | flag | key_id | nonce | ciphertext | tag | +| 1 byte | 1 B | 4 bytes | 12 bytes | N bytes | 16 B | ++---------+------+---------+----------+----------------+--------+ ``` -- `0x01` — envelope-version byte. Future authenticated formats reserve - `0x02..0x0F` (see §11.3). The byte itself is **not** used to +- `version` — `0x01` is the original uncompressed format and requires + `flag=0`. `0x02` is the compression-capable storage format. Compressed + writes use `0x02`, causing V1-only readers to reject rather than return + Snappy-framed plaintext during rolling rollback. The byte itself is **not** used to discriminate cleartext from ciphertext on the read path — that decision lives in MVCC metadata, not in the value bytes (see §7.1). - `flag` — bit 0: `1` if `ciphertext` is the encryption of a @@ -1850,6 +1852,12 @@ so we never pay CPU twice. Because the flag is part of the AES-GCM AAD (§4.1), an attacker cannot flip it post-hoc to crash the decompressor. +Compressed values use envelope version `0x02`; uncompressed values remain +version `0x01`. This format split is the reader-capability boundary: a +pre-compression V1 reader fails closed on V2 instead of serving compressed +bytes. Visibility-only operations authenticate the envelope and value header +without allocating the decompressed value. + This is also the reason we cannot rely on Pebble's built-in block compression alone: by the time bytes reach Pebble's block writer they are already ciphertext, so Pebble's compression is wasted CPU. We @@ -2675,8 +2683,9 @@ eventual code change. top of value-level encryption, accepting the complexity tax. Not needed for the v1 threat model. -3. **Envelope-format versioning.** v1 ships envelope version `0x01`. - We will reserve `0x02..0x0F` for future authenticated formats +3. **Envelope-format versioning.** Uncompressed storage and Raft + envelopes use `0x01`; compression-capable storage envelopes use + `0x02`. We reserve `0x03..0x0F` for future authenticated formats (e.g., AES-256-GCM-SIV if nonce-misuse resistance matters, ChaCha20-Poly1305 for non-AES-NI hosts). The decrypt path dispatches on the version byte; anything unknown errors loudly. diff --git a/docs/design/2026_07_18_implemented_9a_encryption_compression.md b/docs/design/2026_07_18_implemented_9a_encryption_compression.md index 3d97b31dd..4e9082ff3 100644 --- a/docs/design/2026_07_18_implemented_9a_encryption_compression.md +++ b/docs/design/2026_07_18_implemented_9a_encryption_compression.md @@ -13,7 +13,8 @@ This milestone implements the compress-then-encrypt storage path from - The compressed representation is selected only when it is strictly smaller. - `FlagCompressed` is authenticated as part of the envelope AAD. - Reads decompress only after GCM authentication succeeds. -- Unknown v1 flag bits fail closed. +- V1 envelopes require `flag=0`; compressed values use envelope V2. +- Unknown version/flag combinations fail closed. - The cleartext-rebadge guard verifies both valid compression-flag variants. - Pebble block compression is disabled for encryption-wired stores, including database reopen and snapshot-restore temporary databases. @@ -24,16 +25,21 @@ pass after proposal decryption. ## Compatibility -Existing v1 envelopes have `flag=0` and remain readable. New readers accept -both `flag=0` and `FlagCompressed`; legacy cleartext operation and stores with -no encryption wiring retain Pebble's default block-compression policy. +Existing V1 envelopes have `flag=0` and remain readable. Compressed writes use +V2 with `FlagCompressed`, so a V1-only reader rejects the version instead of +returning Snappy-framed bytes as plaintext during a rolling rollback. Legacy +cleartext operation and stores with no encryption wiring retain Pebble's +default block-compression policy. Visibility-only scans authenticate V2 rows +without decompressing values they will discard. ## Verification - Storage round trips for compressed, uncompressed, empty, and already compressed values. - Authenticated compression-flag tamper rejection. +- V1-reader fail-closed rejection of compressed V2 envelopes. - Fail-closed handling for an authenticated malformed Snappy payload. +- Prefix-delete visibility checks that authenticate without Snappy expansion. - Property testing over arbitrary byte slices for compression framing. - Pebble compression-policy tests for encrypted and legacy stores. - Paired cleartext/encrypted 1 KiB storage benchmark for `benchstat` review. diff --git a/internal/encryption/envelope.go b/internal/encryption/envelope.go index 52ff934fb..d41499570 100644 --- a/internal/encryption/envelope.go +++ b/internal/encryption/envelope.go @@ -8,13 +8,14 @@ import ( // Public constants for the §4.1 wire format. const ( - // EnvelopeVersionV1 is the current envelope format version. §11.3 - // reserves 0x02..0x0F for future authenticated formats. The current - // build only understands 0x01; ANY other version byte (including the - // 0x02..0x0F reserved range) causes DecodeEnvelope to return - // ErrEnvelopeVersion. Future decoders that know how to handle the - // reserved range will widen this check. + // EnvelopeVersionV1 is the original uncompressed storage and Raft + // format. It rejects FlagCompressed so V1 readers never return a + // Snappy frame as user plaintext. EnvelopeVersionV1 byte = 0x01 + // EnvelopeVersionV2 identifies storage envelopes whose reader + // understands the authenticated compression flag. V1-only readers + // reject this version before decrypting, making rollback fail closed. + EnvelopeVersionV2 byte = 0x02 // FlagCompressed (bit 0) is set when ciphertext encrypts a Snappy- // compressed plaintext (§6.4). The flag participates in the AAD so a @@ -82,17 +83,12 @@ type Envelope struct { // (uninitialised Version, truncated Body) fails here with a clear // stack trace, rather than surfacing later as a confusing // DecodeEnvelope or Cipher.Decrypt failure on the read side. Returns: -// - ErrEnvelopeVersion if Version is not EnvelopeVersionV1. +// - ErrEnvelopeVersion if Version is unsupported. // - ErrEnvelopeShort if Body is shorter than TagSize (every valid // body must contain at least the GCM tag). func (e *Envelope) Encode() ([]byte, error) { - if e.Version != EnvelopeVersionV1 { - return nil, errors.Wrapf(ErrEnvelopeVersion, - "encode: got 0x%02x, want 0x%02x", e.Version, EnvelopeVersionV1) - } - if e.Flag&^FlagCompressed != 0 { - return nil, errors.Wrapf(ErrEnvelopeFlag, - "encode: got 0x%02x, allowed mask 0x%02x", e.Flag, FlagCompressed) + if err := validateEnvelopeVersionFlag(e.Version, e.Flag); err != nil { + return nil, errors.Wrap(err, "encode") } if len(e.Body) < TagSize { return nil, errors.Wrapf(ErrEnvelopeShort, @@ -116,13 +112,8 @@ func DecodeEnvelope(src []byte) (*Envelope, error) { return nil, errors.Wrapf(ErrEnvelopeShort, "got %d bytes, want >= %d", len(src), HeaderSize+TagSize) } - if src[versionOffset] != EnvelopeVersionV1 { - return nil, errors.Wrapf(ErrEnvelopeVersion, - "got 0x%02x, want 0x%02x", src[versionOffset], EnvelopeVersionV1) - } - if src[flagOffset]&^FlagCompressed != 0 { - return nil, errors.Wrapf(ErrEnvelopeFlag, - "got 0x%02x, allowed mask 0x%02x", src[flagOffset], FlagCompressed) + if err := validateEnvelopeVersionFlag(src[versionOffset], src[flagOffset]); err != nil { + return nil, err } e := &Envelope{ Version: src[versionOffset], @@ -136,6 +127,26 @@ func DecodeEnvelope(src []byte) (*Envelope, error) { return e, nil } +func validateEnvelopeVersionFlag(version, flag byte) error { + switch version { + case EnvelopeVersionV1: + if flag != 0 { + return errors.Wrapf(ErrEnvelopeFlag, + "version 0x%02x requires flag 0x00, got 0x%02x", version, flag) + } + case EnvelopeVersionV2: + if flag&^FlagCompressed != 0 { + return errors.Wrapf(ErrEnvelopeFlag, + "version 0x%02x got flag 0x%02x, allowed mask 0x%02x", version, flag, FlagCompressed) + } + default: + return errors.Wrapf(ErrEnvelopeVersion, + "got 0x%02x, supported versions are 0x%02x and 0x%02x", + version, EnvelopeVersionV1, EnvelopeVersionV2) + } + return nil +} + // HeaderAADBytes returns the first 6 bytes of the envelope header // (version, flag, key_id) in their on-disk order. These bytes participate // in the §4.1 storage-layer AAD (storage AAD = HeaderAADBytes ‖ pebble_key) diff --git a/internal/encryption/envelope_test.go b/internal/encryption/envelope_test.go index 2d041a59b..49a550b6a 100644 --- a/internal/encryption/envelope_test.go +++ b/internal/encryption/envelope_test.go @@ -26,7 +26,7 @@ func TestEnvelope_RoundTrip(t *testing.T) { { name: "compressed flag set", env: encryption.Envelope{ - Version: encryption.EnvelopeVersionV1, + Version: encryption.EnvelopeVersionV2, Flag: encryption.FlagCompressed, KeyID: 0xCAFE_F00D, Body: bytes.Repeat([]byte{0x55}, 64+encryption.TagSize), @@ -35,7 +35,7 @@ func TestEnvelope_RoundTrip(t *testing.T) { { name: "max key_id", env: encryption.Envelope{ - Version: encryption.EnvelopeVersionV1, + Version: encryption.EnvelopeVersionV2, Flag: encryption.FlagCompressed, KeyID: 0xFFFF_FFFF, Body: bytes.Repeat([]byte{0x77}, 1024+encryption.TagSize), @@ -120,7 +120,7 @@ func TestEnvelope_Decode_TooShort(t *testing.T) { } func TestEnvelope_Decode_RejectsUnknownVersion(t *testing.T) { - for _, version := range []byte{0x00, 0x02, 0x10, 0xFE, 0xFF} { + for _, version := range []byte{0x00, 0x03, 0x10, 0xFE, 0xFF} { buf := make([]byte, encryption.HeaderSize+encryption.TagSize) buf[0] = version _, err := encryption.DecodeEnvelope(buf) @@ -130,6 +130,25 @@ func TestEnvelope_Decode_RejectsUnknownVersion(t *testing.T) { } } +func TestEnvelope_V1RejectsCompressedFlag(t *testing.T) { + t.Parallel() + env := encryption.Envelope{ + Version: encryption.EnvelopeVersionV1, + Flag: encryption.FlagCompressed, + KeyID: 1, + Body: bytes.Repeat([]byte{0x55}, encryption.TagSize), + } + if _, err := env.Encode(); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("V1 compressed encode: expected ErrEnvelopeFlag, got %v", err) + } + buf := make([]byte, encryption.HeaderSize+encryption.TagSize) + buf[0] = encryption.EnvelopeVersionV1 + buf[1] = encryption.FlagCompressed + if _, err := encryption.DecodeEnvelope(buf); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("V1 compressed decode: expected ErrEnvelopeFlag, got %v", err) + } +} + func TestEnvelope_RejectsUnknownFlagBits(t *testing.T) { t.Parallel() for _, flag := range []byte{0x02, 0x80, 0xff} { @@ -172,8 +191,8 @@ func TestEnvelope_Decode_DoesNotAliasInput(t *testing.T) { } func TestHeaderAADBytes_Layout(t *testing.T) { - got := encryption.HeaderAADBytes(encryption.EnvelopeVersionV1, encryption.FlagCompressed, 0x12345678) - want := []byte{0x01, 0x01, 0x12, 0x34, 0x56, 0x78} + got := encryption.HeaderAADBytes(encryption.EnvelopeVersionV2, encryption.FlagCompressed, 0x12345678) + want := []byte{0x02, 0x01, 0x12, 0x34, 0x56, 0x78} if !bytes.Equal(got, want) { t.Fatalf("HeaderAADBytes layout mismatch:\n got %x\n want %x", got, want) } @@ -183,8 +202,8 @@ func TestAppendHeaderAADBytes_AppendsAndAllocFree(t *testing.T) { // Append onto an existing buffer. The result should be the original // bytes followed by the 6-byte header. prefix := []byte{0xCA, 0xFE} - got := encryption.AppendHeaderAADBytes(prefix, encryption.EnvelopeVersionV1, encryption.FlagCompressed, 0x12345678) - want := []byte{0xCA, 0xFE, 0x01, 0x01, 0x12, 0x34, 0x56, 0x78} + got := encryption.AppendHeaderAADBytes(prefix, encryption.EnvelopeVersionV2, encryption.FlagCompressed, 0x12345678) + want := []byte{0xCA, 0xFE, 0x02, 0x01, 0x12, 0x34, 0x56, 0x78} if !bytes.Equal(got, want) { t.Fatalf("AppendHeaderAADBytes mismatch:\n got %x\n want %x", got, want) } @@ -202,7 +221,7 @@ func TestAppendHeaderAADBytes_AppendsAndAllocFree(t *testing.T) { func TestEnvelope_Encode_RejectsBadVersion(t *testing.T) { env := encryption.Envelope{ - Version: 0x02, // not EnvelopeVersionV1 + Version: 0x03, Body: bytes.Repeat([]byte{0xAA}, encryption.TagSize), } _, err := env.Encode() diff --git a/internal/encryption/errors.go b/internal/encryption/errors.go index 836247cdb..fcec5a201 100644 --- a/internal/encryption/errors.go +++ b/internal/encryption/errors.go @@ -35,9 +35,9 @@ var ( // current build does not know how to parse. Reserved values per §11.3. ErrEnvelopeVersion = errors.New("encryption: unknown envelope version") - // ErrEnvelopeFlag indicates an envelope set a flag bit that the current - // format does not define. V1 only permits FlagCompressed; accepting other - // bits would make future format extensions ambiguous and could route + // ErrEnvelopeFlag indicates an envelope set a flag bit that its version + // does not define. V1 permits no flags; V2 permits FlagCompressed. Accepting + // other combinations would make extensions ambiguous and could route // authenticated plaintext through the wrong post-decrypt transform. ErrEnvelopeFlag = errors.New("encryption: unknown envelope flag bits") diff --git a/internal/encryption/raft_envelope.go b/internal/encryption/raft_envelope.go index 16df2aa75..0610917cb 100644 --- a/internal/encryption/raft_envelope.go +++ b/internal/encryption/raft_envelope.go @@ -109,6 +109,10 @@ func UnwrapRaftPayload(c *Cipher, encoded []byte) ([]byte, error) { return nil, errors.Wrapf(ErrEnvelopeFlag, "encryption: raft envelope flag must be 0x00, got 0x%02x", env.Flag) } + if env.Version != EnvelopeVersionV1 { + return nil, errors.Wrapf(ErrEnvelopeVersion, + "encryption: raft envelope version must be 0x%02x, got 0x%02x", EnvelopeVersionV1, env.Version) + } aad := BuildRaftAAD(env.Version, env.KeyID) plain, err := c.Decrypt(env.Body, aad, env.KeyID, env.Nonce[:]) if err != nil { diff --git a/internal/encryption/raft_envelope_test.go b/internal/encryption/raft_envelope_test.go index 23f837455..7fddd8c48 100644 --- a/internal/encryption/raft_envelope_test.go +++ b/internal/encryption/raft_envelope_test.go @@ -177,7 +177,7 @@ func TestRaftEnvelope_RejectsStorageCompressionFlag(t *testing.T) { c, keyID := raftFixture(t) nonce := newRandomNonce(t) payload := []byte("raft payload") - aad := encryption.BuildRaftAAD(encryption.EnvelopeVersionV1, keyID) + aad := encryption.BuildRaftAAD(encryption.EnvelopeVersionV2, keyID) body, err := c.Encrypt(payload, aad, keyID, nonce) if err != nil { t.Fatalf("Encrypt: %v", err) @@ -185,7 +185,7 @@ func TestRaftEnvelope_RejectsStorageCompressionFlag(t *testing.T) { var nonceArray [encryption.NonceSize]byte copy(nonceArray[:], nonce) encoded, err := (&encryption.Envelope{ - Version: encryption.EnvelopeVersionV1, + Version: encryption.EnvelopeVersionV2, Flag: encryption.FlagCompressed, KeyID: keyID, Nonce: nonceArray, diff --git a/main.go b/main.go index 9159a1ed1..f6c302fce 100644 --- a/main.go +++ b/main.go @@ -1554,6 +1554,14 @@ func writerRegistryForEncryptionAdmin(runtimes []*raftGroupRuntime, defaultGroup return defaultRuntime.writerRegistry } +func recoveryStateForEncryptionAdmin(runtimes []*raftGroupRuntime, defaultGroup uint64) (raftengine.LeaderView, func() uint64) { + defaultRuntime := findDefaultGroupRuntime(runtimes, defaultGroup) + if defaultRuntime == nil { + return nil, nil + } + return defaultRuntime.engine, appliedIndexForEngine(defaultRuntime.engine) +} + func appliedIndexForEngine(engine raftengine.Engine) func() uint64 { applied, ok := engine.(interface{ AppliedIndex() uint64 }) if !ok { @@ -2633,6 +2641,7 @@ func startRaftServers( enableMutators := encryptionMutatorsEnabled() encryptionCapabilityFanout := buildEncryptionCapabilityFanout(ctx, eg, runtimes, enableMutators) adminWriterRegistry := writerRegistryForEncryptionAdmin(runtimes, defaultGroup) + recoveryLeaderView, recoveryAppliedIndex := recoveryStateForEncryptionAdmin(runtimes, defaultGroup) for _, rt := range runtimes { baseOpts := internalutil.GRPCServerOptions() opts := make([]grpc.ServerOption, 0, len(baseOpts)+extraOptsCap) @@ -2685,9 +2694,10 @@ func startRaftServers( // a stable, distinct value. Codex r1 P1 on PR #760. // Stage 6B-2 mutator gate is resolved once above the // per-shard loop. Each shard's own engine remains the raw - // Proposer + LeaderView for the cutover marker, while - // ShardGroup.Proposer() supplies the wrap-aware post-cutover - // path for normal admin entries. + // proposer, while recovery leadership and applied-index evidence + // come from the default group that owns the shared sidecar and + // writer registry. ShardGroup.Proposer() supplies the wrap-aware + // post-cutover path for normal admin entries. registerEncryptionAdminServer( gs, etcdraftengine.DeriveNodeID(*raftId), @@ -2696,7 +2706,8 @@ func startRaftServers( rt.engine, encryptionCapabilityFanout, adapter.WithEncryptionAdminWriterRegistry(adminWriterRegistry), - adapter.WithEncryptionAdminLatestAppliedIndex(appliedIndexForEngine(rt.engine)), + adapter.WithEncryptionAdminRecoveryLeaderView(recoveryLeaderView), + adapter.WithEncryptionAdminLatestAppliedIndex(recoveryAppliedIndex), adapter.WithEncryptionAdminPostCutoverProposer(proposerForGroup(rt, shardGroups)), adapter.WithEncryptionAdminCutoverBarrier(encWiring.raftEnvelope.barrier()), ) diff --git a/main_encryption_admin_test.go b/main_encryption_admin_test.go index 96a7743be..77186e43f 100644 --- a/main_encryption_admin_test.go +++ b/main_encryption_admin_test.go @@ -46,6 +46,15 @@ type stubEncryptionAdminRegistry struct { name string } +type stubEncryptionRecoveryEngine struct { + raftengine.Engine + applied uint64 +} + +func (s *stubEncryptionRecoveryEngine) AppliedIndex() uint64 { + return s.applied +} + func (stubEncryptionAdminRegistry) GetRegistryRow([]byte) ([]byte, bool, error) { return nil, false, nil } @@ -103,6 +112,35 @@ func TestWriterRegistryForEncryptionAdminMissingDefaultGroup(t *testing.T) { } } +func TestRecoveryStateForEncryptionAdminUsesDefaultGroup(t *testing.T) { + defaultEngine := &stubEncryptionRecoveryEngine{applied: 41} + otherEngine := &stubEncryptionRecoveryEngine{applied: 99} + runtimes := []*raftGroupRuntime{ + {spec: groupSpec{id: 2}, engine: otherEngine}, + {spec: groupSpec{id: 1}, engine: defaultEngine}, + } + + leaderView, appliedIndex := recoveryStateForEncryptionAdmin(runtimes, 1) + if leaderView != defaultEngine { + t.Fatalf("recoveryStateForEncryptionAdmin selected %T, want default group engine", leaderView) + } + if appliedIndex == nil { + t.Fatal("recoveryStateForEncryptionAdmin returned a nil applied-index callback") + } + if got := appliedIndex(); got != 41 { + t.Fatalf("recoveryStateForEncryptionAdmin applied index = %d, want 41", got) + } +} + +func TestRecoveryStateForEncryptionAdminMissingDefaultGroup(t *testing.T) { + leaderView, appliedIndex := recoveryStateForEncryptionAdmin([]*raftGroupRuntime{ + {spec: groupSpec{id: 2}, engine: &stubEncryptionRecoveryEngine{applied: 99}}, + }, 1) + if leaderView != nil || appliedIndex != nil { + t.Fatalf("recoveryStateForEncryptionAdmin returned leader=%T callback-present=%t, want both nil", leaderView, appliedIndex != nil) + } +} + // TestEncryptionAdmin_MutatingRPCRefusedWhenGateOff pins the // Stage 6B-2 double-gate. With either condition of the gate false // (enableMutators=false OR engine=nil), Proposer + LeaderView diff --git a/store/encryption_compression_test.go b/store/encryption_compression_test.go index 3509bd788..941dc8a2a 100644 --- a/store/encryption_compression_test.go +++ b/store/encryption_compression_test.go @@ -40,7 +40,7 @@ func TestEncryption_CompressesOnlyWhenSmaller(t *testing.T) { t.Fatalf("PutAt: %v", err) } - var flag byte + var version, flag byte f.tamperPebbleValue(t, key, 100, func(raw []byte) []byte { sv, err := decodeValue(raw) if err != nil { @@ -50,10 +50,16 @@ func TestEncryption_CompressesOnlyWhenSmaller(t *testing.T) { if err != nil { t.Fatalf("DecodeEnvelope: %v", err) } + version = env.Version flag = env.Flag return raw }) require.Equal(t, tc.compressed, flag&encryption.FlagCompressed != 0) + if tc.compressed { + require.Equal(t, encryption.EnvelopeVersionV2, version) + } else { + require.Equal(t, encryption.EnvelopeVersionV1, version) + } got, err := f.mvcc.GetAt(context.Background(), key, 100) require.NoError(t, err) @@ -105,7 +111,7 @@ func TestEncryption_RejectsAuthenticatedMalformedSnappy(t *testing.T) { pebbleKey := encodeKey(key, commitTS) var header [valueHeaderSize]byte writeValueHeaderBytes(header[:], false, 0, encStateEncrypted) - aad := buildStorageAAD(encryption.EnvelopeVersionV1, encryption.FlagCompressed, f.keyID, header[:], pebbleKey) + aad := buildStorageAAD(encryption.EnvelopeVersionV2, encryption.FlagCompressed, f.keyID, header[:], pebbleKey) var nonce [encryption.NonceSize]byte if _, err := rand.Read(nonce[:]); err != nil { t.Fatalf("rand.Read nonce: %v", err) @@ -115,7 +121,7 @@ func TestEncryption_RejectsAuthenticatedMalformedSnappy(t *testing.T) { t.Fatalf("Encrypt malformed payload: %v", err) } envelopeBytes, err := (&encryption.Envelope{ - Version: encryption.EnvelopeVersionV1, + Version: encryption.EnvelopeVersionV2, Flag: encryption.FlagCompressed, KeyID: f.keyID, Nonce: nonce, @@ -142,6 +148,9 @@ func TestEncryption_RejectsAuthenticatedMalformedSnappy(t *testing.T) { if _, err := f.mvcc.GetAt(context.Background(), key, commitTS); !errors.Is(err, ErrEncryptedReadCompression) { t.Fatalf("malformed authenticated Snappy: expected ErrEncryptedReadCompression, got %v", err) } + if err := f.mvcc.DeletePrefixAt(context.Background(), []byte("malformed-"), nil, commitTS+1); err != nil { + t.Fatalf("visibility-only prefix delete expanded malformed Snappy: %v", err) + } } func TestEncryption_RejectsAuthenticatedOversizedSnappyBeforeDecode(t *testing.T) { @@ -155,7 +164,7 @@ func TestEncryption_RejectsAuthenticatedOversizedSnappyBeforeDecode(t *testing.T pebbleKey := encodeKey(key, commitTS) var header [valueHeaderSize]byte writeValueHeaderBytes(header[:], false, 0, encStateEncrypted) - aad := buildStorageAAD(encryption.EnvelopeVersionV1, encryption.FlagCompressed, f.keyID, header[:], pebbleKey) + aad := buildStorageAAD(encryption.EnvelopeVersionV2, encryption.FlagCompressed, f.keyID, header[:], pebbleKey) compressed := snappy.Encode(nil, bytes.Repeat([]byte("a"), maxSnapshotValueSize+1)) var nonce [encryption.NonceSize]byte if _, err := rand.Read(nonce[:]); err != nil { @@ -166,7 +175,7 @@ func TestEncryption_RejectsAuthenticatedOversizedSnappyBeforeDecode(t *testing.T t.Fatalf("Encrypt oversized payload: %v", err) } envelopeBytes, err := (&encryption.Envelope{ - Version: encryption.EnvelopeVersionV1, + Version: encryption.EnvelopeVersionV2, Flag: encryption.FlagCompressed, KeyID: f.keyID, Nonce: nonce, diff --git a/store/encryption_glue.go b/store/encryption_glue.go index 43492566b..506a7a79e 100644 --- a/store/encryption_glue.go +++ b/store/encryption_glue.go @@ -356,15 +356,19 @@ func (s *pebbleStore) encryptForKey(pebbleKey, plaintext []byte, expireAt uint64 } nonce := nonceArr[:] payload, envelopeFlag := compressForEncryption(plaintext) + envelopeVersion := encryption.EnvelopeVersionV1 + if envelopeFlag&encryption.FlagCompressed != 0 { + envelopeVersion = encryption.EnvelopeVersionV2 + } var hdr [valueHeaderSize]byte writeValueHeaderBytes(hdr[:], false /*tombstone*/, expireAt, encStateEncrypted) - aad := buildStorageAAD(encryption.EnvelopeVersionV1, envelopeFlag, keyID, hdr[:], pebbleKey) + aad := buildStorageAAD(envelopeVersion, envelopeFlag, keyID, hdr[:], pebbleKey) ciphertextAndTag, err := s.cipher.Encrypt(payload, aad, keyID, nonce) if err != nil { return nil, 0, errors.Wrap(err, "store: encrypt value") } env := encryption.Envelope{ - Version: encryption.EnvelopeVersionV1, + Version: envelopeVersion, Flag: envelopeFlag, KeyID: keyID, Nonce: nonceArr, @@ -397,6 +401,18 @@ func (s *pebbleStore) encryptForKey(pebbleKey, plaintext []byte, expireAt uint64 // tombstone / expireAt visibility checks AFTER decrypt succeeds — // the values they observe pre-decrypt are not yet authenticated. func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byte) ([]byte, error) { + return s.decryptForKeyMode(pebbleKey, sv, body, true) +} + +// authenticateForKey authenticates the envelope and value header without +// expanding compressed plaintext. Visibility-only callers use this before +// branching on tombstone or expiry fields. +func (s *pebbleStore) authenticateForKey(pebbleKey []byte, sv storedValue, body []byte) error { + _, err := s.decryptForKeyMode(pebbleKey, sv, body, false) + return err +} + +func (s *pebbleStore) decryptForKeyMode(pebbleKey []byte, sv storedValue, body []byte, decompress bool) ([]byte, error) { if sv.EncState == encStateCleartext { if err := s.rejectRebadgedEnvelope(pebbleKey, sv, body); err != nil { return nil, err @@ -422,9 +438,11 @@ func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byt } return nil, errors.Wrap(err, "store: decrypt value") } - plain, err = decompressAuthenticatedValue(plain, env.Flag) - if err != nil { - return nil, err + if decompress { + plain, err = decompressAuthenticatedValue(plain, env.Flag) + if err != nil { + return nil, err + } } // AES-GCM Open returns a nil dst slice for an empty plaintext; // upstream callers (notably ExistsAt) distinguish "key absent" @@ -501,11 +519,9 @@ func compressForEncryption(plaintext []byte) ([]byte, byte) { // uses, instead of trusting the on-disk header bytes the attacker // could have flipped: // -// - envelope_version = EnvelopeVersionV1 (the encrypt path's -// fixed value; trusting on-disk would let a corrupted version -// byte force the body through DecodeEnvelope's error path) -// - flag: both supported values (uncompressed and Snappy-compressed), -// because the attacker can rewrite the on-disk flag byte +// - version/flag = V1/uncompressed or V2/Snappy-compressed (the two +// combinations emitted by the encrypt path; trusting on-disk would +// let corrupted format bytes bypass the trial) // - tombstone = false (the encrypt path never wraps // tombstones, so any on-disk tombstone bit on an encrypted // entry is necessarily attacker-supplied) @@ -553,10 +569,16 @@ func (s *pebbleStore) rejectRebadgedEnvelope(pebbleKey []byte, sv storedValue, b } for _, kid := range s.cipher.LoadedKeyIDs() { for _, candidateExpire := range candidateExpireAts { - for _, candidateFlag := range []byte{0, encryption.FlagCompressed} { + for _, candidate := range []struct { + version byte + flag byte + }{ + {version: encryption.EnvelopeVersionV1}, + {version: encryption.EnvelopeVersionV2, flag: encryption.FlagCompressed}, + } { var hdr [valueHeaderSize]byte writeValueHeaderBytes(hdr[:], false /*canonical*/, candidateExpire, encStateEncrypted) - aad := buildStorageAAD(encryption.EnvelopeVersionV1, candidateFlag, kid, hdr[:], pebbleKey) + aad := buildStorageAAD(candidate.version, candidate.flag, kid, hdr[:], pebbleKey) if _, err := s.cipher.Decrypt(ct, aad, kid, nonce); err == nil { return errors.Wrap(ErrEncryptedReadIntegrity, "store: cleartext-labelled value verifies as a relabeled envelope under a loaded DEK") diff --git a/store/lsm_store.go b/store/lsm_store.go index 6eaba5aa0..8f2824d29 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -1279,7 +1279,7 @@ func (s *pebbleStore) foundValueVisible(iter *pebble.Iterator, ts uint64) (bool, // This intentionally also runs for cleartext-labelled rows so the // cleartext rebadge guard rejects encrypted envelopes whose encState bit // was flipped before we branch on tombstone or expireAt. - if _, err := s.decryptForKey(iter.Key(), sv, sv.Value); err != nil { + if err := s.authenticateForKey(iter.Key(), sv, sv.Value); err != nil { return false, err } return !sv.Tombstone && (sv.ExpireAt == 0 || sv.ExpireAt > ts), nil @@ -2617,12 +2617,12 @@ func (s *pebbleStore) isVisibleLiveKey(iter *pebble.Iterator, userKey []byte, ve if err != nil { return false, errors.WithStack(err) } - // decryptForKey authenticates the value-header bytes when the + // authenticateForKey authenticates the value-header bytes when the // entry is encrypted (cleartext entries no-op except for the // rebadge guard). We discard the plaintext — we only need the // authentication side-effect; tombstone / expireAt visibility // is then decided on now-trusted bytes. - if _, err := s.decryptForKey(iter.Key(), sv, sv.Value); err != nil { + if err := s.authenticateForKey(iter.Key(), sv, sv.Value); err != nil { return false, err } if sv.Tombstone || (sv.ExpireAt != 0 && sv.ExpireAt <= commitTS) { From 738b64fadf55d021ff787d578167e3f48dd4b185 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 23:00:31 +0900 Subject: [PATCH 08/16] store: preserve encryption policy in SST snapshots --- store/snapshot_pebble_sst.go | 34 ++++++++++++++++--------------- store/snapshot_pebble_sst_test.go | 30 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/store/snapshot_pebble_sst.go b/store/snapshot_pebble_sst.go index 1c495a38d..500eb0107 100644 --- a/store/snapshot_pebble_sst.go +++ b/store/snapshot_pebble_sst.go @@ -66,13 +66,14 @@ type pebbleSSTIngestFileMeta struct { } type pebbleSSTIngestSnapshot struct { - checkpointDir string - fallback *pebbleSnapshot - metadata pebbleSnapshotMetadata - targetFileBytes uint64 - log *slog.Logger - once sync.Once - err error + checkpointDir string + fallback *pebbleSnapshot + metadata pebbleSnapshotMetadata + targetFileBytes uint64 + disableCompression bool + log *slog.Logger + once sync.Once + err error } type pebbleSSTIngestBundle struct { @@ -231,9 +232,10 @@ func (s *pebbleStore) newSSTIngestSnapshot() (Snapshot, error) { snapshot: pebbleSnap, lastCommitTS: metadata.LastCommitTS, }, - metadata: metadata, - targetFileBytes: targetFileBytes, - log: s.log, + metadata: metadata, + targetFileBytes: targetFileBytes, + disableCompression: s.cipher != nil, + log: s.log, }, nil } @@ -296,7 +298,7 @@ func (s *pebbleSSTIngestSnapshot) WriteTo(w io.Writer) (int64, error) { if s == nil || s.fallback == nil { return 0, errors.New("snapshot is not available") } - bundle, err := buildPebbleSSTIngestBundle(s.checkpointDir, s.metadata, s.targetFileBytes) + bundle, err := buildPebbleSSTIngestBundle(s.checkpointDir, s.metadata, s.targetFileBytes, s.disableCompression) if err != nil { if s.log != nil { s.log.Warn("failed to build SST ingest snapshot; using legacy stream", "error", err) @@ -326,8 +328,8 @@ func (s *pebbleSSTIngestSnapshot) Close() error { return errors.WithStack(s.err) } -func buildPebbleSSTIngestBundle(checkpointDir string, metadata pebbleSnapshotMetadata, targetFileBytes uint64) (*pebbleSSTIngestBundle, error) { - opts, cache := defaultPebbleOptionsWithCache(false) +func buildPebbleSSTIngestBundle(checkpointDir string, metadata pebbleSnapshotMetadata, targetFileBytes uint64, disableCompression bool) (*pebbleSSTIngestBundle, error) { + opts, cache := defaultPebbleOptionsWithCache(disableCompression) opts.ReadOnly = true db, err := pebble.Open(checkpointDir, opts) if err != nil { @@ -593,7 +595,7 @@ func (s *pebbleStore) restorePebbleSSTIngestAtomic(r io.Reader) error { return err } metadata := metadataFromSSTIngestManifest(manifest) - if err := ingestSSTSnapshotIntoTempDB(tmpDir, paths, metadata); err != nil { + if err := ingestSSTSnapshotIntoTempDB(tmpDir, paths, metadata, s.cipher != nil); err != nil { _ = os.RemoveAll(tmpDir) return err } @@ -760,8 +762,8 @@ func metadataFromSSTIngestManifest(manifest pebbleSSTIngestManifest) pebbleSnaps return metadata } -func ingestSSTSnapshotIntoTempDB(tmpDir string, paths []string, metadata pebbleSnapshotMetadata) error { - opts, cache := defaultPebbleOptionsWithCache(false) +func ingestSSTSnapshotIntoTempDB(tmpDir string, paths []string, metadata pebbleSnapshotMetadata, disableCompression bool) error { + opts, cache := defaultPebbleOptionsWithCache(disableCompression) db, err := pebble.Open(tmpDir, opts) if err != nil { cache.Unref() diff --git a/store/snapshot_pebble_sst_test.go b/store/snapshot_pebble_sst_test.go index 64277a263..0a2139ea9 100644 --- a/store/snapshot_pebble_sst_test.go +++ b/store/snapshot_pebble_sst_test.go @@ -12,6 +12,7 @@ import ( "testing/iotest" internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/encryption" "github.com/stretchr/testify/require" ) @@ -122,6 +123,35 @@ func TestPebbleStoreSSTIngestSnapshotRoundTrip(t *testing.T) { require.NoDirExists(t, checkpointDir) } +func TestPebbleStoreEncryptedSSTIngestSnapshotRoundTrip(t *testing.T) { + setPebbleCacheBytesForTest(t, 8<<20) + const keyID = uint32(7) + ks := encryption.NewKeystore() + require.NoError(t, ks.Set(keyID, bytes.Repeat([]byte{0x47}, encryption.KeySize))) + cipher, err := encryption.NewCipher(ks) + require.NoError(t, err) + encryptionOption := func(epoch uint16) PebbleStoreOption { + return WithEncryption(cipher, NewCounterNonceFactory(1, epoch), func() (uint32, bool) { + return keyID, true + }) + } + + src := openSSTSnapshotTestStore(t, filepath.Join(t.TempDir(), "src"), + WithSSTIngestSnapshots(true), encryptionOption(1)) + writeSSTSnapshotTestData(t, src) + raw, snapshot := snapshotBytesForTest(t, src) + sstSnapshot, ok := snapshot.(*pebbleSSTIngestSnapshot) + require.True(t, ok) + require.True(t, sstSnapshot.disableCompression) + require.NoError(t, snapshot.Close()) + + dst := openSSTSnapshotTestStore(t, filepath.Join(t.TempDir(), "dst"), encryptionOption(2)) + require.NoError(t, dst.Restore(bytes.NewReader(raw))) + value, err := dst.GetAt(context.Background(), []byte("alpha"), 100) + require.NoError(t, err) + require.Equal(t, strings.Repeat("alpha", 32), string(value)) +} + func TestPebbleStoreSSTIngestSnapshotCorruptionPreservesDestination(t *testing.T) { setPebbleCacheBytesForTest(t, 8<<20) src := openSSTSnapshotTestStore(t, filepath.Join(t.TempDir(), "src"), WithSSTIngestSnapshots(true)) From e34b4de8a67e052e0b38d58a9db15b8bf1c5d0e3 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 20 Jul 2026 00:24:51 +0900 Subject: [PATCH 09/16] encryption: gate compressed writes by cluster capability --- adapter/encryption_admin.go | 12 ++-- adapter/encryption_admin_test.go | 6 ++ ...8_implemented_9a_encryption_compression.md | 5 ++ internal/admin/capability_fanout.go | 29 +++++++-- internal/admin/capability_fanout_test.go | 51 +++++++++++++-- internal/raftadmin/interceptor.go | 6 +- internal/raftadmin/server.go | 4 +- internal/raftadmin/server_test.go | 10 +-- main.go | 19 ++++-- main_encryption_admin.go | 33 +++++++++- main_encryption_admin_test.go | 12 ++++ main_encryption_confchange.go | 63 ++++++++++++++++--- main_encryption_confchange_test.go | 48 ++++++++++++-- main_encryption_fanout.go | 36 +++++++++++ main_encryption_fanout_test.go | 32 ++++++++++ main_encryption_write_wiring.go | 15 +++++ main_encryption_write_wiring_test.go | 23 +++++-- proto/encryption_admin.pb.go | 19 ++++-- proto/encryption_admin.proto | 3 + store/encryption_compression_test.go | 41 ++++++++++++ store/encryption_glue.go | 23 ++++++- store/lsm_store.go | 5 ++ store/snapshot_pebble_sst.go | 14 ++++- store/snapshot_pebble_sst_test.go | 7 +++ 24 files changed, 459 insertions(+), 57 deletions(-) diff --git a/adapter/encryption_admin.go b/adapter/encryption_admin.go index bfbe5f99d..4844dbb15 100644 --- a/adapter/encryption_admin.go +++ b/adapter/encryption_admin.go @@ -371,12 +371,16 @@ func (s *EncryptionAdminServer) Validate() error { // case so the empty epoch never reaches the writer registry. func (s *EncryptionAdminServer) GetCapability(_ context.Context, _ *pb.Empty) (*pb.CapabilityReport, error) { if s.sidecarPath == "" { - return &pb.CapabilityReport{BuildSha: s.buildSHA}, nil + return &pb.CapabilityReport{ + BuildSha: s.buildSHA, + StorageEnvelopeV2Capable: true, + }, nil } report := &pb.CapabilityReport{ - EncryptionCapable: true, - BuildSha: s.buildSHA, - FullNodeId: s.fullNodeID, + EncryptionCapable: true, + StorageEnvelopeV2Capable: true, + BuildSha: s.buildSHA, + FullNodeId: s.fullNodeID, // LocalEpoch stays at 0 until Stage 7 wires the §4.1 // writer-registry counter. The §5.6 step 1a pre-check // happens before any DEK exists, so 0 is the correct diff --git a/adapter/encryption_admin_test.go b/adapter/encryption_admin_test.go index 21718ff21..a6b8bdb41 100644 --- a/adapter/encryption_admin_test.go +++ b/adapter/encryption_admin_test.go @@ -39,6 +39,9 @@ func TestEncryptionAdmin_GetCapability_NoSidecarPath(t *testing.T) { if got.BuildSha != "test-sha" { t.Errorf("BuildSha=%q, want %q", got.BuildSha, "test-sha") } + if !got.StorageEnvelopeV2Capable { + t.Error("StorageEnvelopeV2Capable=false on a V2 reader binary") + } } func TestEncryptionAdmin_GetCapability_SidecarMissing(t *testing.T) { @@ -64,6 +67,9 @@ func TestEncryptionAdmin_GetCapability_SidecarMissing(t *testing.T) { if got.SidecarPresent { t.Errorf("SidecarPresent=true, want false when sidecar file is missing") } + if !got.StorageEnvelopeV2Capable { + t.Error("StorageEnvelopeV2Capable=false on a V2 reader binary") + } } func TestEncryptionAdmin_GetCapability_NotBootstrapped(t *testing.T) { diff --git a/docs/design/2026_07_18_implemented_9a_encryption_compression.md b/docs/design/2026_07_18_implemented_9a_encryption_compression.md index 4e9082ff3..6e88b7bb9 100644 --- a/docs/design/2026_07_18_implemented_9a_encryption_compression.md +++ b/docs/design/2026_07_18_implemented_9a_encryption_compression.md @@ -14,6 +14,11 @@ This milestone implements the compress-then-encrypt storage path from - `FlagCompressed` is authenticated as part of the envelope AAD. - Reads decompress only after GCM authentication succeeds. - V1 envelopes require `flag=0`; compressed values use envelope V2. +- V2 writes remain disabled until every live voter and learner advertises the + V2 reader capability; absent fields from older binaries fail closed. +- Once encryption is bootstrapped, AddVoter/AddLearner probes the target + endpoint and refuses members that do not advertise encryption and V2 reader + capability before proposing the membership change. - Unknown version/flag combinations fail closed. - The cleartext-rebadge guard verifies both valid compression-flag variants. - Pebble block compression is disabled for encryption-wired stores, including diff --git a/internal/admin/capability_fanout.go b/internal/admin/capability_fanout.go index 921b7c5cf..013eb851a 100644 --- a/internal/admin/capability_fanout.go +++ b/internal/admin/capability_fanout.go @@ -86,12 +86,13 @@ type RouteSnapshot struct { // per the §8 failure-modes table, but the verdict separates them so // the operator-facing error message can name the precise reason. type CapabilityVerdict struct { - FullNodeID uint64 - EncryptionCapable bool - BuildSHA string - SidecarPresent bool - Reachable bool - Err error + FullNodeID uint64 + EncryptionCapable bool + StorageEnvelopeV2Capable bool + BuildSHA string + SidecarPresent bool + Reachable bool + Err error } // CapabilityFanoutResult is the aggregated outcome. OK is true iff @@ -107,6 +108,21 @@ type CapabilityFanoutResult struct { OK bool } +// StorageEnvelopeV2Ready reports whether every probed member can read V2 +// storage envelopes. Missing fields from an older binary decode as false, so +// the transition stays fail-closed throughout a rolling upgrade. +func (r CapabilityFanoutResult) StorageEnvelopeV2Ready() bool { + if len(r.Verdicts) == 0 { + return false + } + for _, verdict := range r.Verdicts { + if !verdict.Reachable || !verdict.EncryptionCapable || !verdict.StorageEnvelopeV2Capable { + return false + } + } + return true +} + // DialFunc opens a connection to one node's admin endpoint and // returns an EncryptionAdmin client plus a cleanup closure. The // helper invokes the closure exactly once per successful dial, @@ -343,6 +359,7 @@ func probeCapability(ctx context.Context, member RouteMember, dial DialFunc) Cap } verdict.Reachable = true verdict.EncryptionCapable = report.GetEncryptionCapable() + verdict.StorageEnvelopeV2Capable = report.GetStorageEnvelopeV2Capable() verdict.BuildSHA = report.GetBuildSha() verdict.SidecarPresent = report.GetSidecarPresent() if report.GetFullNodeId() != 0 { diff --git a/internal/admin/capability_fanout_test.go b/internal/admin/capability_fanout_test.go index a5204e9a9..503208d53 100644 --- a/internal/admin/capability_fanout_test.go +++ b/internal/admin/capability_fanout_test.go @@ -12,6 +12,24 @@ import ( "google.golang.org/grpc" ) +func TestCapabilityFanoutResultStorageEnvelopeV2Ready(t *testing.T) { + t.Parallel() + ready := CapabilityFanoutResult{Verdicts: []CapabilityVerdict{ + {Reachable: true, EncryptionCapable: true, StorageEnvelopeV2Capable: true}, + {Reachable: true, EncryptionCapable: true, StorageEnvelopeV2Capable: true}, + }} + if !ready.StorageEnvelopeV2Ready() { + t.Fatal("all-capable fan-out did not enable V2") + } + ready.Verdicts[1].StorageEnvelopeV2Capable = false + if ready.StorageEnvelopeV2Ready() { + t.Fatal("old-binary capability omission enabled V2") + } + if (CapabilityFanoutResult{}).StorageEnvelopeV2Ready() { + t.Fatal("empty fan-out enabled V2") + } +} + // stubEncryptionAdminClient is a minimal in-test client that // implements only the methods CapabilityFanout actually invokes // (GetCapability). Every other method panics so that an accidental @@ -80,9 +98,9 @@ func (s *stubDial) dial(_ context.Context, addr string) (pb.EncryptionAdminClien func TestCapabilityFanout_AllCapable(t *testing.T) { t.Parallel() stub := newStubDial() - stub.addOK("n1:9000", &pb.CapabilityReport{FullNodeId: 1, EncryptionCapable: true, SidecarPresent: true, BuildSha: "abc"}) - stub.addOK("n2:9000", &pb.CapabilityReport{FullNodeId: 2, EncryptionCapable: true, SidecarPresent: true, BuildSha: "abc"}) - stub.addOK("n3:9000", &pb.CapabilityReport{FullNodeId: 3, EncryptionCapable: true, SidecarPresent: true, BuildSha: "abc"}) + stub.addOK("n1:9000", &pb.CapabilityReport{FullNodeId: 1, EncryptionCapable: true, StorageEnvelopeV2Capable: true, SidecarPresent: true, BuildSha: "abc"}) + stub.addOK("n2:9000", &pb.CapabilityReport{FullNodeId: 2, EncryptionCapable: true, StorageEnvelopeV2Capable: true, SidecarPresent: true, BuildSha: "abc"}) + stub.addOK("n3:9000", &pb.CapabilityReport{FullNodeId: 3, EncryptionCapable: true, StorageEnvelopeV2Capable: true, SidecarPresent: true, BuildSha: "abc"}) snapshot := RouteSnapshot{Groups: []RouteGroup{{ GroupID: 1, @@ -104,10 +122,33 @@ func TestCapabilityFanout_AllCapable(t *testing.T) { t.Fatalf("expected 3 verdicts, got %d: %+v", len(res.Verdicts), res.Verdicts) } for _, v := range res.Verdicts { - if !v.Reachable || !v.EncryptionCapable { - t.Errorf("verdict %+v should be Reachable && EncryptionCapable", v) + if !v.Reachable || !v.EncryptionCapable || !v.StorageEnvelopeV2Capable { + t.Errorf("verdict %+v should carry all advertised capabilities", v) } } + if !res.StorageEnvelopeV2Ready() { + t.Fatal("all-capable probe results did not enable V2 readiness") + } +} + +func TestCapabilityFanout_OldBinaryKeepsV2NotReady(t *testing.T) { + t.Parallel() + stub := newStubDial() + stub.addOK("new:9000", &pb.CapabilityReport{FullNodeId: 1, EncryptionCapable: true, StorageEnvelopeV2Capable: true}) + stub.addOK("old:9000", &pb.CapabilityReport{FullNodeId: 2, EncryptionCapable: true}) + res, err := CapabilityFanout(context.Background(), RouteSnapshot{Groups: []RouteGroup{{ + GroupID: 1, + Voters: []RouteMember{{FullNodeID: 1, Address: "new:9000"}, {FullNodeID: 2, Address: "old:9000"}}, + }}}, stub.dial, time.Second) + if err != nil { + t.Fatalf("CapabilityFanout: %v", err) + } + if !res.OK { + t.Fatal("old V1 reader should remain encryption-capable") + } + if res.StorageEnvelopeV2Ready() { + t.Fatal("old binary without the new proto field enabled V2 writes") + } } // TestCapabilityFanout_OneNotCapable pins the §9 case where every diff --git a/internal/raftadmin/interceptor.go b/internal/raftadmin/interceptor.go index 5cdfe7ce5..92cb87750 100644 --- a/internal/raftadmin/interceptor.go +++ b/internal/raftadmin/interceptor.go @@ -22,7 +22,7 @@ import "context" type MembershipChangeInterceptor interface { // PreAddMember runs on the leader before AddVoter/AddLearner // proposes the conf-change. The raftID is the same string the - // caller passed in the AddVoter/AddLearner request (the - // `Id` field). - PreAddMember(ctx context.Context, raftID string) error + // caller passed in the AddVoter/AddLearner request. address is the + // target node's Raft/gRPC endpoint and may be used for capability checks. + PreAddMember(ctx context.Context, raftID, address string) error } diff --git a/internal/raftadmin/server.go b/internal/raftadmin/server.go index bf6b54c9b..9b64b8495 100644 --- a/internal/raftadmin/server.go +++ b/internal/raftadmin/server.go @@ -94,7 +94,7 @@ func (s *Server) AddVoter(ctx context.Context, req *pb.RaftAdminAddVoterRequest) // row exists at apply time and any §6.1 uint16 collision halts here // rather than after the conf-change is durable. if s.interceptor != nil { - if err := s.interceptor.PreAddMember(ctx, req.Id); err != nil { + if err := s.interceptor.PreAddMember(ctx, req.Id, req.Address); err != nil { return nil, adminError(err) } } @@ -113,7 +113,7 @@ func (s *Server) AddLearner(ctx context.Context, req *pb.RaftAdminAddLearnerRequ return nil, grpcStatus(codes.InvalidArgument, "id and address are required") } if s.interceptor != nil { - if err := s.interceptor.PreAddMember(ctx, req.Id); err != nil { + if err := s.interceptor.PreAddMember(ctx, req.Id, req.Address); err != nil { return nil, adminError(err) } } diff --git a/internal/raftadmin/server_test.go b/internal/raftadmin/server_test.go index 3aa100823..adf358611 100644 --- a/internal/raftadmin/server_test.go +++ b/internal/raftadmin/server_test.go @@ -474,10 +474,10 @@ type recordingInterceptor struct { preHook func(raftID string) // optional callback; useful for ordering assertions } -func (r *recordingInterceptor) PreAddMember(_ context.Context, raftID string) error { +func (r *recordingInterceptor) PreAddMember(_ context.Context, raftID, address string) error { r.mu.Lock() defer r.mu.Unlock() - r.calls = append(r.calls, raftID) + r.calls = append(r.calls, raftID+"@"+address) if r.preHook != nil { r.preHook(raftID) } @@ -499,7 +499,7 @@ func TestServer_AddVoter_InvokesInterceptorBeforeConfChange(t *testing.T) { resp, err := server.AddVoter(context.Background(), &pb.RaftAdminAddVoterRequest{Id: "n42", Address: "127.0.0.1:9999"}) require.NoError(t, err) require.NotNil(t, resp) - require.Equal(t, []string{"n42"}, interceptor.calls) + require.Equal(t, []string{"n42@127.0.0.1:9999"}, interceptor.calls) require.Equal(t, []string{"preAdd", "addVoter"}, order) require.Equal(t, 1, len(engine.addVoterCalls)) } @@ -516,7 +516,7 @@ func TestServer_AddVoter_InterceptorErrorAbortsConfChange(t *testing.T) { resp, err := server.AddVoter(context.Background(), &pb.RaftAdminAddVoterRequest{Id: "n42", Address: "127.0.0.1:9999"}) require.Error(t, err) require.Nil(t, resp) - require.Equal(t, []string{"n42"}, interceptor.calls) + require.Equal(t, []string{"n42@127.0.0.1:9999"}, interceptor.calls) require.Equal(t, 0, len(engine.addVoterCalls), "engine.AddVoter must NOT be called when PreAddMember errs") } @@ -546,7 +546,7 @@ func TestServer_AddLearner_InterceptorContract(t *testing.T) { server := NewServerWithInterceptor(engine, interceptor) _, err := server.AddLearner(context.Background(), &pb.RaftAdminAddLearnerRequest{Id: "l1", Address: "127.0.0.1:9000"}) require.NoError(t, err) - require.Equal(t, []string{"l1"}, interceptor.calls) + require.Equal(t, []string{"l1@127.0.0.1:9000"}, interceptor.calls) require.Equal(t, 1, len(engine.addLearnerCalls)) }) t.Run("interceptor error aborts", func(t *testing.T) { diff --git a/main.go b/main.go index f6c302fce..9274bd07b 100644 --- a/main.go +++ b/main.go @@ -2640,9 +2640,11 @@ func startRaftServers( const extraOptsCap = 2 enableMutators := encryptionMutatorsEnabled() encryptionCapabilityFanout := buildEncryptionCapabilityFanout(ctx, eg, runtimes, enableMutators) + startStorageEnvelopeV2CapabilityMonitor(ctx, eg, encryptionCapabilityFanout, encWiring) adminWriterRegistry := writerRegistryForEncryptionAdmin(runtimes, defaultGroup) recoveryLeaderView, recoveryAppliedIndex := recoveryStateForEncryptionAdmin(runtimes, defaultGroup) for _, rt := range runtimes { + mutatorAuthority := encryptionAdminMutatorAuthority(enableMutators, rt.spec.id, defaultGroup) baseOpts := internalutil.GRPCServerOptions() opts := make([]grpc.ServerOption, 0, len(baseOpts)+extraOptsCap) opts = append(opts, baseOpts...) @@ -2698,18 +2700,23 @@ func startRaftServers( // come from the default group that owns the shared sidecar and // writer registry. ShardGroup.Proposer() supplies the wrap-aware // post-cutover path for normal admin entries. + adminOpts := encryptionAdminOptionsForRuntime( + mutatorAuthority, + rt, + shardGroups, + adminWriterRegistry, + recoveryLeaderView, + recoveryAppliedIndex, + encWiring, + ) registerEncryptionAdminServer( gs, etcdraftengine.DeriveNodeID(*raftId), *encryptionSidecarPath, - enableMutators, + mutatorAuthority, rt.engine, encryptionCapabilityFanout, - adapter.WithEncryptionAdminWriterRegistry(adminWriterRegistry), - adapter.WithEncryptionAdminRecoveryLeaderView(recoveryLeaderView), - adapter.WithEncryptionAdminLatestAppliedIndex(recoveryAppliedIndex), - adapter.WithEncryptionAdminPostCutoverProposer(proposerForGroup(rt, shardGroups)), - adapter.WithEncryptionAdminCutoverBarrier(encWiring.raftEnvelope.barrier()), + adminOpts..., ) registerAdminForwardServer(gs, forwardDeps, forwardLogger) rt.registerGRPC(gs) diff --git a/main_encryption_admin.go b/main_encryption_admin.go index 7a92976d6..9243082c4 100644 --- a/main_encryption_admin.go +++ b/main_encryption_admin.go @@ -2,7 +2,9 @@ package main import ( "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" "google.golang.org/grpc" @@ -64,6 +66,33 @@ func encryptionMutatorsEnabled() bool { return *encryptionEnabled && *kekFile != "" && *encryptionSidecarPath != "" } +func encryptionAdminMutatorAuthority(enabled bool, groupID, defaultGroup uint64) bool { + return enabled && groupID == defaultGroup +} + +func encryptionAdminOptionsForRuntime( + mutatorAuthority bool, + rt *raftGroupRuntime, + shardGroups map[uint64]*kv.ShardGroup, + writerRegistry encryption.WriterRegistryStore, + recoveryLeaderView raftengine.LeaderView, + recoveryAppliedIndex func() uint64, + encWiring encryptionWriteWiring, +) []adapter.EncryptionAdminServerOption { + opts := []adapter.EncryptionAdminServerOption{ + adapter.WithEncryptionAdminWriterRegistry(writerRegistry), + adapter.WithEncryptionAdminRecoveryLeaderView(recoveryLeaderView), + adapter.WithEncryptionAdminLatestAppliedIndex(recoveryAppliedIndex), + } + if !mutatorAuthority { + return opts + } + return append(opts, + adapter.WithEncryptionAdminPostCutoverProposer(proposerForGroup(rt, shardGroups)), + adapter.WithEncryptionAdminCutoverBarrier(encWiring.raftEnvelope.barrier()), + ) +} + // encryptionAdminEngine is the subset of raftengine.Engine the // EncryptionAdminServer needs: a Proposer (for the mutating RPCs) // and a LeaderView (for the requireLeader gate). Every shard's @@ -150,8 +179,10 @@ func registerEncryptionAdminServer(gs *grpc.Server, fullNodeID uint64, sidecarPa if capabilityFanout != nil { opts = append(opts, adapter.WithEncryptionAdminCapabilityFanout(capabilityFanout)) } - opts = append(opts, extraOpts...) } + // Read-only recovery options are valid on every shard listener. The caller + // only includes mutator-specific options for the default-group authority. + opts = append(opts, extraOpts...) srv := adapter.NewEncryptionAdminServer(opts...) if err := srv.Validate(); err != nil { panic(errors.Wrap(err, "encryption admin server validation")) diff --git a/main_encryption_admin_test.go b/main_encryption_admin_test.go index 77186e43f..5d7d5cfa1 100644 --- a/main_encryption_admin_test.go +++ b/main_encryption_admin_test.go @@ -112,6 +112,18 @@ func TestWriterRegistryForEncryptionAdminMissingDefaultGroup(t *testing.T) { } } +func TestEncryptionAdminMutatorAuthorityOnlyDefaultGroup(t *testing.T) { + if encryptionAdminMutatorAuthority(true, 2, 1) { + t.Fatal("non-default group received encryption mutator authority") + } + if !encryptionAdminMutatorAuthority(true, 1, 1) { + t.Fatal("default group did not receive encryption mutator authority") + } + if encryptionAdminMutatorAuthority(false, 1, 1) { + t.Fatal("disabled encryption mutators received authority") + } +} + func TestRecoveryStateForEncryptionAdminUsesDefaultGroup(t *testing.T) { defaultEngine := &stubEncryptionRecoveryEngine{applied: 41} otherEngine := &stubEncryptionRecoveryEngine{applied: 99} diff --git a/main_encryption_confchange.go b/main_encryption_confchange.go index 87146f44a..dbe766b83 100644 --- a/main_encryption_confchange.go +++ b/main_encryption_confchange.go @@ -2,15 +2,22 @@ package main import ( "context" + stderrors "errors" "log/slog" "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/raftadmin" "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" ) +var ( + errMemberNotEncryptionCapable = stderrors.New("member is not encryption capable") + errMemberNotStorageV2Capable = stderrors.New("member does not support V2 storage envelopes") +) + // encryptionPreRegister is the Stage 7c §3.1 encryption-aware // implementation of raftadmin.MembershipChangeInterceptor. It runs // on the leader's AddVoter/AddLearner handler before the underlying @@ -35,9 +42,12 @@ type encryptionPreRegister struct { // main.go supplies etcdraftengine.DeriveNodeID today; a future // engine swap is a one-line wiring change here, not an adapter // refactor (gemini medium #1 on PR #868 / design §3.1). - deriveNodeID func(raftID string) uint64 + deriveNodeID func(raftID string) uint64 + capabilityProbe storageEnvelopeV2CapabilityProbe } +type storageEnvelopeV2CapabilityProbe func(context.Context, string) error + // newEncryptionPreRegister constructs the interceptor or returns nil // when encryption is not wired (no shared cache or no default // group). A nil interceptor causes raftadmin.Server to skip the @@ -49,16 +59,22 @@ func newEncryptionPreRegister( cache *encryption.StateCache, sidecarPath string, deriveNodeID func(raftID string) uint64, + capabilityProbes ...storageEnvelopeV2CapabilityProbe, ) raftadmin.MembershipChangeInterceptor { if coordinate == nil || defaultGroup == nil || defaultGroup.Store == nil || cache == nil || deriveNodeID == nil { return nil } + capabilityProbe := probeStorageEnvelopeV2Capability + if len(capabilityProbes) > 0 && capabilityProbes[0] != nil { + capabilityProbe = capabilityProbes[0] + } return &encryptionPreRegister{ - coordinate: coordinate, - defaultGroup: defaultGroup, - cache: cache, - sidecarPath: sidecarPath, - deriveNodeID: deriveNodeID, + coordinate: coordinate, + defaultGroup: defaultGroup, + cache: cache, + sidecarPath: sidecarPath, + deriveNodeID: deriveNodeID, + capabilityProbe: capabilityProbe, } } @@ -68,7 +84,7 @@ func newEncryptionPreRegister( // encryption.ErrWriterUint16Collision on a §6.1 collision; otherwise // proposes RegisterEncryptionWriter(activeDEK, NodeID(raftID), 0) // and surfaces the propose result. -func (e *encryptionPreRegister) PreAddMember(ctx context.Context, raftID string) error { +func (e *encryptionPreRegister) PreAddMember(ctx context.Context, raftID, address string) error { activeDEK, ok := e.cache.ActiveStorageKeyID() if !ok { // Pre-bootstrap cluster: there is no registry to gate on. @@ -76,6 +92,12 @@ func (e *encryptionPreRegister) PreAddMember(ctx context.Context, raftID string) // encryption-disabled with an empty cache.) return nil } + if e.capabilityProbe == nil { + return errors.New("encryption: membership V2 capability probe is not configured") + } + if err := e.capabilityProbe(ctx, address); err != nil { + return errors.Wrap(err, "encryption: refuse member before V2 capability confirmation") + } newNodeFullID := e.deriveNodeID(raftID) if err := e.preRegisterDEK(ctx, activeDEK, newNodeFullID, "storage"); err != nil { return err @@ -90,6 +112,33 @@ func (e *encryptionPreRegister) PreAddMember(ctx context.Context, raftID string) return e.preRegisterDEK(ctx, raftDEK, newNodeFullID, "raft") } +func probeStorageEnvelopeV2Capability(ctx context.Context, address string) error { + if address == "" { + return errors.New("encryption: member address is empty") + } + connCache := &kv.GRPCConnCache{} + defer func() { + if err := connCache.Close(); err != nil { + slog.Warn("encryption: failed to close membership capability connection", slog.String("error", err.Error())) + } + }() + conn, err := connCache.ConnFor(address) + if err != nil { + return errors.Wrapf(err, "dial member %s", address) + } + report, err := pb.NewEncryptionAdminClient(conn).GetCapability(ctx, &pb.Empty{}) + if err != nil { + return errors.Wrapf(err, "GetCapability member %s", address) + } + if !report.GetEncryptionCapable() { + return errors.Wrapf(errMemberNotEncryptionCapable, "member %s", address) + } + if !report.GetStorageEnvelopeV2Capable() { + return errors.Wrapf(errMemberNotStorageV2Capable, "member %s", address) + } + return nil +} + func (e *encryptionPreRegister) activeRaftDEKForPreRegister() (uint32, bool, error) { if e.sidecarPath == "" { return 0, false, nil diff --git a/main_encryption_confchange_test.go b/main_encryption_confchange_test.go index 4a15586e6..a31c755a0 100644 --- a/main_encryption_confchange_test.go +++ b/main_encryption_confchange_test.go @@ -20,6 +20,8 @@ const stubDerivedNodeID uint64 = 0xCAFEF00DDEADBEEF func stubDeriveNodeID(string) uint64 { return stubDerivedNodeID } +func allowStorageEnvelopeV2Capability(context.Context, string) error { return nil } + // TestEncryptionPreRegister_PreBootstrapSkips pins design §5.2: when // the StateCache reports (0, false) — no active storage DEK, either // pre-bootstrap or encryption-disabled — PreAddMember returns nil @@ -33,7 +35,7 @@ func TestEncryptionPreRegister_PreBootstrapSkips(t *testing.T) { if pre == nil { t.Fatal("newEncryptionPreRegister returned nil despite non-nil cache+group") } - if err := pre.PreAddMember(context.Background(), "n1"); err != nil { + if err := pre.PreAddMember(context.Background(), "n1", "n1:50051"); err != nil { t.Errorf("PreAddMember should skip pre-bootstrap: got %v", err) } } @@ -85,12 +87,48 @@ func TestEncryptionPreRegister_IdempotentWhenRowExists(t *testing.T) { sc.Active.Storage = testRegDEKID cache.RefreshFromSidecar(sc) - pre := newEncryptionPreRegister(&kv.ShardedCoordinator{}, &kv.ShardGroup{Store: st}, cache, "", stubDeriveNodeID) - if err := pre.PreAddMember(context.Background(), "raftN"); err != nil { + pre := newEncryptionPreRegister(&kv.ShardedCoordinator{}, &kv.ShardGroup{Store: st}, cache, "", stubDeriveNodeID, allowStorageEnvelopeV2Capability) + if err := pre.PreAddMember(context.Background(), "raftN", "raftN:50051"); err != nil { t.Errorf("PreAddMember should skip when matching row exists: got %v", err) } } +func TestEncryptionPreRegister_RejectsMemberWithoutV2Capability(t *testing.T) { + t.Parallel() + st := newRegistrationTestStore(t) + cache := encryption.NewStateCache() + sc := &encryption.Sidecar{Version: encryption.SidecarVersion, StorageEnvelopeActive: true} + sc.Active.Storage = testRegDEKID + cache.RefreshFromSidecar(sc) + sentinel := errors.New("old binary") + var probedAddress string + pre := newEncryptionPreRegister( + &kv.ShardedCoordinator{}, + &kv.ShardGroup{Store: st}, + cache, + "", + stubDeriveNodeID, + func(_ context.Context, address string) error { + probedAddress = address + return sentinel + }, + ) + err := pre.PreAddMember(context.Background(), "old", "old:50051") + if !errors.Is(err, sentinel) { + t.Fatalf("PreAddMember error = %v, want capability error", err) + } + if probedAddress != "old:50051" { + t.Fatalf("capability probe address = %q, want old:50051", probedAddress) + } + reg, regErr := store.WriterRegistryFor(st) + if regErr != nil { + t.Fatalf("WriterRegistryFor: %v", regErr) + } + if _, ok, regErr := reg.GetRegistryRow(encryption.RegistryKey(testRegDEKID, encryption.NodeID16(stubDerivedNodeID))); regErr != nil || ok { + t.Fatalf("registry changed before capability acceptance: present=%t err=%v", ok, regErr) + } +} + // TestEncryptionPreRegister_Uint16CollisionReturnsTypedError pins // design §5.2 + §3.1's §6.1-collision branch: when a row exists at // the same uint16 truncation with a DIFFERENT FullNodeID, the guard @@ -127,8 +165,8 @@ func TestEncryptionPreRegister_Uint16CollisionReturnsTypedError(t *testing.T) { } collidingDerive := func(string) uint64 { return collidingFullNodeID } - pre := newEncryptionPreRegister(&kv.ShardedCoordinator{}, &kv.ShardGroup{Store: st}, cache, "", collidingDerive) - err = pre.PreAddMember(context.Background(), "raftN") + pre := newEncryptionPreRegister(&kv.ShardedCoordinator{}, &kv.ShardGroup{Store: st}, cache, "", collidingDerive, allowStorageEnvelopeV2Capability) + err = pre.PreAddMember(context.Background(), "raftN", "raftN:50051") if !errors.Is(err, encryption.ErrWriterUint16Collision) { t.Errorf("PreAddMember on §6.1 collision: want ErrWriterUint16Collision, got %v", err) } diff --git a/main_encryption_fanout.go b/main_encryption_fanout.go index a225317f2..d715a7ab8 100644 --- a/main_encryption_fanout.go +++ b/main_encryption_fanout.go @@ -2,6 +2,7 @@ package main import ( "context" + "log/slog" "time" "github.com/bootjp/elastickv/adapter" @@ -23,6 +24,8 @@ import ( // GetCapability round-trip. const capabilityFanoutTimeout = 5 * time.Second +const storageEnvelopeV2CapabilityRetryInterval = 5 * time.Second + // buildEncryptionCapabilityFanout builds the §4 capability fan-out // closure shared across every shard's EncryptionAdmin server, or nil // when encryption mutators are disabled (the cutover RPC is then @@ -46,6 +49,39 @@ func buildEncryptionCapabilityFanout(ctx context.Context, eg *errgroup.Group, ru return buildCapabilityFanoutFn(runtimes, fanoutConnCache, capabilityFanoutTimeout) } +// startStorageEnvelopeV2CapabilityMonitor keeps encrypted writes on V1 until +// a fresh live-membership fan-out confirms V2 reader support on every voter +// and learner. Activation is sticky because existing V2 rows remain part of +// snapshots after the first V2 write. +func startStorageEnvelopeV2CapabilityMonitor( + ctx context.Context, + eg *errgroup.Group, + capabilityFanout adapter.CapabilityFanoutFn, + wiring encryptionWriteWiring, +) { + if eg == nil || capabilityFanout == nil || wiring.storageEnvelopeV2Active == nil { + return + } + eg.Go(func() error { + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-timer.C: + } + result, err := capabilityFanout(ctx) + if err == nil && result.StorageEnvelopeV2Ready() { + wiring.activateStorageEnvelopeV2Writes() + slog.Info("encryption: enabled V2 storage envelope writes after cluster capability confirmation") + return nil + } + timer.Reset(storageEnvelopeV2CapabilityRetryInterval) + } + }) +} + // buildCapabilityFanoutFn assembles the adapter.CapabilityFanoutFn the // EnableStorageEnvelope server invokes for its §4 pre-flight check. The // returned closure snapshots the live membership of every Raft group diff --git a/main_encryption_fanout_test.go b/main_encryption_fanout_test.go index 2051edc9e..e822eeacf 100644 --- a/main_encryption_fanout_test.go +++ b/main_encryption_fanout_test.go @@ -3,12 +3,44 @@ package main import ( "context" "errors" + "sync/atomic" "testing" + "time" + "github.com/bootjp/elastickv/internal/admin" "github.com/bootjp/elastickv/internal/raftengine" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" + "golang.org/x/sync/errgroup" ) +func TestStorageEnvelopeV2CapabilityMonitorActivatesOnce(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + eg, groupCtx := errgroup.WithContext(ctx) + wiring := encryptionWriteWiring{storageEnvelopeV2Active: &atomic.Bool{}} + var calls atomic.Int32 + startStorageEnvelopeV2CapabilityMonitor(groupCtx, eg, func(context.Context) (admin.CapabilityFanoutResult, error) { + calls.Add(1) + return admin.CapabilityFanoutResult{Verdicts: []admin.CapabilityVerdict{{ + Reachable: true, EncryptionCapable: true, StorageEnvelopeV2Capable: true, + }}}, nil + }, wiring) + deadline := time.Now().Add(time.Second) + for !wiring.storageEnvelopeV2WritesActive() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !wiring.storageEnvelopeV2WritesActive() { + t.Fatal("capability monitor did not activate V2 writes") + } + cancel() + if err := eg.Wait(); err != nil { + t.Fatalf("capability monitor: %v", err) + } + if got := calls.Load(); got != 1 { + t.Fatalf("capability monitor calls = %d, want one sticky activation", got) + } +} + // stubConfigReader is a configReader for the snapshot-builder tests. type stubConfigReader struct { cfg raftengine.Configuration diff --git a/main_encryption_write_wiring.go b/main_encryption_write_wiring.go index b3f124e71..b282e98ad 100644 --- a/main_encryption_write_wiring.go +++ b/main_encryption_write_wiring.go @@ -138,6 +138,9 @@ type encryptionWriteWiring struct { cache *encryption.StateCache cipher *encryption.Cipher nonceFactory store.NonceFactory + // storageEnvelopeV2Active stays false until the live membership fan-out + // confirms that every voter and learner can read compressed V2 envelopes. + storageEnvelopeV2Active *atomic.Bool // epoch is the §4.1 local_epoch this process load pinned into the // nonce factory (the value BumpLocalEpoch advanced to, or 0 in the // pre-bootstrap case). Stage 7a's process-start registration @@ -192,6 +195,7 @@ func (w encryptionWriteWiring) pebbleOptions() []store.PebbleStoreOption { return []store.PebbleStoreOption{ store.WithEncryption(w.cipher, w.nonceFactory, w.cache.ActiveStorageKeyID), store.WithStorageEnvelopeGate(w.cache.StorageEnvelopeActive), + store.WithStorageEnvelopeV2Gate(w.storageEnvelopeV2WritesActive), // Stage 7a-2 §4.1 direct-path registration gate: the store // refuses to emit an encrypted envelope on a self-originated // write (catalog bootstrap Save) until this load's writer @@ -204,6 +208,16 @@ func (w encryptionWriteWiring) pebbleOptions() []store.PebbleStoreOption { } } +func (w encryptionWriteWiring) storageEnvelopeV2WritesActive() bool { + return w.storageEnvelopeV2Active != nil && w.storageEnvelopeV2Active.Load() +} + +func (w encryptionWriteWiring) activateStorageEnvelopeV2Writes() { + if w.storageEnvelopeV2Active != nil { + w.storageEnvelopeV2Active.Store(true) + } +} + // buildEncryptionWriteWiring assembles the storage-envelope write-path // wiring. It always returns a wiring with a non-nil StateCache (the // per-shard appliers need it regardless of encryption state); the @@ -259,6 +273,7 @@ func buildEncryptionWriteWiring(encryptionEnabled bool, raftID, sidecarPath stri fullNodeID := etcdraftengine.DeriveNodeID(raftID) nodeID := encryption.NodeID16(fullNodeID) w.cipher = cipher + w.storageEnvelopeV2Active = &atomic.Bool{} w.epoch = epoch w.raftEpoch = raftEpoch w.nonceFactory = encryption.NewDeterministicNonceFactory(nodeID, epoch) diff --git a/main_encryption_write_wiring_test.go b/main_encryption_write_wiring_test.go index acd3b1327..338b3856d 100644 --- a/main_encryption_write_wiring_test.go +++ b/main_encryption_write_wiring_test.go @@ -73,8 +73,8 @@ func TestEncryptionWriteWiring_PebbleOptions_WiredWhenCipherSet(t *testing.T) { cipher: cipher, nonceFactory: encryption.NewDeterministicNonceFactory(0xABCD, 0), } - if opts := w.pebbleOptions(); len(opts) != 3 { - t.Errorf("wired pebbleOptions = %d opts, want 3 (WithEncryption + WithStorageEnvelopeGate + WithStorageRegistrationGate)", len(opts)) + if opts := w.pebbleOptions(); len(opts) != 4 { + t.Errorf("wired pebbleOptions = %d opts, want 4 (encryption + storage cutover + V2 capability + registration gates)", len(opts)) } } @@ -240,17 +240,28 @@ func TestBuildEncryptionWriteWiring_ActiveDEK_WiresCipher(t *testing.T) { if w.raftEpoch != 1 { t.Errorf("raft epoch = %d, want 1 (bumped from 0)", w.raftEpoch) } - // WithEncryption + WithStorageEnvelopeGate + WithStorageRegistrationGate - // (Stage 7a-2 added the third). - if opts := w.pebbleOptions(); len(opts) != 3 { - t.Errorf("pebbleOptions = %d, want 3", len(opts)) + // WithEncryption plus storage cutover, V2 capability, and registration gates. + if opts := w.pebbleOptions(); len(opts) != 4 { + t.Errorf("pebbleOptions = %d, want 4", len(opts)) } + assertStorageEnvelopeV2Activation(t, w) // Cache must reflect the on-disk active DEK + cutover gate. if id, ok := w.cache.ActiveStorageKeyID(); !ok || id != 3 { t.Errorf("cache ActiveStorageKeyID = (%d, %v), want (3, true)", id, ok) } } +func assertStorageEnvelopeV2Activation(t *testing.T, w encryptionWriteWiring) { + t.Helper() + if w.storageEnvelopeV2WritesActive() { + t.Error("V2 writes started active before cluster capability confirmation") + } + w.activateStorageEnvelopeV2Writes() + if !w.storageEnvelopeV2WritesActive() { + t.Error("V2 writes remained inactive after capability confirmation") + } +} + func TestBuildEncryptionWriteWiring_ActiveCutoverWithoutRaftDEKRefusesBeforeStorageBump(t *testing.T) { t.Parallel() path := writeActiveStorageSidecar(t, 2) diff --git a/proto/encryption_admin.pb.go b/proto/encryption_admin.pb.go index 8511ea1ac..27f600fe6 100644 --- a/proto/encryption_admin.pb.go +++ b/proto/encryption_admin.pb.go @@ -119,8 +119,11 @@ type CapabilityReport struct { SidecarPresent bool `protobuf:"varint,3,opt,name=sidecar_present,json=sidecarPresent,proto3" json:"sidecar_present,omitempty"` FullNodeId uint64 `protobuf:"varint,4,opt,name=full_node_id,json=fullNodeId,proto3" json:"full_node_id,omitempty"` LocalEpoch uint32 `protobuf:"varint,5,opt,name=local_epoch,json=localEpoch,proto3" json:"local_epoch,omitempty"` // MUST be <= 0xFFFF on the wire. - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Old binaries omit this field and therefore decode as false. Writers must + // keep emitting V1 envelopes until every voter and learner reports true. + StorageEnvelopeV2Capable bool `protobuf:"varint,6,opt,name=storage_envelope_v2_capable,json=storageEnvelopeV2Capable,proto3" json:"storage_envelope_v2_capable,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CapabilityReport) Reset() { @@ -188,6 +191,13 @@ func (x *CapabilityReport) GetLocalEpoch() uint32 { return 0 } +func (x *CapabilityReport) GetStorageEnvelopeV2Capable() bool { + if x != nil { + return x.StorageEnvelopeV2Capable + } + return false +} + // SidecarStateReport is the §5.5 compaction-fallback RPC. Served on // every node but the cutover-recovery flow only consults the // leader's response. wrapped_deks_by_id covers every unretired DEK. @@ -1209,7 +1219,7 @@ var File_encryption_admin_proto protoreflect.FileDescriptor const file_encryption_admin_proto_rawDesc = "" + "\n" + "\x16encryption_admin.proto\"\a\n" + - "\x05Empty\"\xca\x01\n" + + "\x05Empty\"\x89\x02\n" + "\x10CapabilityReport\x12-\n" + "\x12encryption_capable\x18\x01 \x01(\bR\x11encryptionCapable\x12\x1b\n" + "\tbuild_sha\x18\x02 \x01(\tR\bbuildSha\x12'\n" + @@ -1217,7 +1227,8 @@ const file_encryption_admin_proto_rawDesc = "" + "\ffull_node_id\x18\x04 \x01(\x04R\n" + "fullNodeId\x12\x1f\n" + "\vlocal_epoch\x18\x05 \x01(\rR\n" + - "localEpoch\"\xe5\x04\n" + + "localEpoch\x12=\n" + + "\x1bstorage_envelope_v2_capable\x18\x06 \x01(\bR\x18storageEnvelopeV2Capable\"\xe5\x04\n" + "\x12SidecarStateReport\x12U\n" + "\x12wrapped_deks_by_id\x18\x01 \x03(\v2(.SidecarStateReport.WrappedDeksByIdEntryR\x0fwrappedDeksById\x12*\n" + "\x11active_storage_id\x18\x02 \x01(\rR\x0factiveStorageId\x12$\n" + diff --git a/proto/encryption_admin.proto b/proto/encryption_admin.proto index 6926a1c01..1ff31485f 100644 --- a/proto/encryption_admin.proto +++ b/proto/encryption_admin.proto @@ -67,6 +67,9 @@ message CapabilityReport { bool sidecar_present = 3; uint64 full_node_id = 4; uint32 local_epoch = 5; // MUST be <= 0xFFFF on the wire. + // Old binaries omit this field and therefore decode as false. Writers must + // keep emitting V1 envelopes until every voter and learner reports true. + bool storage_envelope_v2_capable = 6; } // SidecarStateReport is the §5.5 compaction-fallback RPC. Served on diff --git a/store/encryption_compression_test.go b/store/encryption_compression_test.go index 941dc8a2a..6e280322a 100644 --- a/store/encryption_compression_test.go +++ b/store/encryption_compression_test.go @@ -68,6 +68,47 @@ func TestEncryption_CompressesOnlyWhenSmaller(t *testing.T) { } } +func TestEncryption_StorageEnvelopeV2CapabilityGate(t *testing.T) { + t.Parallel() + ks := encryption.NewKeystore() + dek := make([]byte, encryption.KeySize) + if _, err := rand.Read(dek); err != nil { + t.Fatalf("rand.Read DEK: %v", err) + } + const keyID uint32 = 107 + if err := ks.Set(keyID, dek); err != nil { + t.Fatalf("Keystore.Set: %v", err) + } + cipher, err := encryption.NewCipher(ks) + if err != nil { + t.Fatalf("NewCipher: %v", err) + } + v2Active := false + s := &pebbleStore{ + cipher: cipher, + nonceFactory: NewCounterNonceFactory(0x0102, 0x0304), + activeStorageKeyID: func() (uint32, bool) { return keyID, true }, + storageEnvelopeV2Active: func() bool { return v2Active }, + } + plaintext := bytes.Repeat([]byte("compressible"), 512) + pebbleKey := encodeKey([]byte("capability-gate"), 100) + + body, _, err := s.encryptForKey(pebbleKey, plaintext, 0, false) + require.NoError(t, err) + env, err := encryption.DecodeEnvelope(body) + require.NoError(t, err) + require.Equal(t, encryption.EnvelopeVersionV1, env.Version) + require.Zero(t, env.Flag) + + v2Active = true + body, _, err = s.encryptForKey(pebbleKey, plaintext, 0, false) + require.NoError(t, err) + env, err = encryption.DecodeEnvelope(body) + require.NoError(t, err) + require.Equal(t, encryption.EnvelopeVersionV2, env.Version) + require.Equal(t, encryption.FlagCompressed, env.Flag) +} + func TestEncryption_CompressionFlagTamperRejected(t *testing.T) { t.Parallel() f := newEncryptedStoreFixture(t, 102) diff --git a/store/encryption_glue.go b/store/encryption_glue.go index 506a7a79e..8105568d6 100644 --- a/store/encryption_glue.go +++ b/store/encryption_glue.go @@ -160,6 +160,11 @@ type ActiveStorageKeyID func() (uint32, bool) // the design supports) would still decrypt old envelopes correctly. type StorageEnvelopeActive func() bool +// StorageEnvelopeV2Active reports whether every current voter and learner can +// read V2 storage envelopes. When wired and false, encrypted writes remain V1 +// and uncompressed so a rolling-upgrade peer can still read snapshots. +type StorageEnvelopeV2Active func() bool + // StorageRegistered reports whether this process load's §4.1 writer // registration has committed for the currently-active storage DEK // (Stage 7a-2). It gates only the DIRECT write path: when it returns @@ -235,6 +240,19 @@ func WithStorageEnvelopeGate(active StorageEnvelopeActive) PebbleStoreOption { } } +// WithStorageEnvelopeV2Gate prevents V2 writes until the cluster-wide +// capability fan-out has observed support from every voter and learner. +// Reads always accept both V1 and V2. A nil gate preserves the standalone +// and test-fixture behavior from before capability negotiation was wired. +func WithStorageEnvelopeV2Gate(active StorageEnvelopeV2Active) PebbleStoreOption { + return func(s *pebbleStore) { + if active == nil { + return + } + s.storageEnvelopeV2Active = active + } +} + // WithStorageRegistrationGate wires the Stage 7a-2 §4.1 registration // gate in front of the envelope-emit path on the DIRECT write path // only. After this option, when the envelope would encrypt @@ -355,7 +373,10 @@ func (s *pebbleStore) encryptForKey(pebbleKey, plaintext []byte, expireAt uint64 return nil, 0, errors.Wrap(err, "store: nonce factory") } nonce := nonceArr[:] - payload, envelopeFlag := compressForEncryption(plaintext) + payload, envelopeFlag := plaintext, byte(0) + if s.storageEnvelopeV2Active == nil || s.storageEnvelopeV2Active() { + payload, envelopeFlag = compressForEncryption(plaintext) + } envelopeVersion := encryption.EnvelopeVersionV1 if envelopeFlag&encryption.FlagCompressed != 0 { envelopeVersion = encryption.EnvelopeVersionV2 diff --git a/store/lsm_store.go b/store/lsm_store.go index 8f2824d29..21cedacdb 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -289,6 +289,11 @@ type pebbleStore struct { // fixtures depend on that posture and the 6D-6 production wiring // in main.go is what flips the gate on. storageEnvelopeActive StorageEnvelopeActive + // storageEnvelopeV2Active is the rolling-upgrade reader-capability + // gate for compressed V2 envelopes. Production keeps it false until + // every voter and learner advertises V2 support. A nil closure keeps + // the pre-gate embedded/test behavior. + storageEnvelopeV2Active StorageEnvelopeV2Active // storageRegistered is the Stage 7a-2 §4.1 registration gate. When // wired, the DIRECT write path (PutAt / ExpireAt / ApplyMutations) // refuses to emit an encrypted envelope — returning diff --git a/store/snapshot_pebble_sst.go b/store/snapshot_pebble_sst.go index 500eb0107..2f755eeef 100644 --- a/store/snapshot_pebble_sst.go +++ b/store/snapshot_pebble_sst.go @@ -94,6 +94,7 @@ type pebbleSSTExporter struct { largest []byte fileIndex int targetFileBytes uint64 + compression *sstable.CompressionProfile } func resolveSSTIngestSnapshots(raw string) bool { @@ -351,7 +352,7 @@ func buildPebbleSSTIngestBundle(checkpointDir string, metadata pebbleSnapshotMet return nil, err } - manifest, err := exportPebbleSnapshotSSTs(db, exportDir, metadata, targetFileBytes) + manifest, err := exportPebbleSnapshotSSTs(db, exportDir, metadata, targetFileBytes, disableCompression) if err != nil { return cleanup(err) } @@ -376,7 +377,7 @@ func buildPebbleSSTIngestBundle(checkpointDir string, metadata pebbleSnapshotMet return bundle, nil } -func exportPebbleSnapshotSSTs(db *pebble.DB, exportDir string, metadata pebbleSnapshotMetadata, targetFileBytes uint64) (pebbleSSTIngestManifest, error) { +func exportPebbleSnapshotSSTs(db *pebble.DB, exportDir string, metadata pebbleSnapshotMetadata, targetFileBytes uint64, disableCompression bool) (pebbleSSTIngestManifest, error) { if targetFileBytes == 0 { targetFileBytes = defaultSSTIngestTargetFileSize } @@ -384,6 +385,7 @@ func exportPebbleSnapshotSSTs(db *pebble.DB, exportDir string, metadata pebbleSn db: db, dir: exportDir, targetFileBytes: targetFileBytes, + compression: sstExportCompression(disableCompression), manifest: pebbleSSTIngestManifest{ Version: sstIngestSnapshotVersion, LastCommitTS: metadata.LastCommitTS, @@ -419,6 +421,13 @@ func exportPebbleSnapshotSSTs(db *pebble.DB, exportDir string, metadata pebbleSn return exporter.manifest, nil } +func sstExportCompression(disable bool) *sstable.CompressionProfile { + if disable { + return sstable.NoCompression + } + return nil +} + func (e *pebbleSSTExporter) Add(key, value []byte) error { if e.writer == nil { if err := e.startFile(key); err != nil { @@ -461,6 +470,7 @@ func (e *pebbleSSTExporter) startFile(firstKey []byte) error { e.writer = sstable.NewWriter(objstorageprovider.NewFileWritable(file), sstable.WriterOptions{ Comparer: pebble.DefaultComparer, TableFormat: e.db.TableFormat(), + Compression: e.compression, }) e.currentBytes = 0 e.smallest = bytes.Clone(firstKey) diff --git a/store/snapshot_pebble_sst_test.go b/store/snapshot_pebble_sst_test.go index 0a2139ea9..794033b82 100644 --- a/store/snapshot_pebble_sst_test.go +++ b/store/snapshot_pebble_sst_test.go @@ -13,6 +13,7 @@ import ( internalutil "github.com/bootjp/elastickv/internal" "github.com/bootjp/elastickv/internal/encryption" + "github.com/cockroachdb/pebble/v2/sstable" "github.com/stretchr/testify/require" ) @@ -41,6 +42,12 @@ func openSSTSnapshotTestStore(t *testing.T, dir string, opts ...PebbleStoreOptio return store } +func TestSSTExportCompressionPolicy(t *testing.T) { + t.Parallel() + require.Nil(t, sstExportCompression(false)) + require.Same(t, sstable.NoCompression, sstExportCompression(true)) +} + func withSSTIngestTargetFileBytesForTest(target uint64) PebbleStoreOption { return func(store *pebbleStore) { store.sstIngestTargetFileBytes = target From a39527d37dfb0320f5b25da5f74415fc56032a3f Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 20 Jul 2026 00:49:25 +0900 Subject: [PATCH 10/16] encryption: harden V2 membership capability gate --- ...8_implemented_9a_encryption_compression.md | 12 +-- main_encryption_confchange.go | 90 ++++++++++++++----- main_encryption_confchange_test.go | 78 ++++++++++++++-- main_encryption_fanout.go | 24 +++-- main_encryption_fanout_test.go | 35 +++++++- 5 files changed, 199 insertions(+), 40 deletions(-) diff --git a/docs/design/2026_07_18_implemented_9a_encryption_compression.md b/docs/design/2026_07_18_implemented_9a_encryption_compression.md index 6e88b7bb9..9b65f3861 100644 --- a/docs/design/2026_07_18_implemented_9a_encryption_compression.md +++ b/docs/design/2026_07_18_implemented_9a_encryption_compression.md @@ -14,11 +14,13 @@ This milestone implements the compress-then-encrypt storage path from - `FlagCompressed` is authenticated as part of the envelope AAD. - Reads decompress only after GCM authentication succeeds. - V1 envelopes require `flag=0`; compressed values use envelope V2. -- V2 writes remain disabled until every live voter and learner advertises the - V2 reader capability; absent fields from older binaries fail closed. -- Once encryption is bootstrapped, AddVoter/AddLearner probes the target - endpoint and refuses members that do not advertise encryption and V2 reader - capability before proposing the membership change. +- V2 writes remain disabled until encryption is bootstrapped and every live + voter and learner advertises the V2 reader capability; absent fields from + older binaries fail closed. +- AddVoter/AddLearner probes the target endpoint before and after bootstrap, + bounds the probe to five seconds, and requires the reported full node ID to + match the requested Raft ID. A timeout, mismatch, or missing encryption/V2 + reader capability is refused before proposing the membership change. - Unknown version/flag combinations fail closed. - The cleartext-rebadge guard verifies both valid compression-flag variants. - Pebble block compression is disabled for encryption-wired stores, including diff --git a/main_encryption_confchange.go b/main_encryption_confchange.go index dbe766b83..6aa56d789 100644 --- a/main_encryption_confchange.go +++ b/main_encryption_confchange.go @@ -4,6 +4,7 @@ import ( "context" stderrors "errors" "log/slog" + "time" "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/raftadmin" @@ -16,8 +17,11 @@ import ( var ( errMemberNotEncryptionCapable = stderrors.New("member is not encryption capable") errMemberNotStorageV2Capable = stderrors.New("member does not support V2 storage envelopes") + errMemberCapabilityIDMismatch = stderrors.New("member capability node id does not match requested raft id") ) +const membershipCapabilityProbeTimeout = 5 * time.Second + // encryptionPreRegister is the Stage 7c §3.1 encryption-aware // implementation of raftadmin.MembershipChangeInterceptor. It runs // on the leader's AddVoter/AddLearner handler before the underlying @@ -46,7 +50,9 @@ type encryptionPreRegister struct { capabilityProbe storageEnvelopeV2CapabilityProbe } -type storageEnvelopeV2CapabilityProbe func(context.Context, string) error +type storageEnvelopeV2CapabilityProbe func(context.Context, string, uint64) error + +type storageEnvelopeV2CapabilityRPC func(context.Context, string) (*pb.CapabilityReport, error) // newEncryptionPreRegister constructs the interceptor or returns nil // when encryption is not wired (no shared cache or no default @@ -85,20 +91,20 @@ func newEncryptionPreRegister( // proposes RegisterEncryptionWriter(activeDEK, NodeID(raftID), 0) // and surfaces the propose result. func (e *encryptionPreRegister) PreAddMember(ctx context.Context, raftID, address string) error { - activeDEK, ok := e.cache.ActiveStorageKeyID() - if !ok { - // Pre-bootstrap cluster: there is no registry to gate on. - // (Encryption-enabled but not yet bootstrapped, or - // encryption-disabled with an empty cache.) - return nil - } if e.capabilityProbe == nil { return errors.New("encryption: membership V2 capability probe is not configured") } - if err := e.capabilityProbe(ctx, address); err != nil { + newNodeFullID := e.deriveNodeID(raftID) + if err := e.capabilityProbe(ctx, address, newNodeFullID); err != nil { return errors.Wrap(err, "encryption: refuse member before V2 capability confirmation") } - newNodeFullID := e.deriveNodeID(raftID) + activeDEK, ok := e.cache.ActiveStorageKeyID() + if !ok { + // Pre-bootstrap clusters have no registry row to create, but the + // capability probe above still prevents a future V2 latch from + // racing an incompatible membership addition. + return nil + } if err := e.preRegisterDEK(ctx, activeDEK, newNodeFullID, "storage"); err != nil { return err } @@ -112,10 +118,60 @@ func (e *encryptionPreRegister) PreAddMember(ctx context.Context, raftID, addres return e.preRegisterDEK(ctx, raftDEK, newNodeFullID, "raft") } -func probeStorageEnvelopeV2Capability(ctx context.Context, address string) error { +func probeStorageEnvelopeV2Capability(ctx context.Context, address string, expectedFullNodeID uint64) error { + return probeStorageEnvelopeV2CapabilityWithRPC( + ctx, + address, + expectedFullNodeID, + membershipCapabilityProbeTimeout, + getStorageEnvelopeV2Capability, + ) +} + +func probeStorageEnvelopeV2CapabilityWithRPC( + ctx context.Context, + address string, + expectedFullNodeID uint64, + timeout time.Duration, + rpc storageEnvelopeV2CapabilityRPC, +) error { if address == "" { return errors.New("encryption: member address is empty") } + if timeout <= 0 { + return errors.New("encryption: membership capability probe timeout must be positive") + } + if rpc == nil { + return errors.New("encryption: membership capability RPC is not configured") + } + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + report, err := rpc(probeCtx, address) + if err != nil { + return errors.Wrapf(err, "GetCapability member %s", address) + } + if report == nil { + return errors.Errorf("GetCapability member %s returned a nil report", address) + } + if report.GetFullNodeId() != expectedFullNodeID { + return errors.Wrapf( + errMemberCapabilityIDMismatch, + "member %s reported full_node_id=%d, want %d", + address, + report.GetFullNodeId(), + expectedFullNodeID, + ) + } + if !report.GetEncryptionCapable() { + return errors.Wrapf(errMemberNotEncryptionCapable, "member %s", address) + } + if !report.GetStorageEnvelopeV2Capable() { + return errors.Wrapf(errMemberNotStorageV2Capable, "member %s", address) + } + return nil +} + +func getStorageEnvelopeV2Capability(ctx context.Context, address string) (*pb.CapabilityReport, error) { connCache := &kv.GRPCConnCache{} defer func() { if err := connCache.Close(); err != nil { @@ -124,19 +180,13 @@ func probeStorageEnvelopeV2Capability(ctx context.Context, address string) error }() conn, err := connCache.ConnFor(address) if err != nil { - return errors.Wrapf(err, "dial member %s", address) + return nil, errors.Wrapf(err, "dial member %s", address) } report, err := pb.NewEncryptionAdminClient(conn).GetCapability(ctx, &pb.Empty{}) if err != nil { - return errors.Wrapf(err, "GetCapability member %s", address) - } - if !report.GetEncryptionCapable() { - return errors.Wrapf(errMemberNotEncryptionCapable, "member %s", address) + return nil, errors.Wrapf(err, "GetCapability member %s", address) } - if !report.GetStorageEnvelopeV2Capable() { - return errors.Wrapf(errMemberNotStorageV2Capable, "member %s", address) - } - return nil + return report, nil } func (e *encryptionPreRegister) activeRaftDEKForPreRegister() (uint32, bool, error) { diff --git a/main_encryption_confchange_test.go b/main_encryption_confchange_test.go index a31c755a0..3284d692e 100644 --- a/main_encryption_confchange_test.go +++ b/main_encryption_confchange_test.go @@ -3,6 +3,7 @@ package main import ( "context" "testing" + "time" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/encryption" @@ -20,23 +21,37 @@ const stubDerivedNodeID uint64 = 0xCAFEF00DDEADBEEF func stubDeriveNodeID(string) uint64 { return stubDerivedNodeID } -func allowStorageEnvelopeV2Capability(context.Context, string) error { return nil } +func allowStorageEnvelopeV2Capability(context.Context, string, uint64) error { return nil } -// TestEncryptionPreRegister_PreBootstrapSkips pins design §5.2: when -// the StateCache reports (0, false) — no active storage DEK, either -// pre-bootstrap or encryption-disabled — PreAddMember returns nil -// without proposing or reading the registry. -func TestEncryptionPreRegister_PreBootstrapSkips(t *testing.T) { +// TestEncryptionPreRegister_PreBootstrapProbesThenSkipsRegistry pins that +// pre-bootstrap joins still prove V2 reader support before returning without +// a registry proposal. This prevents a later sticky V2 latch from racing an +// incompatible membership addition. +func TestEncryptionPreRegister_PreBootstrapProbesThenSkipsRegistry(t *testing.T) { t.Parallel() cache := encryption.NewStateCache() // zero-value: ActiveStorageKeyID()=(0,false) st := newRegistrationTestStore(t) defaultGroup := &kv.ShardGroup{Store: st} - pre := newEncryptionPreRegister(&kv.ShardedCoordinator{}, defaultGroup, cache, "", stubDeriveNodeID) + var probedFullNodeID uint64 + pre := newEncryptionPreRegister( + &kv.ShardedCoordinator{}, + defaultGroup, + cache, + "", + stubDeriveNodeID, + func(_ context.Context, _ string, fullNodeID uint64) error { + probedFullNodeID = fullNodeID + return nil + }, + ) if pre == nil { t.Fatal("newEncryptionPreRegister returned nil despite non-nil cache+group") } if err := pre.PreAddMember(context.Background(), "n1", "n1:50051"); err != nil { - t.Errorf("PreAddMember should skip pre-bootstrap: got %v", err) + t.Errorf("PreAddMember should skip registry work pre-bootstrap: got %v", err) + } + if probedFullNodeID != stubDerivedNodeID { + t.Fatalf("capability probe full node id = %d, want %d", probedFullNodeID, stubDerivedNodeID) } } @@ -108,8 +123,11 @@ func TestEncryptionPreRegister_RejectsMemberWithoutV2Capability(t *testing.T) { cache, "", stubDeriveNodeID, - func(_ context.Context, address string) error { + func(_ context.Context, address string, fullNodeID uint64) error { probedAddress = address + if fullNodeID != stubDerivedNodeID { + t.Fatalf("capability probe full node id = %d, want %d", fullNodeID, stubDerivedNodeID) + } return sentinel }, ) @@ -129,6 +147,48 @@ func TestEncryptionPreRegister_RejectsMemberWithoutV2Capability(t *testing.T) { } } +func TestProbeStorageEnvelopeV2CapabilityTimesOut(t *testing.T) { + t.Parallel() + const timeout = 20 * time.Millisecond + started := time.Now() + err := probeStorageEnvelopeV2CapabilityWithRPC( + context.Background(), + "unreachable:50051", + stubDerivedNodeID, + timeout, + func(ctx context.Context, _ string) (*pb.CapabilityReport, error) { + <-ctx.Done() + return nil, ctx.Err() + }, + ) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("capability probe error = %v, want deadline exceeded", err) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("capability probe elapsed = %v, want bounded return", elapsed) + } +} + +func TestProbeStorageEnvelopeV2CapabilityRejectsNodeIDMismatch(t *testing.T) { + t.Parallel() + err := probeStorageEnvelopeV2CapabilityWithRPC( + context.Background(), + "other:50051", + stubDerivedNodeID, + time.Second, + func(context.Context, string) (*pb.CapabilityReport, error) { + return &pb.CapabilityReport{ + FullNodeId: stubDerivedNodeID + 1, + EncryptionCapable: true, + StorageEnvelopeV2Capable: true, + }, nil + }, + ) + if !errors.Is(err, errMemberCapabilityIDMismatch) { + t.Fatalf("capability probe error = %v, want node id mismatch", err) + } +} + // TestEncryptionPreRegister_Uint16CollisionReturnsTypedError pins // design §5.2 + §3.1's §6.1-collision branch: when a row exists at // the same uint16 truncation with a DIFFERENT FullNodeID, the guard diff --git a/main_encryption_fanout.go b/main_encryption_fanout.go index d715a7ab8..63a06ae7f 100644 --- a/main_encryption_fanout.go +++ b/main_encryption_fanout.go @@ -59,7 +59,7 @@ func startStorageEnvelopeV2CapabilityMonitor( capabilityFanout adapter.CapabilityFanoutFn, wiring encryptionWriteWiring, ) { - if eg == nil || capabilityFanout == nil || wiring.storageEnvelopeV2Active == nil { + if eg == nil || capabilityFanout == nil || wiring.cache == nil || wiring.storageEnvelopeV2Active == nil { return } eg.Go(func() error { @@ -71,10 +71,7 @@ func startStorageEnvelopeV2CapabilityMonitor( return nil case <-timer.C: } - result, err := capabilityFanout(ctx) - if err == nil && result.StorageEnvelopeV2Ready() { - wiring.activateStorageEnvelopeV2Writes() - slog.Info("encryption: enabled V2 storage envelope writes after cluster capability confirmation") + if tryActivateStorageEnvelopeV2Writes(ctx, capabilityFanout, wiring) { return nil } timer.Reset(storageEnvelopeV2CapabilityRetryInterval) @@ -82,6 +79,23 @@ func startStorageEnvelopeV2CapabilityMonitor( }) } +func tryActivateStorageEnvelopeV2Writes( + ctx context.Context, + capabilityFanout adapter.CapabilityFanoutFn, + wiring encryptionWriteWiring, +) bool { + if _, bootstrapped := wiring.cache.ActiveStorageKeyID(); !bootstrapped { + return false + } + result, err := capabilityFanout(ctx) + if err != nil || !result.StorageEnvelopeV2Ready() { + return false + } + wiring.activateStorageEnvelopeV2Writes() + slog.Info("encryption: enabled V2 storage envelope writes after cluster capability confirmation") + return true +} + // buildCapabilityFanoutFn assembles the adapter.CapabilityFanoutFn the // EnableStorageEnvelope server invokes for its §4 pre-flight check. The // returned closure snapshots the live membership of every Raft group diff --git a/main_encryption_fanout_test.go b/main_encryption_fanout_test.go index e822eeacf..e018fd57e 100644 --- a/main_encryption_fanout_test.go +++ b/main_encryption_fanout_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/bootjp/elastickv/internal/admin" + "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/raftengine" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "golang.org/x/sync/errgroup" @@ -17,7 +18,11 @@ func TestStorageEnvelopeV2CapabilityMonitorActivatesOnce(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) eg, groupCtx := errgroup.WithContext(ctx) - wiring := encryptionWriteWiring{storageEnvelopeV2Active: &atomic.Bool{}} + cache := encryption.NewStateCache() + sidecar := &encryption.Sidecar{Version: encryption.SidecarVersion} + sidecar.Active.Storage = 1 + cache.RefreshFromSidecar(sidecar) + wiring := encryptionWriteWiring{cache: cache, storageEnvelopeV2Active: &atomic.Bool{}} var calls atomic.Int32 startStorageEnvelopeV2CapabilityMonitor(groupCtx, eg, func(context.Context) (admin.CapabilityFanoutResult, error) { calls.Add(1) @@ -41,6 +46,34 @@ func TestStorageEnvelopeV2CapabilityMonitorActivatesOnce(t *testing.T) { } } +func TestStorageEnvelopeV2CapabilityMonitorWaitsForBootstrap(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + eg, groupCtx := errgroup.WithContext(ctx) + wiring := encryptionWriteWiring{ + cache: encryption.NewStateCache(), + storageEnvelopeV2Active: &atomic.Bool{}, + } + var calls atomic.Int32 + startStorageEnvelopeV2CapabilityMonitor(groupCtx, eg, func(context.Context) (admin.CapabilityFanoutResult, error) { + calls.Add(1) + return admin.CapabilityFanoutResult{Verdicts: []admin.CapabilityVerdict{{ + Reachable: true, EncryptionCapable: true, StorageEnvelopeV2Capable: true, + }}}, nil + }, wiring) + time.Sleep(20 * time.Millisecond) + if wiring.storageEnvelopeV2WritesActive() { + t.Fatal("capability monitor activated V2 writes before encryption bootstrap") + } + if got := calls.Load(); got != 0 { + t.Fatalf("capability fanout calls before bootstrap = %d, want zero", got) + } + cancel() + if err := eg.Wait(); err != nil { + t.Fatalf("capability monitor: %v", err) + } +} + // stubConfigReader is a configReader for the snapshot-builder tests. type stubConfigReader struct { cfg raftengine.Configuration From 4fa5a513bbf6933b793bb90f8f9acce5a12e5952 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 21:22:06 +0900 Subject: [PATCH 11/16] encryption: add production KEK providers --- ...6_04_29_partial_data_at_rest_encryption.md | 5 +- ...2026_07_18_implemented_9b_kek_providers.md | 64 +++++++++ go.mod | 32 +++++ go.sum | 85 +++++++++++ internal/encryption/errors.go | 4 +- internal/encryption/kek/aws_kms.go | 106 ++++++++++++++ internal/encryption/kek/aws_kms_test.go | 87 ++++++++++++ internal/encryption/kek/env.go | 91 ++++++++++++ internal/encryption/kek/env_test.go | 52 +++++++ internal/encryption/kek/gcp_kms.go | 134 ++++++++++++++++++ internal/encryption/kek/gcp_kms_test.go | 80 +++++++++++ internal/encryption/kek/kek.go | 4 +- internal/encryption/kek/provider.go | 33 +++++ internal/encryption/kek/source.go | 91 ++++++++++++ internal/encryption/kek/source_test.go | 57 ++++++++ internal/encryption/kek/vault.go | 127 +++++++++++++++++ internal/encryption/kek/vault_test.go | 65 +++++++++ internal/encryption/startup.go | 6 +- main.go | 40 +++--- main_encryption_admin.go | 10 +- main_encryption_kek_source_test.go | 100 +++++++++++++ 21 files changed, 1240 insertions(+), 33 deletions(-) create mode 100644 docs/design/2026_07_18_implemented_9b_kek_providers.md create mode 100644 internal/encryption/kek/aws_kms.go create mode 100644 internal/encryption/kek/aws_kms_test.go create mode 100644 internal/encryption/kek/env.go create mode 100644 internal/encryption/kek/env_test.go create mode 100644 internal/encryption/kek/gcp_kms.go create mode 100644 internal/encryption/kek/gcp_kms_test.go create mode 100644 internal/encryption/kek/provider.go create mode 100644 internal/encryption/kek/source.go create mode 100644 internal/encryption/kek/source_test.go create mode 100644 internal/encryption/kek/vault.go create mode 100644 internal/encryption/kek/vault_test.go create mode 100644 main_encryption_kek_source_test.go diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index fc953defa..ea229f11c 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -1,6 +1,6 @@ # Data-at-rest encryption for elastickv -Status: Partial — Stages 0–8 and 9A shipped (5E deferred); remaining Stage 9 work open +Status: Partial — Stages 0–8 and 9A–9B shipped (5E deferred); remaining Stage 9 work open Author: bootjp Date: 2026-04-29 @@ -33,7 +33,8 @@ Date: 2026-04-29 | 7 | Writer registry + deterministic nonce (§4.1). Covers process-start registration, storage-layer registration gate, runtime re-registration for cutover and rotation, conf-change-time pre-registration, and §5.5 registry projection in `GetSidecarState` / `ResyncSidecar` (`WriterRegistryForCaller`). | shipped | `2026_05_26_implemented_7a_process_start_registration.md` + `2026_05_26_implemented_7a2_storage_layer_registration_enforcement.md` + `2026_05_28_implemented_7b_runtime_reregistration.md` + `2026_05_28_implemented_7b_prime_runtime_reregistration_rotation.md` + `2026_05_29_implemented_7c_confchange_time_registration.md` + `2026_07_11_implemented_7d_sidecar_registry_projection.md` | | 8 | Snapshot header v2 (§4.4); WAL coverage closure (§4.3 / §4.6) | shipped | [`2026_05_29_implemented_8a_snapshot_header_v2.md`](2026_05_29_implemented_8a_snapshot_header_v2.md) + [`2026_06_01_implemented_8b_wal_coverage_closure.md`](2026_06_01_implemented_8b_wal_coverage_closure.md) | | 9A | Compress-then-encrypt, authenticated compression flag, encrypted-store Pebble compression policy, storage benchmark (§6.4, §8.3) | shipped | `2026_07_18_implemented_9a_encryption_compression.md` | -| 9B+ | KMS-backed wrappers, rotation/retire/rewrite, remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8) | open | — | +| 9B | AWS KMS, GCP KMS, Vault Transit, and test/CI env KEK providers; mutually-exclusive source loader and loaded-provider mutator gate (§5.1, §6.1, §6.5) | shipped | `2026_07_18_implemented_9b_kek_providers.md` | +| 9C+ | Rotation budget/rewrap/retire/rewrite, metrics, remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8, §9.2) | open | — | Stages 0–4 ship the entire byte-tag pipeline (storage envelope, raft envelope, FSM dispatch, halt-on-error) but leave it **production diff --git a/docs/design/2026_07_18_implemented_9b_kek_providers.md b/docs/design/2026_07_18_implemented_9b_kek_providers.md new file mode 100644 index 000000000..d8bae8392 --- /dev/null +++ b/docs/design/2026_07_18_implemented_9b_kek_providers.md @@ -0,0 +1,64 @@ +# Stage 9B: production KEK providers + +Status: Implemented +Author: bootjp +Date: 2026-07-18 + +## Scope + +Stage 9B completes the §5.1 KEK source matrix: + +- `--kekUri=aws-kms://` uses AWS KMS `Encrypt` / `Decrypt`, derives + the client region from the ARN, and binds every request to the fixed + `elastickv-purpose=dek-wrap-v1` encryption context. +- `--kekUri=gcp-kms://` uses Google Cloud KMS with fixed + AAD. Request and response CRC32C values are supplied and verified before a + wrapped DEK or plaintext DEK is accepted. +- `--kekUri=vault-transit:///` uses Vault Transit through standard + `VAULT_*` client configuration. Binary DEKs are base64 encoded for the API; + only versioned `vault:v...` ciphertexts are accepted. +- `ELASTICKV_KEK_BASE64` supplies a 32-byte test/CI-only static KEK. The + variable is unset immediately after the decode attempt, including malformed + input. It is also unset when source ambiguity is rejected. +- `--kekFile` keeps its existing owner-only regular-file contract. + +File, URI, and environment sources are mutually exclusive. Ambiguous +configuration fails closed instead of selecting a source by precedence. + +## Runtime wiring + +The startup loader returns the actual loaded `kek.Wrapper`. Startup guards now +derive `KEKConfigured` from that non-nil wrapper rather than from `--kekFile` +text. The same loaded-state boolean is threaded to the EncryptionAdmin mutator +gate, so URI and environment providers can bootstrap/rotate while a failed or +absent provider cannot expose mutating RPCs. + +Remote wrappers use a 30-second operation deadline and reject nil, empty, +integrity-invalid, or non-32-byte provider responses before sidecar state can be +updated. Provider clients are safe for concurrent use and are hidden behind +narrow interfaces for deterministic tests. + +## Compatibility and verification + +No envelope, sidecar, Raft opcode, or snapshot format changes. Existing wrapped +DEKs remain provider-specific opaque bytes, and the `kek.Wrapper` contract is +unchanged. + +Verification includes: + +- fake-client unit tests for AWS encryption context, GCP AAD/CRC32C, and Vault + binary request/response encoding; +- malformed provider response, wrong DEK length, invalid URI, source conflict, + and environment-unset tests; +- a production-wiring integration test that loads the environment KEK through + the source selector, bootstraps both DEKs, enables the storage envelope, + writes encrypted data, snapshots it, restores it into an encrypted Pebble + store, and reads the original plaintext; +- startup/mutator caller audit confirming the gate consumes loaded-wrapper + state rather than the legacy file flag. + +## Remaining work + +Stage 5E discovery batching and Stage 9C+ rotation-budget, rewrap, rewrite, +retirement, metrics, remaining benchmark, and encrypted Jepsen requirements are +not part of this milestone. diff --git a/go.mod b/go.mod index 02cfe9faf..431a2970a 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,13 @@ go 1.26 toolchain go1.26.5 require ( + cloud.google.com/go/kms v1.31.0 github.com/Jille/grpc-multi-resolver v1.3.0 github.com/aws/aws-sdk-go-v2 v1.42.1 github.com/aws/aws-sdk-go-v2/config v1.32.29 github.com/aws/aws-sdk-go-v2/credentials v1.19.28 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.60.0 + github.com/aws/aws-sdk-go-v2/service/kms v1.54.1 github.com/aws/smithy-go v1.27.3 github.com/cockroachdb/errors v1.14.0 github.com/cockroachdb/pebble/v2 v2.1.6 @@ -19,6 +21,7 @@ require ( github.com/goccy/go-json v0.10.6 github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e github.com/hanwen/go-fuse/v2 v2.10.1 + github.com/hashicorp/vault/api v1.23.0 github.com/klauspost/compress v1.19.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.23.2 @@ -41,6 +44,12 @@ require ( ) require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.7.0 // indirect + cloud.google.com/go/longrunning v0.9.0 // indirect github.com/DataDog/zstd v1.5.7 // indirect github.com/RaduBerinde/axisds v0.1.0 // indirect github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect @@ -56,6 +65,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.44.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect @@ -65,23 +75,40 @@ require ( github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.7.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect + github.com/googleapis/gax-go/v2 v2.21.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/tidwall/btree v1.1.0 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -92,6 +119,7 @@ require ( go.etcd.io/etcd/pkg/v3 v3.7.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect @@ -101,7 +129,11 @@ require ( golang.org/x/crypto v0.52.0 // indirect golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect golang.org/x/net v0.55.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/text v0.37.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/api v0.274.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 1203c7d30..d3710ade0 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,17 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= +cloud.google.com/go/kms v1.31.0 h1:LS8N92OxFDgOLg5NCo3OmbvjtQAIVT5gUHVLKIDHaFE= +cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U= +cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY= +cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jille/grpc-multi-resolver v1.3.0 h1:cbVm1TtWP7YxdiCCZ8gU4/78pYO2OXpzZSFAAUMdFLs= @@ -30,6 +44,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.7 h1:uqsK github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.7/go.mod h1:Js/P8Zbwe1mRejnD+OpFLyQiJ8ioQlo3GMAg7Dfxk7w= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= +github.com/aws/aws-sdk-go-v2/service/kms v1.54.1 h1:aeJAJyvWS3gQ679pJbz8ZdOh3MViD1zvEdoZMVEawbg= +github.com/aws/aws-sdk-go-v2/service/kms v1.54.1/go.mod h1:0RXNc6Yf3AvSMldGD6Lcch96Ojlw2TtGnHsqfD/L4u8= github.com/aws/aws-sdk-go-v2/service/signin v1.4.0 h1:sLzmJGCMv+C8KqiJgEqDLB6vxaJGmobRh4rr//ZpA3w= github.com/aws/aws-sdk-go-v2/service/signin v1.4.0/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg= github.com/aws/aws-sdk-go-v2/service/sso v1.32.0 h1:qjMmry/cBDee1E/2gyvel0uRYCi3mwRZ2hf6N+GAodo= @@ -46,8 +62,12 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= @@ -71,23 +91,37 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/getsentry/sentry-go v0.47.0 h1:AnSMSyrYA5qZCIN/2xpgAAwv63sVULV+vBq37ajouc8= github.com/getsentry/sentry-go v0.47.0/go.mod h1:h+b4VHpKnK7aUXB5wc+KDnPgp9ZtfliRD4eV85FbiSA= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -100,12 +134,41 @@ github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e h1:4bw4WeyTYPp0sma github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl/wMbiI= +github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hanwen/go-fuse/v2 v2.10.1 h1:QAqZuc9+aBtTou+OPruU/hkYQYCkgPtQd2QaepHkTTs= github.com/hanwen/go-fuse/v2 v2.10.1/go.mod h1:aU7NkGYZUmuJrZapoI3mEcNve7PZTySUOLBuch/vR6U= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= +github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= +github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A= +github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -120,8 +183,16 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -131,6 +202,8 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= @@ -146,6 +219,8 @@ github.com/redis/go-redis/v9 v9.21.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3b github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= @@ -186,6 +261,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= @@ -223,6 +300,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -237,6 +316,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -247,6 +328,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA= +google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= diff --git a/internal/encryption/errors.go b/internal/encryption/errors.go index fcec5a201..ee7feaa73 100644 --- a/internal/encryption/errors.go +++ b/internal/encryption/errors.go @@ -159,7 +159,7 @@ var ( // and the operator's clear intent (enable encryption) cannot be // satisfied. Fail fast at startup rather than discover the // mismatch later via a halted apply loop. - ErrKEKRequiredWithFlag = errors.New("encryption: --encryption-enabled is set but no KEK source (--kekFile) was provided; refusing to start (set --kekFile or unset --encryption-enabled)") + ErrKEKRequiredWithFlag = errors.New("encryption: --encryption-enabled is set but no KEK source was provided; refusing to start (set --kekFile, --kekUri, or ELASTICKV_KEK_BASE64, or unset --encryption-enabled)") // ErrKEKMismatch is the §9.1 startup-refusal guard raised when // the data dir contains a sidecar whose wrapped DEKs do NOT @@ -171,7 +171,7 @@ var ( // DEK. Recovery requires the operator to either point // --kekFile at the correct KEK file or restore the data dir // from a backup that matches the supplied KEK. - ErrKEKMismatch = errors.New("encryption: configured KEK cannot unwrap one or more wrapped DEKs in the sidecar; refusing to start (verify --kekFile matches the KEK that bootstrapped this data dir)") + ErrKEKMismatch = errors.New("encryption: configured KEK cannot unwrap one or more wrapped DEKs in the sidecar; refusing to start (verify the configured KEK source matches the KEK that bootstrapped this data dir)") // ErrLocalEpochExhausted is the §9.1 startup-refusal guard // raised when any active DEK in the sidecar has reached the diff --git a/internal/encryption/kek/aws_kms.go b/internal/encryption/kek/aws_kms.go new file mode 100644 index 000000000..f4400be82 --- /dev/null +++ b/internal/encryption/kek/aws_kms.go @@ -0,0 +1,106 @@ +package kek + +import ( + "context" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + awskms "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/cockroachdb/errors" +) + +const awsEncryptionContextKey = "elastickv-purpose" +const awsEncryptionContextValue = "dek-wrap-v1" + +type awsKMSClient interface { + Encrypt(context.Context, *awskms.EncryptInput, ...func(*awskms.Options)) (*awskms.EncryptOutput, error) + Decrypt(context.Context, *awskms.DecryptInput, ...func(*awskms.Options)) (*awskms.DecryptOutput, error) +} + +// AWSKMSWrapper wraps DEKs using an AWS KMS symmetric ENCRYPT_DECRYPT key. +type AWSKMSWrapper struct { + client awsKMSClient + keyARN string + timeout time.Duration +} + +// NewAWSKMSWrapper loads the standard AWS credential chain and derives the KMS +// endpoint region from keyARN. +func NewAWSKMSWrapper(ctx context.Context, keyARN string) (*AWSKMSWrapper, error) { + parsed, err := arn.Parse(keyARN) + if err != nil || parsed.Service != "kms" || parsed.Region == "" || parsed.AccountID == "" || + (!strings.HasPrefix(parsed.Resource, "key/") && !strings.HasPrefix(parsed.Resource, "alias/")) { + return nil, errors.Wrapf(ErrInvalidKEKURI, "invalid AWS KMS key ARN %q", keyARN) + } + cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(parsed.Region)) + if err != nil { + return nil, errors.Wrap(err, "kek: load AWS configuration") + } + return newAWSKMSWrapper(awskms.NewFromConfig(cfg), keyARN), nil +} + +func newAWSKMSWrapper(client awsKMSClient, keyARN string) *AWSKMSWrapper { + return &AWSKMSWrapper{client: client, keyARN: keyARN, timeout: providerRequestTimeout} +} + +// Wrap calls AWS KMS Encrypt with a fixed encryption context that is required +// again by Unwrap. +func (w *AWSKMSWrapper) Wrap(dek []byte) ([]byte, error) { + if w == nil || w.client == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS client is nil") + } + if err := validateDEK(dek); err != nil { + return nil, err + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + out, err := w.client.Encrypt(ctx, &awskms.EncryptInput{ + KeyId: aws.String(w.keyARN), + Plaintext: dek, + EncryptionContext: map[string]string{ + awsEncryptionContextKey: awsEncryptionContextValue, + }, + }) + if err != nil { + return nil, errors.Wrap(err, "kek: AWS KMS Encrypt") + } + if out == nil || len(out.CiphertextBlob) == 0 { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS returned empty ciphertext") + } + return append([]byte(nil), out.CiphertextBlob...), nil +} + +// Unwrap calls AWS KMS Decrypt and rejects any non-32-byte plaintext response. +func (w *AWSKMSWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w == nil || w.client == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS client is nil") + } + if len(wrapped) == 0 { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS ciphertext is empty") + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + out, err := w.client.Decrypt(ctx, &awskms.DecryptInput{ + CiphertextBlob: wrapped, + KeyId: aws.String(w.keyARN), + EncryptionContext: map[string]string{ + awsEncryptionContextKey: awsEncryptionContextValue, + }, + }) + if err != nil { + return nil, errors.Wrap(err, "kek: AWS KMS Decrypt") + } + if out == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS returned nil plaintext") + } + if err := validateDEK(out.Plaintext); err != nil { + return nil, errors.Wrap(err, "kek: AWS KMS plaintext") + } + return append([]byte(nil), out.Plaintext...), nil +} + +// Name returns the provider and configured key ARN. +func (w *AWSKMSWrapper) Name() string { return "aws-kms:" + w.keyARN } diff --git a/internal/encryption/kek/aws_kms_test.go b/internal/encryption/kek/aws_kms_test.go new file mode 100644 index 000000000..97fb30767 --- /dev/null +++ b/internal/encryption/kek/aws_kms_test.go @@ -0,0 +1,87 @@ +package kek + +import ( + "bytes" + "context" + "testing" + + awskms "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +type fakeAWSKMSClient struct { + encryptInput *awskms.EncryptInput + decryptInput *awskms.DecryptInput + encryptOutput *awskms.EncryptOutput + decryptOutput *awskms.DecryptOutput + err error +} + +func (f *fakeAWSKMSClient) Encrypt(_ context.Context, input *awskms.EncryptInput, _ ...func(*awskms.Options)) (*awskms.EncryptOutput, error) { + f.encryptInput = input + return f.encryptOutput, f.err +} + +func (f *fakeAWSKMSClient) Decrypt(_ context.Context, input *awskms.DecryptInput, _ ...func(*awskms.Options)) (*awskms.DecryptOutput, error) { + f.decryptInput = input + return f.decryptOutput, f.err +} + +func TestAWSKMSWrapperRequestBinding(t *testing.T) { + const keyARN = "arn:aws:kms:us-east-1:123456789012:key/1234abcd" + dek := bytes.Repeat([]byte{0x42}, fileKEKSize) + client := &fakeAWSKMSClient{ + encryptOutput: &awskms.EncryptOutput{CiphertextBlob: []byte("wrapped")}, + decryptOutput: &awskms.DecryptOutput{Plaintext: append([]byte(nil), dek...)}, + } + wrapper := newAWSKMSWrapper(client, keyARN) + + wrapped, err := wrapper.Wrap(dek) + require.NoError(t, err) + require.Equal(t, []byte("wrapped"), wrapped) + require.Equal(t, keyARN, *client.encryptInput.KeyId) + require.Equal(t, dek, client.encryptInput.Plaintext) + require.Equal(t, awsEncryptionContextValue, client.encryptInput.EncryptionContext[awsEncryptionContextKey]) + + plain, err := wrapper.Unwrap(wrapped) + require.NoError(t, err) + require.Equal(t, dek, plain) + require.Equal(t, keyARN, *client.decryptInput.KeyId) + require.Equal(t, awsEncryptionContextValue, client.decryptInput.EncryptionContext[awsEncryptionContextKey]) + require.Equal(t, "aws-kms:"+keyARN, wrapper.Name()) +} + +func TestAWSKMSWrapperRejectsProviderFailures(t *testing.T) { + const keyARN = "arn:aws:kms:us-east-1:123456789012:key/1234abcd" + dek := bytes.Repeat([]byte{0x42}, fileKEKSize) + for _, tc := range []struct { + name string + client *fakeAWSKMSClient + unwrap bool + want error + }{ + {name: "encrypt error", client: &fakeAWSKMSClient{err: errors.New("denied")}, want: errors.New("sentinel")}, + {name: "empty ciphertext", client: &fakeAWSKMSClient{encryptOutput: &awskms.EncryptOutput{}}, want: ErrInvalidProviderResponse}, + {name: "short plaintext", client: &fakeAWSKMSClient{decryptOutput: &awskms.DecryptOutput{Plaintext: []byte("short")}}, unwrap: true, want: ErrInvalidDEKLength}, + } { + t.Run(tc.name, func(t *testing.T) { + wrapper := newAWSKMSWrapper(tc.client, keyARN) + var err error + if tc.unwrap { + _, err = wrapper.Unwrap([]byte("wrapped")) + } else { + _, err = wrapper.Wrap(dek) + } + require.Error(t, err) + if errors.Is(tc.want, ErrInvalidProviderResponse) || errors.Is(tc.want, ErrInvalidDEKLength) { + require.ErrorIs(t, err, tc.want) + } + }) + } +} + +func TestNewAWSKMSWrapperRejectsInvalidARNBeforeConfigLoad(t *testing.T) { + _, err := NewAWSKMSWrapper(context.Background(), "not-an-arn") + require.ErrorIs(t, err, ErrInvalidKEKURI) +} diff --git a/internal/encryption/kek/env.go b/internal/encryption/kek/env.go new file mode 100644 index 000000000..59fa5c48a --- /dev/null +++ b/internal/encryption/kek/env.go @@ -0,0 +1,91 @@ +package kek + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "os" + + "github.com/cockroachdb/errors" +) + +// EnvVar is the test/CI-only environment variable accepted as a static KEK +// source. Production deployments should use a remote KMS provider. +const EnvVar = "ELASTICKV_KEK_BASE64" + +var ErrNilEnvWrapper = errors.New("kek: EnvWrapper is nil or uninitialised; construct with NewEnvWrapper") + +// EnvWrapper wraps DEKs with a process-local AES-256-GCM key loaded from +// EnvVar. NewEnvWrapper removes EnvVar immediately after decoding it so the +// raw KEK is not retained in the process environment. +type EnvWrapper struct { + aead cipher.AEAD +} + +// NewEnvWrapper reads a standard-base64 encoded 32-byte KEK. EnvVar is unset +// after the decode attempt on both success and failure paths. +func NewEnvWrapper() (*EnvWrapper, error) { + encoded, ok := os.LookupEnv(EnvVar) + if !ok { + return nil, errors.Errorf("kek: %s is not set", EnvVar) + } + raw, decodeErr := base64.StdEncoding.DecodeString(encoded) + unsetErr := os.Unsetenv(EnvVar) + if decodeErr != nil { + return nil, errors.Wrapf(decodeErr, "kek: decode %s", EnvVar) + } + defer clear(raw) + if unsetErr != nil { + return nil, errors.Wrapf(unsetErr, "kek: unset %s", EnvVar) + } + if err := validateDEK(raw); err != nil { + return nil, errors.Wrapf(err, "kek: %s", EnvVar) + } + block, err := aes.NewCipher(raw) + if err != nil { + return nil, errors.Wrap(err, "kek: env aes.NewCipher") + } + aead, err := cipher.NewGCM(block) + if err != nil { + return nil, errors.Wrap(err, "kek: env cipher.NewGCM") + } + return &EnvWrapper{aead: aead}, nil +} + +// Wrap returns nonce || AES-GCM(KEK, DEK), matching FileWrapper's local +// provider format. +func (w *EnvWrapper) Wrap(dek []byte) ([]byte, error) { + if w == nil || w.aead == nil { + return nil, errors.WithStack(ErrNilEnvWrapper) + } + if err := validateDEK(dek); err != nil { + return nil, err + } + nonce := make([]byte, fileNonceSize) + if _, err := rand.Read(nonce); err != nil { + return nil, errors.Wrap(err, "kek: env random nonce") + } + out := make([]byte, 0, fileNonceSize+fileKEKSize+fileTagSize) + out = append(out, nonce...) + return w.aead.Seal(out, nonce, dek, nil), nil +} + +// Unwrap authenticates and decrypts an EnvWrapper payload. +func (w *EnvWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w == nil || w.aead == nil { + return nil, errors.WithStack(ErrNilEnvWrapper) + } + if len(wrapped) != fileNonceSize+fileKEKSize+fileTagSize { + return nil, errors.Errorf("kek: env wrapped DEK is %d bytes, want %d", + len(wrapped), fileNonceSize+fileKEKSize+fileTagSize) + } + plain, err := w.aead.Open(nil, wrapped[:fileNonceSize], wrapped[fileNonceSize:], nil) + if err != nil { + return nil, errors.Wrap(err, "kek: env AES-GCM Open") + } + return plain, nil +} + +// Name identifies the environment-backed provider without exposing key bytes. +func (*EnvWrapper) Name() string { return "env" } diff --git a/internal/encryption/kek/env_test.go b/internal/encryption/kek/env_test.go new file mode 100644 index 000000000..a63cdb8ba --- /dev/null +++ b/internal/encryption/kek/env_test.go @@ -0,0 +1,52 @@ +package kek + +import ( + "bytes" + "encoding/base64" + "os" + "testing" + + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestEnvWrapperRoundTripAndUnset(t *testing.T) { + key := bytes.Repeat([]byte{0x41}, fileKEKSize) + t.Setenv(EnvVar, base64.StdEncoding.EncodeToString(key)) + wrapper, err := NewEnvWrapper() + require.NoError(t, err) + _, stillSet := os.LookupEnv(EnvVar) + require.False(t, stillSet) + require.Equal(t, "env", wrapper.Name()) + + dek := bytes.Repeat([]byte{0xA5}, fileKEKSize) + wrapped, err := wrapper.Wrap(dek) + require.NoError(t, err) + require.NotEqual(t, dek, wrapped) + plain, err := wrapper.Unwrap(wrapped) + require.NoError(t, err) + require.Equal(t, dek, plain) + + wrapped[len(wrapped)-1] ^= 0x01 + _, err = wrapper.Unwrap(wrapped) + require.Error(t, err) +} + +func TestEnvWrapperInvalidInputStillUnsets(t *testing.T) { + t.Setenv(EnvVar, "not-base64") + _, err := NewEnvWrapper() + require.Error(t, err) + _, stillSet := os.LookupEnv(EnvVar) + require.False(t, stillSet) +} + +func TestEnvWrapperRejectsInvalidDEKLength(t *testing.T) { + t.Setenv(EnvVar, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{1}, fileKEKSize))) + wrapper, err := NewEnvWrapper() + require.NoError(t, err) + _, err = wrapper.Wrap([]byte("short")) + require.ErrorIs(t, err, ErrInvalidDEKLength) + var nilWrapper *EnvWrapper + _, err = nilWrapper.Wrap(bytes.Repeat([]byte{1}, fileKEKSize)) + require.True(t, errors.Is(err, ErrNilEnvWrapper)) +} diff --git a/internal/encryption/kek/gcp_kms.go b/internal/encryption/kek/gcp_kms.go new file mode 100644 index 000000000..09c85af6d --- /dev/null +++ b/internal/encryption/kek/gcp_kms.go @@ -0,0 +1,134 @@ +package kek + +import ( + "context" + "hash/crc32" + "strings" + "time" + + gcpkms "cloud.google.com/go/kms/apiv1" + "cloud.google.com/go/kms/apiv1/kmspb" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +var gcpKMSAAD = []byte("elastickv-dek-wrap-v1") + +type gcpKMSClient interface { + Encrypt(context.Context, *kmspb.EncryptRequest) (*kmspb.EncryptResponse, error) + Decrypt(context.Context, *kmspb.DecryptRequest) (*kmspb.DecryptResponse, error) +} + +type gcpSDKClient struct { + client *gcpkms.KeyManagementClient +} + +func (c gcpSDKClient) Encrypt(ctx context.Context, req *kmspb.EncryptRequest) (*kmspb.EncryptResponse, error) { + out, err := c.client.Encrypt(ctx, req) + return out, errors.Wrap(err, "GCP KMS SDK Encrypt") +} + +func (c gcpSDKClient) Decrypt(ctx context.Context, req *kmspb.DecryptRequest) (*kmspb.DecryptResponse, error) { + out, err := c.client.Decrypt(ctx, req) + return out, errors.Wrap(err, "GCP KMS SDK Decrypt") +} + +// GCPKMSWrapper wraps DEKs using a Google Cloud KMS symmetric CryptoKey. +type GCPKMSWrapper struct { + client gcpKMSClient + keyName string + timeout time.Duration +} + +// NewGCPKMSWrapper uses Application Default Credentials to construct a Cloud +// KMS client for keyName. +func NewGCPKMSWrapper(ctx context.Context, keyName string) (*GCPKMSWrapper, error) { + if !validGCPKeyName(keyName) { + return nil, errors.Wrapf(ErrInvalidKEKURI, "invalid GCP KMS CryptoKey name %q", keyName) + } + client, err := gcpkms.NewKeyManagementClient(ctx) + if err != nil { + return nil, errors.Wrap(err, "kek: create GCP KMS client") + } + return newGCPKMSWrapper(gcpSDKClient{client: client}, keyName), nil +} + +func newGCPKMSWrapper(client gcpKMSClient, keyName string) *GCPKMSWrapper { + return &GCPKMSWrapper{client: client, keyName: keyName, timeout: providerRequestTimeout} +} + +func validGCPKeyName(name string) bool { + parts := strings.Split(name, "/") + return len(parts) == 8 && parts[0] == "projects" && parts[1] != "" && + parts[2] == "locations" && parts[3] != "" && parts[4] == "keyRings" && + parts[5] != "" && parts[6] == "cryptoKeys" && parts[7] != "" +} + +func crc32c(data []byte) *wrapperspb.Int64Value { + return wrapperspb.Int64(int64(crc32.Checksum(data, crc32.MakeTable(crc32.Castagnoli)))) +} + +func checksumMatches(data []byte, checksum *wrapperspb.Int64Value) bool { + return checksum != nil && crc32c(data).Value == checksum.Value +} + +// Wrap calls Cloud KMS Encrypt with fixed AAD and verifies request/response +// CRC32C integrity metadata. +func (w *GCPKMSWrapper) Wrap(dek []byte) ([]byte, error) { + if w == nil || w.client == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: GCP KMS client is nil") + } + if err := validateDEK(dek); err != nil { + return nil, err + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + out, err := w.client.Encrypt(ctx, &kmspb.EncryptRequest{ + Name: w.keyName, + Plaintext: dek, + AdditionalAuthenticatedData: gcpKMSAAD, + PlaintextCrc32C: crc32c(dek), + AdditionalAuthenticatedDataCrc32C: crc32c(gcpKMSAAD), + }) + if err != nil { + return nil, errors.Wrap(err, "kek: GCP KMS Encrypt") + } + if out == nil || len(out.Ciphertext) == 0 || !out.VerifiedPlaintextCrc32C || + !out.VerifiedAdditionalAuthenticatedDataCrc32C || !checksumMatches(out.Ciphertext, out.CiphertextCrc32C) { + return nil, errors.WithStack(ErrInvalidProviderResponse) + } + return append([]byte(nil), out.Ciphertext...), nil +} + +// Unwrap calls Cloud KMS Decrypt and verifies the plaintext CRC32C before +// accepting the 32-byte DEK. +func (w *GCPKMSWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w == nil || w.client == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: GCP KMS client is nil") + } + if len(wrapped) == 0 { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: GCP KMS ciphertext is empty") + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + out, err := w.client.Decrypt(ctx, &kmspb.DecryptRequest{ + Name: w.keyName, + Ciphertext: wrapped, + AdditionalAuthenticatedData: gcpKMSAAD, + CiphertextCrc32C: crc32c(wrapped), + AdditionalAuthenticatedDataCrc32C: crc32c(gcpKMSAAD), + }) + if err != nil { + return nil, errors.Wrap(err, "kek: GCP KMS Decrypt") + } + if out == nil || !checksumMatches(out.Plaintext, out.PlaintextCrc32C) { + return nil, errors.WithStack(ErrInvalidProviderResponse) + } + if err := validateDEK(out.Plaintext); err != nil { + return nil, errors.Wrap(err, "kek: GCP KMS plaintext") + } + return append([]byte(nil), out.Plaintext...), nil +} + +// Name returns the provider and configured CryptoKey resource name. +func (w *GCPKMSWrapper) Name() string { return "gcp-kms:" + w.keyName } diff --git a/internal/encryption/kek/gcp_kms_test.go b/internal/encryption/kek/gcp_kms_test.go new file mode 100644 index 000000000..a0c394aaa --- /dev/null +++ b/internal/encryption/kek/gcp_kms_test.go @@ -0,0 +1,80 @@ +package kek + +import ( + "bytes" + "context" + "testing" + + "cloud.google.com/go/kms/apiv1/kmspb" + "github.com/stretchr/testify/require" +) + +type fakeGCPKMSClient struct { + encryptInput *kmspb.EncryptRequest + decryptInput *kmspb.DecryptRequest + encryptOutput *kmspb.EncryptResponse + decryptOutput *kmspb.DecryptResponse + err error +} + +func (f *fakeGCPKMSClient) Encrypt(_ context.Context, input *kmspb.EncryptRequest) (*kmspb.EncryptResponse, error) { + f.encryptInput = input + return f.encryptOutput, f.err +} + +func (f *fakeGCPKMSClient) Decrypt(_ context.Context, input *kmspb.DecryptRequest) (*kmspb.DecryptResponse, error) { + f.decryptInput = input + return f.decryptOutput, f.err +} + +func TestGCPKMSWrapperRequestAndResponseIntegrity(t *testing.T) { + const keyName = "projects/p/locations/global/keyRings/r/cryptoKeys/k" + dek := bytes.Repeat([]byte{0x52}, fileKEKSize) + ciphertext := []byte("gcp-wrapped") + client := &fakeGCPKMSClient{ + encryptOutput: &kmspb.EncryptResponse{ + Ciphertext: ciphertext, + CiphertextCrc32C: crc32c(ciphertext), + VerifiedPlaintextCrc32C: true, + VerifiedAdditionalAuthenticatedDataCrc32C: true, + }, + decryptOutput: &kmspb.DecryptResponse{Plaintext: dek, PlaintextCrc32C: crc32c(dek)}, + } + wrapper := newGCPKMSWrapper(client, keyName) + + wrapped, err := wrapper.Wrap(dek) + require.NoError(t, err) + require.Equal(t, ciphertext, wrapped) + require.Equal(t, keyName, client.encryptInput.Name) + require.Equal(t, gcpKMSAAD, client.encryptInput.AdditionalAuthenticatedData) + require.True(t, checksumMatches(dek, client.encryptInput.PlaintextCrc32C)) + + plain, err := wrapper.Unwrap(wrapped) + require.NoError(t, err) + require.Equal(t, dek, plain) + require.Equal(t, gcpKMSAAD, client.decryptInput.AdditionalAuthenticatedData) + require.True(t, checksumMatches(wrapped, client.decryptInput.CiphertextCrc32C)) + require.Equal(t, "gcp-kms:"+keyName, wrapper.Name()) +} + +func TestGCPKMSWrapperRejectsIntegrityFailures(t *testing.T) { + const keyName = "projects/p/locations/global/keyRings/r/cryptoKeys/k" + dek := bytes.Repeat([]byte{0x52}, fileKEKSize) + client := &fakeGCPKMSClient{encryptOutput: &kmspb.EncryptResponse{ + Ciphertext: []byte("ciphertext"), + CiphertextCrc32C: crc32c([]byte("different")), + VerifiedPlaintextCrc32C: true, + VerifiedAdditionalAuthenticatedDataCrc32C: true, + }} + _, err := newGCPKMSWrapper(client, keyName).Wrap(dek) + require.ErrorIs(t, err, ErrInvalidProviderResponse) + + client.decryptOutput = &kmspb.DecryptResponse{Plaintext: dek, PlaintextCrc32C: crc32c([]byte("different"))} + _, err = newGCPKMSWrapper(client, keyName).Unwrap([]byte("ciphertext")) + require.ErrorIs(t, err, ErrInvalidProviderResponse) +} + +func TestValidGCPKeyName(t *testing.T) { + require.True(t, validGCPKeyName("projects/p/locations/global/keyRings/r/cryptoKeys/k")) + require.False(t, validGCPKeyName("projects/p/keyRings/r/cryptoKeys/k")) +} diff --git a/internal/encryption/kek/kek.go b/internal/encryption/kek/kek.go index a4550af2e..93dc8e8f2 100644 --- a/internal/encryption/kek/kek.go +++ b/internal/encryption/kek/kek.go @@ -6,8 +6,8 @@ // in a KMS, a sealed file, or HashiCorp Vault — and only exercised at // process boot and at DEK rotation. // -// Stage 0 ships only the FileWrapper for tests / single-host clusters. -// AWS KMS, GCP KMS, and Vault providers are added in Stage 9. +// Implementations include AWS KMS, GCP KMS, Vault Transit, a static file, and +// the test/CI-only environment provider described by the design. package kek // Wrapper wraps and unwraps DEK bytes under an externally-held KEK. diff --git a/internal/encryption/kek/provider.go b/internal/encryption/kek/provider.go new file mode 100644 index 000000000..e2350fbad --- /dev/null +++ b/internal/encryption/kek/provider.go @@ -0,0 +1,33 @@ +package kek + +import ( + "context" + "time" + + "github.com/cockroachdb/errors" +) + +const providerRequestTimeout = 30 * time.Second + +var ( + // ErrInvalidDEKLength rejects non-AES-256 data keys at every provider + // boundary, including malformed provider responses. + ErrInvalidDEKLength = errors.New("kek: DEK must be exactly 32 bytes") + // ErrInvalidProviderResponse rejects an empty or integrity-invalid response + // before it can be persisted in the encryption sidecar. + ErrInvalidProviderResponse = errors.New("kek: invalid provider response") +) + +func validateDEK(dek []byte) error { + if len(dek) != fileKEKSize { + return errors.Wrapf(ErrInvalidDEKLength, "got %d bytes", len(dek)) + } + return nil +} + +func requestContext(timeout time.Duration) (context.Context, context.CancelFunc) { + if timeout <= 0 { + timeout = providerRequestTimeout + } + return context.WithTimeout(context.Background(), timeout) +} diff --git a/internal/encryption/kek/source.go b/internal/encryption/kek/source.go new file mode 100644 index 000000000..fecf5bc7a --- /dev/null +++ b/internal/encryption/kek/source.go @@ -0,0 +1,91 @@ +package kek + +import ( + "context" + "os" + "strings" + + "github.com/cockroachdb/errors" +) + +const ( + awsKMSScheme = "aws-kms://" + gcpKMSScheme = "gcp-kms://" + vaultTransitScheme = "vault-transit://" +) + +var ( + // ErrMultipleKEKSources rejects ambiguous key configuration rather than + // silently selecting one source by precedence. + ErrMultipleKEKSources = errors.New("kek: configure exactly one of --kekFile, --kekUri, or ELASTICKV_KEK_BASE64") + // ErrInvalidKEKURI rejects unknown providers and malformed provider targets. + ErrInvalidKEKURI = errors.New("kek: invalid KEK URI") +) + +// NewWrapperFromSources resolves exactly one file, URI, or environment-backed +// KEK. No configured source returns nil so encryption-disabled deployments keep +// their existing startup behavior. +func NewWrapperFromSources(ctx context.Context, filePath, uri string) (Wrapper, error) { + _, envSet := os.LookupEnv(EnvVar) + if countConfiguredSources(filePath != "", uri != "", envSet) > 1 { + return nil, rejectSourceConflict(envSet) + } + switch { + case filePath != "": + wrapper, err := NewFileWrapper(filePath) + if err != nil { + return nil, errors.Wrapf(err, "kek: load file %q", filePath) + } + return wrapper, nil + case uri != "": + return newURIWrapper(ctx, uri) + case envSet: + return NewEnvWrapper() + default: + return nil, nil + } +} + +func countConfiguredSources(sources ...bool) int { + configured := 0 + for _, present := range sources { + if present { + configured++ + } + } + return configured +} + +func rejectSourceConflict(envSet bool) error { + if envSet { + if err := os.Unsetenv(EnvVar); err != nil { + return errors.Wrapf(err, "kek: unset %s after source conflict", EnvVar) + } + } + return errors.WithStack(ErrMultipleKEKSources) +} + +func newURIWrapper(ctx context.Context, uri string) (Wrapper, error) { + switch { + case strings.HasPrefix(uri, awsKMSScheme): + target := strings.TrimPrefix(uri, awsKMSScheme) + if target == "" { + return nil, errors.WithStack(ErrInvalidKEKURI) + } + return NewAWSKMSWrapper(ctx, target) + case strings.HasPrefix(uri, gcpKMSScheme): + target := strings.TrimPrefix(uri, gcpKMSScheme) + if target == "" { + return nil, errors.WithStack(ErrInvalidKEKURI) + } + return NewGCPKMSWrapper(ctx, target) + case strings.HasPrefix(uri, vaultTransitScheme): + target := strings.TrimPrefix(uri, vaultTransitScheme) + if target == "" { + return nil, errors.WithStack(ErrInvalidKEKURI) + } + return NewVaultTransitWrapper(target) + default: + return nil, errors.Wrapf(ErrInvalidKEKURI, "unsupported URI %q", uri) + } +} diff --git a/internal/encryption/kek/source_test.go b/internal/encryption/kek/source_test.go new file mode 100644 index 000000000..b6aa17505 --- /dev/null +++ b/internal/encryption/kek/source_test.go @@ -0,0 +1,57 @@ +package kek + +import ( + "bytes" + "context" + "encoding/base64" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewWrapperFromSourcesNoSource(t *testing.T) { + t.Setenv(EnvVar, "") + require.NoError(t, os.Unsetenv(EnvVar)) + wrapper, err := NewWrapperFromSources(context.Background(), "", "") + require.NoError(t, err) + require.Nil(t, wrapper) +} + +func TestNewWrapperFromSourcesRejectsAmbiguity(t *testing.T) { + t.Setenv(EnvVar, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{1}, fileKEKSize))) + _, err := NewWrapperFromSources(context.Background(), "/tmp/kek", "") + require.ErrorIs(t, err, ErrMultipleKEKSources) + _, stillSet := os.LookupEnv(EnvVar) + require.False(t, stillSet) +} + +func TestNewWrapperFromSourcesFileAndEnv(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "kek.bin") + require.NoError(t, os.WriteFile(path, bytes.Repeat([]byte{2}, fileKEKSize), 0o600)) + + wrapper, err := NewWrapperFromSources(context.Background(), path, "") + require.NoError(t, err) + require.Contains(t, wrapper.Name(), "file:") + + t.Setenv(EnvVar, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{3}, fileKEKSize))) + wrapper, err = NewWrapperFromSources(context.Background(), "", "") + require.NoError(t, err) + require.Equal(t, "env", wrapper.Name()) + _, stillSet := os.LookupEnv(EnvVar) + require.False(t, stillSet) +} + +func TestNewURIWrapperRejectsInvalidTargetsWithoutNetwork(t *testing.T) { + for _, uri := range []string{ + "unknown://key", + "aws-kms://not-an-arn", + "gcp-kms://projects/incomplete", + "vault-transit://transit", + } { + _, err := newURIWrapper(context.Background(), uri) + require.ErrorIsf(t, err, ErrInvalidKEKURI, "uri=%q", uri) + } +} diff --git a/internal/encryption/kek/vault.go b/internal/encryption/kek/vault.go new file mode 100644 index 000000000..fc3e7f391 --- /dev/null +++ b/internal/encryption/kek/vault.go @@ -0,0 +1,127 @@ +package kek + +import ( + "context" + "encoding/base64" + "os" + "strings" + "time" + + "github.com/cockroachdb/errors" + vaultapi "github.com/hashicorp/vault/api" +) + +type vaultLogicalClient interface { + WriteWithContext(context.Context, string, map[string]interface{}) (*vaultapi.Secret, error) +} + +// VaultTransitWrapper wraps DEKs with a Vault Transit symmetric key. +type VaultTransitWrapper struct { + logical vaultLogicalClient + mount string + keyName string + timeout time.Duration +} + +// NewVaultTransitWrapper uses the standard VAULT_ADDR, VAULT_TOKEN, TLS, and +// namespace environment configuration. target is /. +func NewVaultTransitWrapper(target string) (*VaultTransitWrapper, error) { + mount, keyName, err := parseVaultTarget(target) + if err != nil { + return nil, err + } + config := vaultapi.DefaultConfig() + if err := config.ReadEnvironment(); err != nil { + return nil, errors.Wrap(err, "kek: read Vault environment") + } + client, err := vaultapi.NewClient(config) + if err != nil { + return nil, errors.Wrap(err, "kek: create Vault client") + } + client.SetToken(os.Getenv("VAULT_TOKEN")) + return newVaultTransitWrapper(client.Logical(), mount, keyName), nil +} + +func newVaultTransitWrapper(logical vaultLogicalClient, mount, keyName string) *VaultTransitWrapper { + return &VaultTransitWrapper{logical: logical, mount: mount, keyName: keyName, timeout: providerRequestTimeout} +} + +func parseVaultTarget(target string) (string, string, error) { + parts := strings.Split(target, "/") + if len(parts) < 2 || parts[0] == "" { + return "", "", errors.Wrapf(ErrInvalidKEKURI, "invalid Vault Transit target %q", target) + } + for _, part := range parts { + if part == "" || part == "." || part == ".." { + return "", "", errors.Wrapf(ErrInvalidKEKURI, "invalid Vault Transit target %q", target) + } + } + return parts[0], strings.Join(parts[1:], "/"), nil +} + +// Wrap base64-encodes the binary DEK for Vault's JSON API and stores Vault's +// versioned ciphertext string as the wrapped sidecar bytes. +func (w *VaultTransitWrapper) Wrap(dek []byte) ([]byte, error) { + if w == nil || w.logical == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: Vault client is nil") + } + if err := validateDEK(dek); err != nil { + return nil, err + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + secret, err := w.logical.WriteWithContext(ctx, w.mount+"/encrypt/"+w.keyName, map[string]interface{}{ + "plaintext": base64.StdEncoding.EncodeToString(dek), + }) + if err != nil { + return nil, errors.Wrap(err, "kek: Vault Transit encrypt") + } + ciphertext, ok := vaultString(secret, "ciphertext") + if !ok || !strings.HasPrefix(ciphertext, "vault:v") { + return nil, errors.WithStack(ErrInvalidProviderResponse) + } + return []byte(ciphertext), nil +} + +// Unwrap asks Vault Transit to decrypt its versioned ciphertext and validates +// the returned base64 plaintext as a 32-byte DEK. +func (w *VaultTransitWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w == nil || w.logical == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: Vault client is nil") + } + if len(wrapped) == 0 || !strings.HasPrefix(string(wrapped), "vault:v") { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: invalid Vault ciphertext") + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + secret, err := w.logical.WriteWithContext(ctx, w.mount+"/decrypt/"+w.keyName, map[string]interface{}{ + "ciphertext": string(wrapped), + }) + if err != nil { + return nil, errors.Wrap(err, "kek: Vault Transit decrypt") + } + encoded, ok := vaultString(secret, "plaintext") + if !ok { + return nil, errors.WithStack(ErrInvalidProviderResponse) + } + plain, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: Vault plaintext is not base64") + } + if err := validateDEK(plain); err != nil { + clear(plain) + return nil, errors.Wrap(err, "kek: Vault plaintext") + } + return plain, nil +} + +func vaultString(secret *vaultapi.Secret, field string) (string, bool) { + if secret == nil || secret.Data == nil { + return "", false + } + value, ok := secret.Data[field].(string) + return value, ok && value != "" +} + +// Name returns the provider and Transit mount/key path. +func (w *VaultTransitWrapper) Name() string { return "vault-transit:" + w.mount + "/" + w.keyName } diff --git a/internal/encryption/kek/vault_test.go b/internal/encryption/kek/vault_test.go new file mode 100644 index 000000000..60f864822 --- /dev/null +++ b/internal/encryption/kek/vault_test.go @@ -0,0 +1,65 @@ +package kek + +import ( + "bytes" + "context" + "encoding/base64" + "testing" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +type fakeVaultLogical struct { + path string + data map[string]interface{} + dek []byte +} + +func (f *fakeVaultLogical) WriteWithContext(_ context.Context, path string, data map[string]interface{}) (*api.Secret, error) { + f.path = path + f.data = data + if _, ok := data["plaintext"]; ok { + return &api.Secret{Data: map[string]interface{}{"ciphertext": "vault:v1:ciphertext"}}, nil + } + return &api.Secret{Data: map[string]interface{}{"plaintext": base64.StdEncoding.EncodeToString(f.dek)}}, nil +} + +func TestVaultTransitWrapperRequestBinding(t *testing.T) { + dek := bytes.Repeat([]byte{0x62}, fileKEKSize) + logical := &fakeVaultLogical{dek: dek} + wrapper := newVaultTransitWrapper(logical, "transit", "service/key") + + wrapped, err := wrapper.Wrap(dek) + require.NoError(t, err) + require.Equal(t, []byte("vault:v1:ciphertext"), wrapped) + require.Equal(t, "transit/encrypt/service/key", logical.path) + require.Equal(t, base64.StdEncoding.EncodeToString(dek), logical.data["plaintext"]) + + plain, err := wrapper.Unwrap(wrapped) + require.NoError(t, err) + require.Equal(t, dek, plain) + require.Equal(t, "transit/decrypt/service/key", logical.path) + require.Equal(t, "vault:v1:ciphertext", logical.data["ciphertext"]) + require.Equal(t, "vault-transit:transit/service/key", wrapper.Name()) +} + +func TestParseVaultTarget(t *testing.T) { + mount, keyName, err := parseVaultTarget("transit/service/key") + require.NoError(t, err) + require.Equal(t, "transit", mount) + require.Equal(t, "service/key", keyName) + for _, target := range []string{"", "transit", "transit//key", "../key", "transit/../key"} { + _, _, err := parseVaultTarget(target) + require.ErrorIsf(t, err, ErrInvalidKEKURI, "target=%q", target) + } +} + +func TestVaultTransitWrapperRejectsMalformedResponses(t *testing.T) { + logical := &fakeVaultLogical{dek: []byte("short")} + wrapper := newVaultTransitWrapper(logical, "transit", "key") + _, err := wrapper.Unwrap([]byte("vault:v1:ciphertext")) + require.ErrorIs(t, err, ErrInvalidDEKLength) + _, err = wrapper.Unwrap([]byte("not-vault")) + require.ErrorIs(t, err, ErrInvalidProviderResponse) +} diff --git a/internal/encryption/startup.go b/internal/encryption/startup.go index 411aa42ef..6f8278bf4 100644 --- a/internal/encryption/startup.go +++ b/internal/encryption/startup.go @@ -28,7 +28,7 @@ type StartupConfig struct { // (downgrade prevention). EncryptionEnabled bool - // KEKConfigured is true iff --kekFile is non-empty. The KEK + // KEKConfigured is true iff one configured KEK source loaded. The KEK // itself is supplied via KEK below; KEKConfigured exists // independently so the helper can distinguish "operator did // not supply a KEK source" from "supplied but failed to load" @@ -203,7 +203,7 @@ func guardSidecarWithoutFlag(cfg StartupConfig, sidecarPresent bool) error { } // guardKEKRequired fires when the operator turned on -// --encryption-enabled without supplying --kekFile. A flag-on / +// --encryption-enabled without supplying a KEK source. A flag-on / // KEK-off node would refuse every mutator at the Stage 6B-2 RPC // gate AND HaltApply if a mutator ever did commit, neither of // which matches the operator's stated intent. Fail fast at startup. @@ -220,7 +220,7 @@ func guardKEKRequired(cfg StartupConfig) error { // guardKEKMatchesSidecar attempts to KEK-unwrap every wrapped DEK // in the sidecar. A single failure fires ErrKEKMismatch with the // offending key_id annotated — the classic operator error here is -// "wrong --kekFile points at a key from a different cluster" and +// "the configured KEK belongs to a different cluster" and // the key_id identifies which DEK could not be unwrapped, which is // almost always enough to root-cause. // diff --git a/main.go b/main.go index 9274bd07b..dbbf19027 100644 --- a/main.go +++ b/main.go @@ -192,22 +192,22 @@ var ( // // Mutating RPCs (BootstrapEncryption / RotateDEK / // RegisterEncryptionWriter) are gated by Stage 6B-2 on the - // AND of --encryption-enabled and --kekFile being non-empty. + // AND of --encryption-enabled and a successfully loaded KEK source. // Setting --encryptionSidecarPath ALONE no longer enables // mutators; the operator must explicitly opt in to encryption // AND supply a KEK source. With either gate condition false, // registerEncryptionAdminServer omits the Proposer + LeaderView // options and every mutator short-circuits at the gRPC boundary // with FailedPrecondition before any Raft proposal is created. - encryptionSidecarPath = flag.String("encryptionSidecarPath", "", "§5.1 keys.json path; enables read-only EncryptionAdmin capability probing. Mutating RPCs (Bootstrap / RotateDEK / RegisterEncryptionWriter) are additionally gated on this flag being non-empty AND --encryption-enabled AND --kekFile being non-empty (all three required so the applier's WithKEK + WithKeystore + WithSidecarPath options are all wired before mutators can commit).") + encryptionSidecarPath = flag.String("encryptionSidecarPath", "", "§5.1 keys.json path; enables read-only EncryptionAdmin capability probing. Mutating RPCs additionally require --encryption-enabled and one loaded KEK source.") // Stage 6B-2: cluster-wide encryption opt-in flag. The mutating // EncryptionAdmin RPCs (BootstrapEncryption, RotateDEK, // RegisterEncryptionWriter) become reachable only when this - // flag is set AND --kekFile points at a valid KEK source. + // flag is set AND exactly one configured KEK source loads successfully. // Default off; pre-Stage-6 clusters and operators who have // not yet committed to encryption are unaffected. - encryptionEnabled = flag.Bool("encryption-enabled", false, "§6.5 opt-in to encryption-mutating EncryptionAdmin RPCs. Requires --kekFile to be set; without that, mutators still refuse with FailedPrecondition. Default off.") + encryptionEnabled = flag.Bool("encryption-enabled", false, "§6.5 opt-in to encryption-mutating EncryptionAdmin RPCs. Requires exactly one KEK source from --kekFile, --kekUri, or ELASTICKV_KEK_BASE64. Default off.") // Stage 6F: operator-requested DEK rotation at boot. The flag is // intentionally a request, not a guarantee: only the leader of the @@ -216,15 +216,14 @@ var ( // leadership during this process uptime. encryptionRotateOnStartup = flag.Bool("encryption-rotate-on-startup", false, "§6.5 request a one-shot DEK rotation after this node becomes leader of the default Raft group. Safe for rolling restarts: followers keep the request in memory and only fire if they acquire leadership during this process uptime.") - // Stage 6B-2: KEK source. The KEK never appears in elastickv's + // KEK sources. The KEK never appears in elastickv's // data dir; it is held externally and exercised only at process - // boot and at DEK bootstrap/rotation per §5.1. Stage 6B-2 ships - // only the file-backed wrapper (kek.FileWrapper); KMS providers - // (--kekUri) land in Stage 9. Empty disables KEK loading; the + // boot and at DEK bootstrap/rotation per §5.1. Empty disables KEK loading; the // applier's ApplyBootstrap and ApplyRotation paths then return // ErrKEKNotConfigured at apply time, which is masked at the // RPC boundary by the mutator gate documented above. kekFile = flag.String("kekFile", "", "§5.1 KEK file path (32 raw bytes, owner-only mode). When set, the file-backed kek.Wrapper is constructed at startup and threaded into the §6.3 EncryptionApplier so ApplyBootstrap and ApplyRotation can KEK-unwrap.") + kekURI = flag.String("kekUri", "", "§5.1 remote KEK URI: aws-kms://, gcp-kms://, or vault-transit:///. Mutually exclusive with --kekFile and ELASTICKV_KEK_BASE64.") // Key visualizer sampler flags. The sampler runs entirely in-memory // on each node, feeds AdminServer.GetKeyVizMatrix, and is disabled @@ -438,7 +437,7 @@ func run() error { // visible to every shard's storage cipher. // // Both are nil-safe in the applier path: WithKEK / WithKeystore - // are only attached to the applier when --kekFile is non-empty + // are only attached to the applier when a KEK source is loaded // (else the applier stays in the Stage 6A posture where // ApplyBootstrap / ApplyRotation return ErrKEKNotConfigured). kekWrapper, err := loadKEKAfterPreNonceStartupGuards(cfg) @@ -600,6 +599,7 @@ func run() error { redisApplyObserver: redisApplyObserver, cleanup: &cleanup, encWiring: encWiring, + kekConfigured: kekWrapper != nil, keyvizSampler: sampler, encryptionConfChangeInterceptor: encryptionConfChangeInterceptor, }); err != nil { @@ -1585,7 +1585,7 @@ func loadKEKAfterPreNonceStartupGuards(cfg runtimeConfig) (kek.Wrapper, error) { return loadKEKAndRunStartupGuards() } -// loadKEKAndRunStartupGuards loads the file-backed KEK wrapper and +// loadKEKAndRunStartupGuards loads the configured KEK wrapper and // runs the §9.1 startup-refusal guards (Stage 6C-1) BEFORE // buildShardGroups constructs any Raft engine or storage state. The // two operations are paired in a single helper because the guards @@ -1614,7 +1614,7 @@ func loadKEKAndRunStartupGuards() (kek.Wrapper, error) { } if err := encryption.CheckStartupGuards(encryption.StartupConfig{ EncryptionEnabled: *encryptionEnabled, - KEKConfigured: *kekFile != "", + KEKConfigured: kekWrapper != nil, KEK: kekWrapper, SidecarPath: *encryptionSidecarPath, }); err != nil { @@ -1623,20 +1623,17 @@ func loadKEKAndRunStartupGuards() (kek.Wrapper, error) { return kekWrapper, nil } -// loadKEKWrapperFromFlag constructs the file-backed KEK wrapper -// from the --kekFile flag, returning nil if the flag is empty. +// loadKEKWrapperFromFlag constructs the configured KEK wrapper from the +// mutually-exclusive file, URI, or environment source. // Returns the kek.Wrapper interface rather than the concrete // *kek.FileWrapper so the call site (buildShardGroups → applier) // stays decoupled from the file-mode provider — Stage 9 KMS // providers (AWS KMS, GCP KMS, Vault) will satisfy the same // interface and slot in without rewriting the dispatch site. func loadKEKWrapperFromFlag() (kek.Wrapper, error) { - if *kekFile == "" { - return nil, nil - } - w, err := kek.NewFileWrapper(*kekFile) + w, err := kek.NewWrapperFromSources(context.Background(), *kekFile, *kekURI) if err != nil { - return nil, errors.Wrapf(err, "failed to load KEK from %s", *kekFile) + return nil, errors.Wrap(err, "failed to load KEK source") } return w, nil } @@ -1790,6 +1787,7 @@ type serversInput struct { metricsRegistry *monitoring.Registry cfg runtimeConfig encWiring encryptionWriteWiring + kekConfigured bool redisApplyObserver *adapter.RedisApplyObserver cleanup *internalutil.CleanupStack // keyvizSampler is the in-memory key visualizer sampler, or nil @@ -1873,6 +1871,7 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, pubsubRelay: adapter.NewRedisPubSubRelay(), readTracker: in.readTracker, encWiring: in.encWiring, + kekConfigured: in.kekConfigured, redisApplyObserver: in.redisApplyObserver, dynamoAddress: *dynamoAddr, defaultGroup: in.cfg.defaultGroup, @@ -2629,6 +2628,7 @@ func startRaftServers( forwardDeps adminForwardServerDeps, confChangeInterceptor internalraftadmin.MembershipChangeInterceptor, encWiring encryptionWriteWiring, + kekConfigured bool, defaultGroup uint64, s3BlobObserver adapter.S3BlobOffloadObserver, s3BlobPushBlocked func() bool, @@ -2638,7 +2638,7 @@ func startRaftServers( // options appended below. Sized as a constant so the magic-number // linter does not complain. const extraOptsCap = 2 - enableMutators := encryptionMutatorsEnabled() + enableMutators := encryptionMutatorsEnabled(kekConfigured) encryptionCapabilityFanout := buildEncryptionCapabilityFanout(ctx, eg, runtimes, enableMutators) startStorageEnvelopeV2CapabilityMonitor(ctx, eg, encryptionCapabilityFanout, encWiring) adminWriterRegistry := writerRegistryForEncryptionAdmin(runtimes, defaultGroup) @@ -3042,6 +3042,7 @@ type runtimeServerRunner struct { readTracker *kv.ActiveTimestampTracker redisApplyObserver *adapter.RedisApplyObserver encWiring encryptionWriteWiring + kekConfigured bool dynamoAddress string defaultGroup uint64 leaderDynamo map[string]string @@ -3162,6 +3163,7 @@ func (r *runtimeServerRunner) startRaftTransport() error { forwardDeps, r.encryptionConfChangeInterceptor, r.encWiring, + r.kekConfigured, r.defaultGroup, r.metricsRegistry.S3BlobOffloadObserver(), s3BlobPushBlocked, diff --git a/main_encryption_admin.go b/main_encryption_admin.go index 9243082c4..f2c98a9b6 100644 --- a/main_encryption_admin.go +++ b/main_encryption_admin.go @@ -14,7 +14,7 @@ import ( // readback. Returns true iff THIS NODE has all three of: // // - --encryption-enabled (explicit operator opt-in) -// - --kekFile non-empty (KEK source loaded so ApplyBootstrap +// - a KEK source loaded (file, URI, or test/CI environment source, so ApplyBootstrap // / ApplyRotation can KEK-unwrap) // - --encryptionSidecarPath non-empty (so the applier can // crash-durably persist Active.{Storage,Raft} + keys[]) @@ -62,8 +62,8 @@ import ( // Kept in this file (not main.go) so the flag-driven gate logic // is colocated with the registerEncryptionAdminServer helper // that consumes it. -func encryptionMutatorsEnabled() bool { - return *encryptionEnabled && *kekFile != "" && *encryptionSidecarPath != "" +func encryptionMutatorsEnabled(kekConfigured bool) bool { + return *encryptionEnabled && kekConfigured && *encryptionSidecarPath != "" } func encryptionAdminMutatorAuthority(enabled bool, groupID, defaultGroup uint64) bool { @@ -123,7 +123,7 @@ type encryptionAdminEngine interface { // // Stage 6B-2: the mutator wiring (Proposer + LeaderView) is now // gated on the supplied enableMutators boolean, which the caller -// in main.go computes as (--encryption-enabled AND --kekFile +// in main.go computes as (--encryption-enabled AND loaded KEK // non-empty AND engine non-nil). When enableMutators is false, // Proposer + LeaderView stay unwired and EncryptionAdminServer's // BootstrapEncryption / RotateDEK / RegisterEncryptionWriter @@ -142,7 +142,7 @@ type encryptionAdminEngine interface { // means the cluster has explicitly chosen NOT to participate // in the §7.1 rollout, so mutator RPCs MUST refuse even on // a fully-keyed binary. -// - --kekFile being non-empty means a KEK source is loaded; +// - kekConfigured means a file, URI, or environment KEK source is loaded; // without it, a mutator that committed would land in the // applier with no KEK and return ErrKEKNotConfigured from // the §6.3 HaltApply path — that is fail-closed but it diff --git a/main_encryption_kek_source_test.go b/main_encryption_kek_source_test.go new file mode 100644 index 000000000..f0ef447ba --- /dev/null +++ b/main_encryption_kek_source_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/fsmwire" + "github.com/bootjp/elastickv/internal/encryption/kek" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestEncryptionMutatorsEnabledUsesLoadedKEKState(t *testing.T) { + previousEnabled := *encryptionEnabled + previousSidecar := *encryptionSidecarPath + t.Cleanup(func() { + *encryptionEnabled = previousEnabled + *encryptionSidecarPath = previousSidecar + }) + + *encryptionEnabled = true + *encryptionSidecarPath = "/data/encryption/keys.json" + require.True(t, encryptionMutatorsEnabled(true)) + require.False(t, encryptionMutatorsEnabled(false)) + *encryptionEnabled = false + require.False(t, encryptionMutatorsEnabled(true)) + *encryptionEnabled = true + *encryptionSidecarPath = "" + require.False(t, encryptionMutatorsEnabled(true)) +} + +func TestEnvKEKBootstrapCutoverSnapshotRestore(t *testing.T) { + staticKEK := bytes.Repeat([]byte{0x71}, encryption.KeySize) + t.Setenv(kek.EnvVar, base64.StdEncoding.EncodeToString(staticKEK)) + wrapper, err := kek.NewWrapperFromSources(context.Background(), "", "") + require.NoError(t, err) + + storageDEK := bytes.Repeat([]byte{0x72}, encryption.KeySize) + raftDEK := bytes.Repeat([]byte{0x73}, encryption.KeySize) + wrappedStorage, err := wrapper.Wrap(storageDEK) + require.NoError(t, err) + wrappedRaft, err := wrapper.Wrap(raftDEK) + require.NoError(t, err) + + dir := t.TempDir() + sidecarPath := dir + "/keys.json" + keystore := encryption.NewKeystore() + wiring, err := buildEncryptionWriteWiring(true, "n1", sidecarPath, wrapper, keystore, []groupSpec{{id: 1}}) + require.NoError(t, err) + source, err := store.NewPebbleStore(dir+"/source", wiring.pebbleOptions()...) + require.NoError(t, err) + t.Cleanup(func() { _ = source.Close() }) + registry, err := store.WriterRegistryFor(source) + require.NoError(t, err) + applier, err := encryption.NewApplier( + registry, + encryption.WithKEK(wrapper), + encryption.WithKeystore(keystore), + encryption.WithSidecarPath(sidecarPath), + encryption.WithStateCache(wiring.cache), + ) + require.NoError(t, err) + + require.NoError(t, applier.ApplyBootstrap(1, fsmwire.BootstrapPayload{ + StorageDEKID: e2eStorageDEKID, + WrappedStorage: wrappedStorage, + RaftDEKID: e2eRaftDEKID, + WrappedRaft: wrappedRaft, + BatchRegistry: []fsmwire.RegistrationPayload{ + {DEKID: e2eStorageDEKID, FullNodeID: e2eNodeID, LocalEpoch: 0}, + }, + })) + require.NoError(t, applier.ApplyRotation(2, fsmwire.RotationPayload{ + SubTag: fsmwire.RotateSubEnableStorageEnvelope, + DEKID: e2eStorageDEKID, + Purpose: fsmwire.PurposeStorage, + Wrapped: []byte{}, + ProposerRegistration: fsmwire.RegistrationPayload{DEKID: e2eStorageDEKID, FullNodeID: e2eNodeID, LocalEpoch: 1}, + })) + wiring.cache.MarkRegistered(e2eStorageDEKID) + require.NoError(t, source.PutAt(context.Background(), []byte("env-kek"), []byte("secret"), 100, 0)) + + snapshot, err := source.Snapshot() + require.NoError(t, err) + var snapshotBytes bytes.Buffer + _, err = snapshot.WriteTo(&snapshotBytes) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + + target, err := store.NewPebbleStore(dir+"/target", wiring.pebbleOptions()...) + require.NoError(t, err) + t.Cleanup(func() { _ = target.Close() }) + require.NoError(t, target.Restore(bytes.NewReader(snapshotBytes.Bytes()))) + got, err := target.GetAt(context.Background(), []byte("env-kek"), 100) + require.NoError(t, err) + require.Equal(t, []byte("secret"), got) +} From df1c19fe53ef1b875c7d47f6812a6c9fe50b6818 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 21:27:03 +0900 Subject: [PATCH 12/16] Reuse GCP KMS checksum table --- internal/encryption/kek/gcp_kms.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/encryption/kek/gcp_kms.go b/internal/encryption/kek/gcp_kms.go index 09c85af6d..418df7cdc 100644 --- a/internal/encryption/kek/gcp_kms.go +++ b/internal/encryption/kek/gcp_kms.go @@ -12,7 +12,10 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" ) -var gcpKMSAAD = []byte("elastickv-dek-wrap-v1") +var ( + gcpKMSAAD = []byte("elastickv-dek-wrap-v1") + crc32cCastagnoli = crc32.MakeTable(crc32.Castagnoli) +) type gcpKMSClient interface { Encrypt(context.Context, *kmspb.EncryptRequest) (*kmspb.EncryptResponse, error) @@ -65,7 +68,7 @@ func validGCPKeyName(name string) bool { } func crc32c(data []byte) *wrapperspb.Int64Value { - return wrapperspb.Int64(int64(crc32.Checksum(data, crc32.MakeTable(crc32.Castagnoli)))) + return wrapperspb.Int64(int64(crc32.Checksum(data, crc32cCastagnoli))) } func checksumMatches(data []byte, checksum *wrapperspb.Int64Value) bool { From 017ba51eca88dec3daec41304b0102e6796d0ccd Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 18:06:19 +0900 Subject: [PATCH 13/16] encryption: harden KEK provider startup --- ...6_04_29_partial_data_at_rest_encryption.md | 8 +- ...2026_07_18_implemented_9b_kek_providers.md | 17 +++- internal/encryption/kek/aws_kms.go | 2 +- internal/encryption/kek/aws_kms_test.go | 13 ++- internal/encryption/kek/provider.go | 33 ++++++++ internal/encryption/kek/provider_test.go | 82 +++++++++++++++++++ internal/encryption/kek/source_test.go | 6 +- internal/encryption/kek/vault.go | 19 ++++- internal/encryption/kek/vault_test.go | 28 +++++-- main.go | 13 +++ main_encryption_kek_source_test.go | 29 +++++++ 11 files changed, 231 insertions(+), 19 deletions(-) create mode 100644 internal/encryption/kek/provider_test.go diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index ea229f11c..21fadc4f4 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -1020,7 +1020,8 @@ Two-tier hierarchy: - **KEK (Key Encryption Key).** Held outside the cluster, never on the cluster's disks. Sources, in order of preference: - 1. AWS KMS: `--kekUri=aws-kms://arn:aws:kms:...`. The KMS key never + 1. AWS KMS: `--kekUri=aws-kms://arn:aws:kms:...:key/...` with an + immutable key ARN (mutable alias ARNs are rejected). The KMS key never leaves AWS; we call `Encrypt` / `Decrypt` to wrap/unwrap DEKs. 2. GCP KMS: `--kekUri=gcp-kms://projects/.../keys/...`. Same shape. 3. HashiCorp Vault Transit: `--kekUri=vault-transit://...`. @@ -1032,7 +1033,10 @@ Two-tier hierarchy: inspection); supported only for tests and CI. No default. If `--encryption-enabled` is set without a KEK source, - the process refuses to start. + the process refuses to start. Before encryption mutators are exposed, every + configured provider must also complete a random 32-byte DEK wrap/unwrap + preflight; this catches credentials, reachability, permission, key-binding, + and malformed-response failures before a Raft entry can commit. - **DEK (Data Encryption Key).** 32-byte AES key. Two DEKs are issued in v1: `dek_storage` (used by §4.1) and `dek_raft` diff --git a/docs/design/2026_07_18_implemented_9b_kek_providers.md b/docs/design/2026_07_18_implemented_9b_kek_providers.md index d8bae8392..5cd2b0451 100644 --- a/docs/design/2026_07_18_implemented_9b_kek_providers.md +++ b/docs/design/2026_07_18_implemented_9b_kek_providers.md @@ -8,7 +8,8 @@ Date: 2026-07-18 Stage 9B completes the §5.1 KEK source matrix: -- `--kekUri=aws-kms://` uses AWS KMS `Encrypt` / `Decrypt`, derives +- `--kekUri=aws-kms://` uses an immutable AWS KMS `key/` ARN with + `Encrypt` / `Decrypt`, derives the client region from the ARN, and binds every request to the fixed `elastickv-purpose=dek-wrap-v1` encryption context. - `--kekUri=gcp-kms://` uses Google Cloud KMS with fixed @@ -16,7 +17,7 @@ Stage 9B completes the §5.1 KEK source matrix: wrapped DEK or plaintext DEK is accepted. - `--kekUri=vault-transit:///` uses Vault Transit through standard `VAULT_*` client configuration. Binary DEKs are base64 encoded for the API; - only versioned `vault:v...` ciphertexts are accepted. + only `vault:v:` ciphertexts are accepted. - `ELASTICKV_KEK_BASE64` supplies a 32-byte test/CI-only static KEK. The variable is unset immediately after the decode attempt, including malformed input. It is also unset when source ambiguity is rejected. @@ -33,6 +34,13 @@ text. The same loaded-state boolean is threaded to the EncryptionAdmin mutator gate, so URI and environment providers can bootstrap/rotate while a failed or absent provider cannot expose mutating RPCs. +Before that gate can open, startup performs a real random 32-byte DEK +wrap/unwrap round trip and compares the result in constant time. This proves +provider reachability, credentials, encrypt/decrypt permission, key binding, +and response shape on a fresh data directory where no sidecar DEK exists to +exercise the provider. A failed preflight refuses process start before Raft or +the mutating RPC surface is available. + Remote wrappers use a 30-second operation deadline and reject nil, empty, integrity-invalid, or non-32-byte provider responses before sidecar state can be updated. Provider clients are safe for concurrent use and are hidden behind @@ -48,8 +56,9 @@ Verification includes: - fake-client unit tests for AWS encryption context, GCP AAD/CRC32C, and Vault binary request/response encoding; -- malformed provider response, wrong DEK length, invalid URI, source conflict, - and environment-unset tests; +- malformed provider response, strict Vault ciphertext, immutable AWS key ARN, + wrong DEK length, invalid URI, source conflict, environment-unset, and + startup wrap/unwrap preflight tests; - a production-wiring integration test that loads the environment KEK through the source selector, bootstraps both DEKs, enables the storage envelope, writes encrypted data, snapshots it, restores it into an encrypted Pebble diff --git a/internal/encryption/kek/aws_kms.go b/internal/encryption/kek/aws_kms.go index f4400be82..3f7beed32 100644 --- a/internal/encryption/kek/aws_kms.go +++ b/internal/encryption/kek/aws_kms.go @@ -32,7 +32,7 @@ type AWSKMSWrapper struct { func NewAWSKMSWrapper(ctx context.Context, keyARN string) (*AWSKMSWrapper, error) { parsed, err := arn.Parse(keyARN) if err != nil || parsed.Service != "kms" || parsed.Region == "" || parsed.AccountID == "" || - (!strings.HasPrefix(parsed.Resource, "key/") && !strings.HasPrefix(parsed.Resource, "alias/")) { + !strings.HasPrefix(parsed.Resource, "key/") || strings.TrimPrefix(parsed.Resource, "key/") == "" { return nil, errors.Wrapf(ErrInvalidKEKURI, "invalid AWS KMS key ARN %q", keyARN) } cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(parsed.Region)) diff --git a/internal/encryption/kek/aws_kms_test.go b/internal/encryption/kek/aws_kms_test.go index 97fb30767..f8bb012f7 100644 --- a/internal/encryption/kek/aws_kms_test.go +++ b/internal/encryption/kek/aws_kms_test.go @@ -82,6 +82,15 @@ func TestAWSKMSWrapperRejectsProviderFailures(t *testing.T) { } func TestNewAWSKMSWrapperRejectsInvalidARNBeforeConfigLoad(t *testing.T) { - _, err := NewAWSKMSWrapper(context.Background(), "not-an-arn") - require.ErrorIs(t, err, ErrInvalidKEKURI) + for _, keyARN := range []string{ + "not-an-arn", + "arn:aws:kms:us-east-1:123456789012:key/", + "arn:aws:kms:us-east-1:123456789012:alias/current", + "arn:aws:kms:us-east-1:123456789012:alias/", + } { + t.Run(keyARN, func(t *testing.T) { + _, err := NewAWSKMSWrapper(context.Background(), keyARN) + require.ErrorIs(t, err, ErrInvalidKEKURI) + }) + } } diff --git a/internal/encryption/kek/provider.go b/internal/encryption/kek/provider.go index e2350fbad..5991bc7ca 100644 --- a/internal/encryption/kek/provider.go +++ b/internal/encryption/kek/provider.go @@ -2,6 +2,8 @@ package kek import ( "context" + "crypto/rand" + "crypto/subtle" "time" "github.com/cockroachdb/errors" @@ -16,8 +18,39 @@ var ( // ErrInvalidProviderResponse rejects an empty or integrity-invalid response // before it can be persisted in the encryption sidecar. ErrInvalidProviderResponse = errors.New("kek: invalid provider response") + // ErrKEKPreflightFailed prevents mutators from opening when the configured + // provider cannot complete a real wrap/unwrap round trip. + ErrKEKPreflightFailed = errors.New("kek: provider preflight failed") ) +// VerifyWrapper proves credentials, provider reachability, encrypt/decrypt +// permissions, and key binding before any encryption mutator can commit a +// wrapped DEK. Provider constructors alone only validate local configuration. +func VerifyWrapper(wrapper Wrapper) error { + if wrapper == nil { + return errors.Wrap(ErrKEKPreflightFailed, "wrapper is nil") + } + dek := make([]byte, fileKEKSize) + defer clear(dek) + if _, err := rand.Read(dek); err != nil { + return errors.Wrapf(ErrKEKPreflightFailed, "generate probe DEK: %v", err) + } + wrapped, err := wrapper.Wrap(dek) + if err != nil { + return errors.Wrapf(ErrKEKPreflightFailed, "wrap with %s: %v", wrapper.Name(), err) + } + defer clear(wrapped) + plain, err := wrapper.Unwrap(wrapped) + if err != nil { + return errors.Wrapf(ErrKEKPreflightFailed, "unwrap with %s: %v", wrapper.Name(), err) + } + defer clear(plain) + if len(plain) != len(dek) || subtle.ConstantTimeCompare(plain, dek) != 1 { + return errors.Wrapf(ErrKEKPreflightFailed, "%s returned a different DEK", wrapper.Name()) + } + return nil +} + func validateDEK(dek []byte) error { if len(dek) != fileKEKSize { return errors.Wrapf(ErrInvalidDEKLength, "got %d bytes", len(dek)) diff --git a/internal/encryption/kek/provider_test.go b/internal/encryption/kek/provider_test.go new file mode 100644 index 000000000..af913df00 --- /dev/null +++ b/internal/encryption/kek/provider_test.go @@ -0,0 +1,82 @@ +package kek + +import ( + "bytes" + "testing" + + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +type preflightWrapper struct { + wrapErr error + unwrapErr error + mismatch bool +} + +func (w *preflightWrapper) Wrap(dek []byte) ([]byte, error) { + if w.wrapErr != nil { + return nil, w.wrapErr + } + return append([]byte(nil), dek...), nil +} + +func (w *preflightWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w.unwrapErr != nil { + return nil, w.unwrapErr + } + plain := append([]byte(nil), wrapped...) + if w.mismatch { + plain[0] ^= 0xFF + } + return plain, nil +} + +func (*preflightWrapper) Name() string { return "test" } + +func TestVerifyWrapper(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + wrapper Wrapper + wantErr bool + }{ + {name: "round trip", wrapper: &preflightWrapper{}}, + {name: "nil wrapper", wrapper: nil, wantErr: true}, + {name: "wrap failure", wrapper: &preflightWrapper{wrapErr: errors.New("denied")}, wantErr: true}, + {name: "unwrap failure", wrapper: &preflightWrapper{unwrapErr: errors.New("denied")}, wantErr: true}, + {name: "mismatched plaintext", wrapper: &preflightWrapper{mismatch: true}, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := VerifyWrapper(tc.wrapper) + if tc.wantErr { + require.ErrorIs(t, err, ErrKEKPreflightFailed) + return + } + require.NoError(t, err) + }) + } +} + +func TestVerifyWrapperUsesRandomDEKLength(t *testing.T) { + t.Parallel() + wrapper := &capturingPreflightWrapper{} + require.NoError(t, VerifyWrapper(wrapper)) + require.Len(t, wrapper.seen, fileKEKSize) + require.False(t, bytes.Equal(wrapper.seen, make([]byte, fileKEKSize))) +} + +type capturingPreflightWrapper struct { + seen []byte +} + +func (w *capturingPreflightWrapper) Wrap(dek []byte) ([]byte, error) { + w.seen = append([]byte(nil), dek...) + return append([]byte(nil), dek...), nil +} + +func (*capturingPreflightWrapper) Unwrap(wrapped []byte) ([]byte, error) { + return append([]byte(nil), wrapped...), nil +} + +func (*capturingPreflightWrapper) Name() string { return "capture" } diff --git a/internal/encryption/kek/source_test.go b/internal/encryption/kek/source_test.go index b6aa17505..4bb834f7d 100644 --- a/internal/encryption/kek/source_test.go +++ b/internal/encryption/kek/source_test.go @@ -51,7 +51,9 @@ func TestNewURIWrapperRejectsInvalidTargetsWithoutNetwork(t *testing.T) { "gcp-kms://projects/incomplete", "vault-transit://transit", } { - _, err := newURIWrapper(context.Background(), uri) - require.ErrorIsf(t, err, ErrInvalidKEKURI, "uri=%q", uri) + t.Run(uri, func(t *testing.T) { + _, err := newURIWrapper(context.Background(), uri) + require.ErrorIsf(t, err, ErrInvalidKEKURI, "uri=%q", uri) + }) } } diff --git a/internal/encryption/kek/vault.go b/internal/encryption/kek/vault.go index fc3e7f391..d9790a90e 100644 --- a/internal/encryption/kek/vault.go +++ b/internal/encryption/kek/vault.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "os" + "strconv" "strings" "time" @@ -77,7 +78,7 @@ func (w *VaultTransitWrapper) Wrap(dek []byte) ([]byte, error) { return nil, errors.Wrap(err, "kek: Vault Transit encrypt") } ciphertext, ok := vaultString(secret, "ciphertext") - if !ok || !strings.HasPrefix(ciphertext, "vault:v") { + if !ok || !validVaultCiphertext(ciphertext) { return nil, errors.WithStack(ErrInvalidProviderResponse) } return []byte(ciphertext), nil @@ -89,7 +90,7 @@ func (w *VaultTransitWrapper) Unwrap(wrapped []byte) ([]byte, error) { if w == nil || w.logical == nil { return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: Vault client is nil") } - if len(wrapped) == 0 || !strings.HasPrefix(string(wrapped), "vault:v") { + if !validVaultCiphertext(string(wrapped)) { return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: invalid Vault ciphertext") } ctx, cancel := requestContext(w.timeout) @@ -115,6 +116,20 @@ func (w *VaultTransitWrapper) Unwrap(wrapped []byte) ([]byte, error) { return plain, nil } +func validVaultCiphertext(ciphertext string) bool { + const prefix = "vault:v" + if !strings.HasPrefix(ciphertext, prefix) { + return false + } + rest := strings.TrimPrefix(ciphertext, prefix) + separator := strings.IndexByte(rest, ':') + if separator <= 0 || separator == len(rest)-1 { + return false + } + version, err := strconv.ParseUint(rest[:separator], 10, 64) + return err == nil && version > 0 +} + func vaultString(secret *vaultapi.Secret, field string) (string, bool) { if secret == nil || secret.Data == nil { return "", false diff --git a/internal/encryption/kek/vault_test.go b/internal/encryption/kek/vault_test.go index 60f864822..bb09ccff4 100644 --- a/internal/encryption/kek/vault_test.go +++ b/internal/encryption/kek/vault_test.go @@ -11,16 +11,21 @@ import ( ) type fakeVaultLogical struct { - path string - data map[string]interface{} - dek []byte + path string + data map[string]interface{} + dek []byte + ciphertext string } func (f *fakeVaultLogical) WriteWithContext(_ context.Context, path string, data map[string]interface{}) (*api.Secret, error) { f.path = path f.data = data if _, ok := data["plaintext"]; ok { - return &api.Secret{Data: map[string]interface{}{"ciphertext": "vault:v1:ciphertext"}}, nil + ciphertext := f.ciphertext + if ciphertext == "" { + ciphertext = "vault:v1:ciphertext" + } + return &api.Secret{Data: map[string]interface{}{"ciphertext": ciphertext}}, nil } return &api.Secret{Data: map[string]interface{}{"plaintext": base64.StdEncoding.EncodeToString(f.dek)}}, nil } @@ -50,8 +55,10 @@ func TestParseVaultTarget(t *testing.T) { require.Equal(t, "transit", mount) require.Equal(t, "service/key", keyName) for _, target := range []string{"", "transit", "transit//key", "../key", "transit/../key"} { - _, _, err := parseVaultTarget(target) - require.ErrorIsf(t, err, ErrInvalidKEKURI, "target=%q", target) + t.Run(target, func(t *testing.T) { + _, _, err := parseVaultTarget(target) + require.ErrorIsf(t, err, ErrInvalidKEKURI, "target=%q", target) + }) } } @@ -62,4 +69,13 @@ func TestVaultTransitWrapperRejectsMalformedResponses(t *testing.T) { require.ErrorIs(t, err, ErrInvalidDEKLength) _, err = wrapper.Unwrap([]byte("not-vault")) require.ErrorIs(t, err, ErrInvalidProviderResponse) + for _, ciphertext := range []string{"vault:v", "vault:v1:", "vault:v0:data", "vault:vx:data"} { + t.Run(ciphertext, func(t *testing.T) { + invalid := newVaultTransitWrapper(&fakeVaultLogical{dek: bytes.Repeat([]byte{1}, fileKEKSize), ciphertext: ciphertext}, "transit", "key") + _, wrapErr := invalid.Wrap(bytes.Repeat([]byte{2}, fileKEKSize)) + require.ErrorIs(t, wrapErr, ErrInvalidProviderResponse) + _, unwrapErr := invalid.Unwrap([]byte(ciphertext)) + require.ErrorIs(t, unwrapErr, ErrInvalidProviderResponse) + }) + } } diff --git a/main.go b/main.go index dbbf19027..96f7bdaaf 100644 --- a/main.go +++ b/main.go @@ -1620,9 +1620,22 @@ func loadKEKAndRunStartupGuards() (kek.Wrapper, error) { }); err != nil { return nil, errors.Wrap(err, "encryption startup guards refused process start") } + if err := verifyKEKBeforeMutators(kekWrapper); err != nil { + return nil, err + } return kekWrapper, nil } +func verifyKEKBeforeMutators(wrapper kek.Wrapper) error { + if !*encryptionEnabled || *encryptionSidecarPath == "" || wrapper == nil { + return nil + } + if err := kek.VerifyWrapper(wrapper); err != nil { + return errors.Wrap(err, "encryption KEK preflight refused process start") + } + return nil +} + // loadKEKWrapperFromFlag constructs the configured KEK wrapper from the // mutually-exclusive file, URI, or environment source. // Returns the kek.Wrapper interface rather than the concrete diff --git a/main_encryption_kek_source_test.go b/main_encryption_kek_source_test.go index f0ef447ba..dead8a56b 100644 --- a/main_encryption_kek_source_test.go +++ b/main_encryption_kek_source_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/base64" + "errors" "testing" "github.com/bootjp/elastickv/internal/encryption" @@ -32,11 +33,39 @@ func TestEncryptionMutatorsEnabledUsesLoadedKEKState(t *testing.T) { require.False(t, encryptionMutatorsEnabled(true)) } +type failingStartupKEK struct{} + +func (failingStartupKEK) Wrap([]byte) ([]byte, error) { return nil, errors.New("provider unavailable") } +func (failingStartupKEK) Unwrap([]byte) ([]byte, error) { + return nil, errors.New("provider unavailable") +} +func (failingStartupKEK) Name() string { return "failing" } + +func TestVerifyKEKBeforeMutators(t *testing.T) { + previousEnabled := *encryptionEnabled + previousSidecar := *encryptionSidecarPath + t.Cleanup(func() { + *encryptionEnabled = previousEnabled + *encryptionSidecarPath = previousSidecar + }) + + *encryptionEnabled = true + *encryptionSidecarPath = "/data/encryption/keys.json" + require.ErrorIs(t, verifyKEKBeforeMutators(failingStartupKEK{}), kek.ErrKEKPreflightFailed) + + *encryptionEnabled = false + require.NoError(t, verifyKEKBeforeMutators(failingStartupKEK{})) + *encryptionEnabled = true + *encryptionSidecarPath = "" + require.NoError(t, verifyKEKBeforeMutators(failingStartupKEK{})) +} + func TestEnvKEKBootstrapCutoverSnapshotRestore(t *testing.T) { staticKEK := bytes.Repeat([]byte{0x71}, encryption.KeySize) t.Setenv(kek.EnvVar, base64.StdEncoding.EncodeToString(staticKEK)) wrapper, err := kek.NewWrapperFromSources(context.Background(), "", "") require.NoError(t, err) + require.NoError(t, kek.VerifyWrapper(wrapper)) storageDEK := bytes.Repeat([]byte{0x72}, encryption.KeySize) raftDEK := bytes.Repeat([]byte{0x73}, encryption.KeySize) From 46b533e2818e6afb8046391bb6db70812a67e9fb Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 19:37:05 +0900 Subject: [PATCH 14/16] adapter: bound Lua cache test cost --- adapter/redis_lua_context.go | 43 +++++++++++-------- adapter/redis_lua_negative_type_cache_test.go | 12 +++--- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index 3b10a46ac..260c6f861 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -48,7 +48,8 @@ type luaScriptContext struct { // buggy script from probing unbounded unique non-existent keys and // blowing up memory. Once full, further misses fall back to the // server-side probe (still correct, just not cached). - negativeType map[string]bool + negativeType map[string]bool + negativeTypeLimit int // keyTypeProbeCount counts how many times the server-side keyTypeAt // helper was invoked during this Eval. Only read by tests via @@ -261,21 +262,22 @@ func newLuaScriptContext(ctx context.Context, server *RedisServer) (*luaScriptCo } startTS := server.readTS() return &luaScriptContext{ - server: server, - startTS: startTS, - readPin: server.pinReadTS(startTS), - ctx: ctx, - touched: map[string]struct{}{}, - deleted: map[string]bool{}, - everDeleted: map[string]bool{}, - negativeType: map[string]bool{}, - strings: map[string]*luaStringState{}, - lists: map[string]*luaListState{}, - hashes: map[string]*luaHashState{}, - sets: map[string]*luaSetState{}, - zsets: map[string]*luaZSetState{}, - streams: map[string]*luaStreamState{}, - ttls: map[string]*luaTTLState{}, + server: server, + startTS: startTS, + readPin: server.pinReadTS(startTS), + ctx: ctx, + touched: map[string]struct{}{}, + deleted: map[string]bool{}, + everDeleted: map[string]bool{}, + negativeType: map[string]bool{}, + negativeTypeLimit: maxNegativeTypeCacheEntries, + strings: map[string]*luaStringState{}, + lists: map[string]*luaListState{}, + hashes: map[string]*luaHashState{}, + sets: map[string]*luaSetState{}, + zsets: map[string]*luaZSetState{}, + streams: map[string]*luaStreamState{}, + ttls: map[string]*luaTTLState{}, }, nil } @@ -479,7 +481,7 @@ func (c *luaScriptContext) keyType(key []byte) (redisValueType, error) { if err != nil { return redisTypeNone, err } - if typ == redisTypeNone && len(c.negativeType) < maxNegativeTypeCacheEntries { + if typ == redisTypeNone && len(c.negativeType) < c.effectiveNegativeTypeLimit() { // Pin the absence result for the rest of this Eval so repeated // BullMQ-style polling of a missing key (e.g. a "delayed" zset) // does not re-run the ~8-seek rawKeyTypeAt probe on every @@ -493,6 +495,13 @@ func (c *luaScriptContext) keyType(key []byte) (redisValueType, error) { return typ, nil } +func (c *luaScriptContext) effectiveNegativeTypeLimit() int { + if c.negativeTypeLimit > 0 { + return c.negativeTypeLimit + } + return maxNegativeTypeCacheEntries +} + func (c *luaScriptContext) ensureKeyNotExpired(key []byte) error { ttl, err := c.loadTTL(key) if err != nil { diff --git a/adapter/redis_lua_negative_type_cache_test.go b/adapter/redis_lua_negative_type_cache_test.go index 6253f7e30..6a3a29db3 100644 --- a/adapter/redis_lua_negative_type_cache_test.go +++ b/adapter/redis_lua_negative_type_cache_test.go @@ -156,17 +156,19 @@ func TestLuaNegativeTypeCache_BoundedSize(t *testing.T) { sc, err := newLuaScriptContext(ctx, nodes[0].redisServer) require.NoError(t, err) defer sc.Close() + const cacheLimit = 4 + sc.negativeTypeLimit = cacheLimit // Probe cap+overflow unique missing keys. Each probe is a miss // (redisTypeNone); only the first `cap` should be memoized. - const overflow = 50 - total := maxNegativeTypeCacheEntries + overflow + const overflow = 2 + total := cacheLimit + overflow for i := 0; i < total; i++ { typ, kerr := sc.keyType([]byte(fmt.Sprintf("lua:neg:cap:%d", i))) require.NoError(t, kerr) require.Equal(t, redisTypeNone, typ) } - require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType), + require.Equal(t, cacheLimit, len(sc.negativeType), "negativeType map must be capped at maxNegativeTypeCacheEntries") // Each unique key above required exactly one probe on first access. @@ -182,11 +184,11 @@ func TestLuaNegativeTypeCache_BoundedSize(t *testing.T) { require.Equal(t, total, sc.keyTypeProbeCount, "a key inserted before the cap must remain cached") - overflowKey := []byte(fmt.Sprintf("lua:neg:cap:%d", maxNegativeTypeCacheEntries+1)) + overflowKey := []byte(fmt.Sprintf("lua:neg:cap:%d", cacheLimit+1)) _, kerr = sc.keyType(overflowKey) require.NoError(t, kerr) require.Equal(t, total+1, sc.keyTypeProbeCount, "a key probed after the cap was reached must fall back to the server probe") - require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType), + require.Equal(t, cacheLimit, len(sc.negativeType), "fallback probe must NOT grow the bounded cache") } From 12d34f721443436a6f9e7b0d9723ca8e66a05a55 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 22:23:25 +0900 Subject: [PATCH 15/16] encryption: harden Vault Transit key binding --- ...6_04_29_partial_data_at_rest_encryption.md | 3 +- ...2026_07_18_implemented_9b_kek_providers.md | 9 +++-- internal/encryption/kek/vault.go | 18 +++++++-- internal/encryption/kek/vault_test.go | 39 +++++++++++++++---- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 21fadc4f4..91af40095 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -1023,7 +1023,8 @@ Two-tier hierarchy: 1. AWS KMS: `--kekUri=aws-kms://arn:aws:kms:...:key/...` with an immutable key ARN (mutable alias ARNs are rejected). The KMS key never leaves AWS; we call `Encrypt` / `Decrypt` to wrap/unwrap DEKs. - 2. GCP KMS: `--kekUri=gcp-kms://projects/.../keys/...`. Same shape. + 2. GCP KMS: `--kekUri=gcp-kms://projects//locations//keyRings//cryptoKeys/`. + Same shape. 3. HashiCorp Vault Transit: `--kekUri=vault-transit://...`. 4. Static file: `--kekFile=/etc/elastickv/kek.bin` — 32 bytes raw. Recommended only when the file lives on a tmpfs or sealed diff --git a/docs/design/2026_07_18_implemented_9b_kek_providers.md b/docs/design/2026_07_18_implemented_9b_kek_providers.md index 5cd2b0451..d178ea543 100644 --- a/docs/design/2026_07_18_implemented_9b_kek_providers.md +++ b/docs/design/2026_07_18_implemented_9b_kek_providers.md @@ -16,8 +16,11 @@ Stage 9B completes the §5.1 KEK source matrix: AAD. Request and response CRC32C values are supplied and verified before a wrapped DEK or plaintext DEK is accepted. - `--kekUri=vault-transit:///` uses Vault Transit through standard - `VAULT_*` client configuration. Binary DEKs are base64 encoded for the API; - only `vault:v:` ciphertexts are accepted. + `VAULT_*` client configuration, including nested mount paths with the final + path segment interpreted as the key name. The existing key is read before + encrypt to prevent implicit key creation, and encrypt/decrypt use fixed AAD. + Binary DEKs are base64 encoded for the API; only + `vault:v:` ciphertexts are accepted. - `ELASTICKV_KEK_BASE64` supplies a 32-byte test/CI-only static KEK. The variable is unset immediately after the decode attempt, including malformed input. It is also unset when source ambiguity is rejected. @@ -55,7 +58,7 @@ unchanged. Verification includes: - fake-client unit tests for AWS encryption context, GCP AAD/CRC32C, and Vault - binary request/response encoding; + key existence, nested mount, AAD, and binary request/response encoding; - malformed provider response, strict Vault ciphertext, immutable AWS key ARN, wrong DEK length, invalid URI, source conflict, environment-unset, and startup wrap/unwrap preflight tests; diff --git a/internal/encryption/kek/vault.go b/internal/encryption/kek/vault.go index d9790a90e..9a36cef7d 100644 --- a/internal/encryption/kek/vault.go +++ b/internal/encryption/kek/vault.go @@ -13,9 +13,12 @@ import ( ) type vaultLogicalClient interface { + ReadWithContext(context.Context, string) (*vaultapi.Secret, error) WriteWithContext(context.Context, string, map[string]interface{}) (*vaultapi.Secret, error) } +var vaultTransitAAD = base64.StdEncoding.EncodeToString([]byte("elastickv-dek-wrap-v1")) + // VaultTransitWrapper wraps DEKs with a Vault Transit symmetric key. type VaultTransitWrapper struct { logical vaultLogicalClient @@ -57,7 +60,7 @@ func parseVaultTarget(target string) (string, string, error) { return "", "", errors.Wrapf(ErrInvalidKEKURI, "invalid Vault Transit target %q", target) } } - return parts[0], strings.Join(parts[1:], "/"), nil + return strings.Join(parts[:len(parts)-1], "/"), parts[len(parts)-1], nil } // Wrap base64-encodes the binary DEK for Vault's JSON API and stores Vault's @@ -71,8 +74,16 @@ func (w *VaultTransitWrapper) Wrap(dek []byte) ([]byte, error) { } ctx, cancel := requestContext(w.timeout) defer cancel() + key, err := w.logical.ReadWithContext(ctx, w.mount+"/keys/"+w.keyName) + if err != nil { + return nil, errors.Wrap(err, "kek: read Vault Transit key") + } + if _, ok := vaultString(key, "type"); !ok { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: Vault Transit key does not exist") + } secret, err := w.logical.WriteWithContext(ctx, w.mount+"/encrypt/"+w.keyName, map[string]interface{}{ - "plaintext": base64.StdEncoding.EncodeToString(dek), + "plaintext": base64.StdEncoding.EncodeToString(dek), + "associated_data": vaultTransitAAD, }) if err != nil { return nil, errors.Wrap(err, "kek: Vault Transit encrypt") @@ -96,7 +107,8 @@ func (w *VaultTransitWrapper) Unwrap(wrapped []byte) ([]byte, error) { ctx, cancel := requestContext(w.timeout) defer cancel() secret, err := w.logical.WriteWithContext(ctx, w.mount+"/decrypt/"+w.keyName, map[string]interface{}{ - "ciphertext": string(wrapped), + "ciphertext": string(wrapped), + "associated_data": vaultTransitAAD, }) if err != nil { return nil, errors.Wrap(err, "kek: Vault Transit decrypt") diff --git a/internal/encryption/kek/vault_test.go b/internal/encryption/kek/vault_test.go index bb09ccff4..900ea5115 100644 --- a/internal/encryption/kek/vault_test.go +++ b/internal/encryption/kek/vault_test.go @@ -11,14 +11,24 @@ import ( ) type fakeVaultLogical struct { - path string + readPath string + writePath string data map[string]interface{} dek []byte ciphertext string + key *api.Secret +} + +func (f *fakeVaultLogical) ReadWithContext(_ context.Context, path string) (*api.Secret, error) { + f.readPath = path + if f.key != nil { + return f.key, nil + } + return &api.Secret{Data: map[string]interface{}{"type": "aes256-gcm96"}}, nil } func (f *fakeVaultLogical) WriteWithContext(_ context.Context, path string, data map[string]interface{}) (*api.Secret, error) { - f.path = path + f.writePath = path f.data = data if _, ok := data["plaintext"]; ok { ciphertext := f.ciphertext @@ -33,27 +43,30 @@ func (f *fakeVaultLogical) WriteWithContext(_ context.Context, path string, data func TestVaultTransitWrapperRequestBinding(t *testing.T) { dek := bytes.Repeat([]byte{0x62}, fileKEKSize) logical := &fakeVaultLogical{dek: dek} - wrapper := newVaultTransitWrapper(logical, "transit", "service/key") + wrapper := newVaultTransitWrapper(logical, "security/transit", "orders") wrapped, err := wrapper.Wrap(dek) require.NoError(t, err) require.Equal(t, []byte("vault:v1:ciphertext"), wrapped) - require.Equal(t, "transit/encrypt/service/key", logical.path) + require.Equal(t, "security/transit/keys/orders", logical.readPath) + require.Equal(t, "security/transit/encrypt/orders", logical.writePath) require.Equal(t, base64.StdEncoding.EncodeToString(dek), logical.data["plaintext"]) + require.Equal(t, vaultTransitAAD, logical.data["associated_data"]) plain, err := wrapper.Unwrap(wrapped) require.NoError(t, err) require.Equal(t, dek, plain) - require.Equal(t, "transit/decrypt/service/key", logical.path) + require.Equal(t, "security/transit/decrypt/orders", logical.writePath) require.Equal(t, "vault:v1:ciphertext", logical.data["ciphertext"]) - require.Equal(t, "vault-transit:transit/service/key", wrapper.Name()) + require.Equal(t, vaultTransitAAD, logical.data["associated_data"]) + require.Equal(t, "vault-transit:security/transit/orders", wrapper.Name()) } func TestParseVaultTarget(t *testing.T) { mount, keyName, err := parseVaultTarget("transit/service/key") require.NoError(t, err) - require.Equal(t, "transit", mount) - require.Equal(t, "service/key", keyName) + require.Equal(t, "transit/service", mount) + require.Equal(t, "key", keyName) for _, target := range []string{"", "transit", "transit//key", "../key", "transit/../key"} { t.Run(target, func(t *testing.T) { _, _, err := parseVaultTarget(target) @@ -62,6 +75,16 @@ func TestParseVaultTarget(t *testing.T) { } } +func TestVaultTransitWrapperRequiresExistingKey(t *testing.T) { + logical := &fakeVaultLogical{key: &api.Secret{Data: map[string]interface{}{}}} + wrapper := newVaultTransitWrapper(logical, "transit", "missing") + + _, err := wrapper.Wrap(bytes.Repeat([]byte{0x42}, fileKEKSize)) + require.ErrorIs(t, err, ErrInvalidProviderResponse) + require.Equal(t, "transit/keys/missing", logical.readPath) + require.Empty(t, logical.writePath) +} + func TestVaultTransitWrapperRejectsMalformedResponses(t *testing.T) { logical := &fakeVaultLogical{dek: []byte("short")} wrapper := newVaultTransitWrapper(logical, "transit", "key") From 2308534284bdd86eac29d103cfacc06b7f104145 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 17:17:34 +0900 Subject: [PATCH 16/16] proto: reserve removed raft status fields --- proto/service.pb.go | 4 ++-- proto/service.proto | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/proto/service.pb.go b/proto/service.pb.go index 04d7cce82..4ea699619 100644 --- a/proto/service.pb.go +++ b/proto/service.pb.go @@ -2558,7 +2558,7 @@ const file_service_proto_rawDesc = "" + "\bstart_ts\x18\x01 \x01(\x04R\astartTs\",\n" + "\x10RollbackResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\"\x18\n" + - "\x16RaftAdminStatusRequest\"\xa2\x03\n" + + "\x16RaftAdminStatusRequest\"\xd8\x03\n" + "\x17RaftAdminStatusResponse\x12%\n" + "\x05state\x18\x01 \x01(\x0e2\x0f.RaftAdminStateR\x05state\x12\x1b\n" + "\tleader_id\x18\x02 \x01(\tR\bleaderId\x12%\n" + @@ -2572,7 +2572,7 @@ const file_service_proto_rawDesc = "" + "fsmPending\x12\x1b\n" + "\tnum_peers\x18\n" + " \x01(\x04R\bnumPeers\x12,\n" + - "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanos\"\x1f\n" + + "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanosJ\x04\b\f\x10\rJ\x04\b\r\x10\x0eR\x13configuration_indexR\x13pending_conf_change\"\x1f\n" + "\x1dRaftAdminConfigurationRequest\"W\n" + "\x0fRaftAdminMember\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x18\n" + diff --git a/proto/service.proto b/proto/service.proto index a2212b101..9e4d13d6d 100644 --- a/proto/service.proto +++ b/proto/service.proto @@ -200,6 +200,8 @@ message RaftAdminStatusResponse { uint64 fsm_pending = 9; uint64 num_peers = 10; int64 last_contact_nanos = 11; + reserved 12, 13; + reserved "configuration_index", "pending_conf_change"; } message RaftAdminConfigurationRequest {}