Add production KEK providers

adapter/redis_lua_context.go

@@ -48,7 +48,8 @@ type luaScriptContext struct {
 	// buggy script from probing unbounded unique non-existent keys and
 	// blowing up memory. Once full, further misses fall back to the
 	// server-side probe (still correct, just not cached).
-	negativeType map[string]bool
+	negativeType      map[string]bool
+	negativeTypeLimit int
 
 	// keyTypeProbeCount counts how many times the server-side keyTypeAt
 	// helper was invoked during this Eval. Only read by tests via
@@ -261,21 +262,22 @@ func newLuaScriptContext(ctx context.Context, server *RedisServer) (*luaScriptCo
 	}
 	startTS := server.readTS()
 	return &luaScriptContext{
-		server:       server,
-		startTS:      startTS,
-		readPin:      server.pinReadTS(startTS),
-		ctx:          ctx,
-		touched:      map[string]struct{}{},
-		deleted:      map[string]bool{},
-		everDeleted:  map[string]bool{},
-		negativeType: map[string]bool{},
-		strings:      map[string]*luaStringState{},
-		lists:        map[string]*luaListState{},
-		hashes:       map[string]*luaHashState{},
-		sets:         map[string]*luaSetState{},
-		zsets:        map[string]*luaZSetState{},
-		streams:      map[string]*luaStreamState{},
-		ttls:         map[string]*luaTTLState{},
+		server:            server,
+		startTS:           startTS,
+		readPin:           server.pinReadTS(startTS),
+		ctx:               ctx,
+		touched:           map[string]struct{}{},
+		deleted:           map[string]bool{},
+		everDeleted:       map[string]bool{},
+		negativeType:      map[string]bool{},
+		negativeTypeLimit: maxNegativeTypeCacheEntries,
+		strings:           map[string]*luaStringState{},
+		lists:             map[string]*luaListState{},
+		hashes:            map[string]*luaHashState{},
+		sets:              map[string]*luaSetState{},
+		zsets:             map[string]*luaZSetState{},
+		streams:           map[string]*luaStreamState{},
+		ttls:              map[string]*luaTTLState{},
 	}, nil
 }
 
@@ -479,7 +481,7 @@ func (c *luaScriptContext) keyType(key []byte) (redisValueType, error) {
 	if err != nil {
 		return redisTypeNone, err
 	}
-	if typ == redisTypeNone && len(c.negativeType) < maxNegativeTypeCacheEntries {
+	if typ == redisTypeNone && len(c.negativeType) < c.effectiveNegativeTypeLimit() {
 		// Pin the absence result for the rest of this Eval so repeated
 		// BullMQ-style polling of a missing key (e.g. a "delayed" zset)
 		// does not re-run the ~8-seek rawKeyTypeAt probe on every
@@ -493,6 +495,13 @@ func (c *luaScriptContext) keyType(key []byte) (redisValueType, error) {
 	return typ, nil
 }
 
+func (c *luaScriptContext) effectiveNegativeTypeLimit() int {
+	if c.negativeTypeLimit > 0 {
+		return c.negativeTypeLimit
+	}
+	return maxNegativeTypeCacheEntries
+}
+
 func (c *luaScriptContext) ensureKeyNotExpired(key []byte) error {
 	ttl, err := c.loadTTL(key)
 	if err != nil {

adapter/redis_lua_negative_type_cache_test.go

@@ -156,17 +156,19 @@ func TestLuaNegativeTypeCache_BoundedSize(t *testing.T) {
 	sc, err := newLuaScriptContext(ctx, nodes[0].redisServer)
 	require.NoError(t, err)
 	defer sc.Close()
+	const cacheLimit = 4
+	sc.negativeTypeLimit = cacheLimit
 
 	// Probe cap+overflow unique missing keys. Each probe is a miss
 	// (redisTypeNone); only the first `cap` should be memoized.
-	const overflow = 50
-	total := maxNegativeTypeCacheEntries + overflow
+	const overflow = 2
+	total := cacheLimit + overflow
 	for i := 0; i < total; i++ {
 		typ, kerr := sc.keyType([]byte(fmt.Sprintf("lua:neg:cap:%d", i)))
 		require.NoError(t, kerr)
 		require.Equal(t, redisTypeNone, typ)
 	}
-	require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType),
+	require.Equal(t, cacheLimit, len(sc.negativeType),
 		"negativeType map must be capped at maxNegativeTypeCacheEntries")
 
 	// Each unique key above required exactly one probe on first access.
@@ -182,11 +184,11 @@ func TestLuaNegativeTypeCache_BoundedSize(t *testing.T) {
 	require.Equal(t, total, sc.keyTypeProbeCount,
 		"a key inserted before the cap must remain cached")
 
-	overflowKey := []byte(fmt.Sprintf("lua:neg:cap:%d", maxNegativeTypeCacheEntries+1))
+	overflowKey := []byte(fmt.Sprintf("lua:neg:cap:%d", cacheLimit+1))
 	_, kerr = sc.keyType(overflowKey)
 	require.NoError(t, kerr)
 	require.Equal(t, total+1, sc.keyTypeProbeCount,
 		"a key probed after the cap was reached must fall back to the server probe")
-	require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType),
+	require.Equal(t, cacheLimit, len(sc.negativeType),
 		"fallback probe must NOT grow the bounded cache")
 }

docs/design/2026_04_29_partial_data_at_rest_encryption.md

@@ -1,6 +1,6 @@
 # Data-at-rest encryption for elastickv
 
-Status: Partial — Stages 0–8 and 9A shipped (5E deferred); remaining Stage 9 work open
+Status: Partial — Stages 0–8 and 9A–9B shipped (5E deferred); remaining Stage 9 work open
 Author: bootjp
 Date: 2026-04-29
 
@@ -33,7 +33,8 @@ Date: 2026-04-29
 | 7 | Writer registry + deterministic nonce (§4.1). Covers process-start registration, storage-layer registration gate, runtime re-registration for cutover and rotation, conf-change-time pre-registration, and §5.5 registry projection in `GetSidecarState` / `ResyncSidecar` (`WriterRegistryForCaller`). | shipped | `2026_05_26_implemented_7a_process_start_registration.md` + `2026_05_26_implemented_7a2_storage_layer_registration_enforcement.md` + `2026_05_28_implemented_7b_runtime_reregistration.md` + `2026_05_28_implemented_7b_prime_runtime_reregistration_rotation.md` + `2026_05_29_implemented_7c_confchange_time_registration.md` + `2026_07_11_implemented_7d_sidecar_registry_projection.md` |
 | 8 | Snapshot header v2 (§4.4); WAL coverage closure (§4.3 / §4.6) | shipped | [`2026_05_29_implemented_8a_snapshot_header_v2.md`](2026_05_29_implemented_8a_snapshot_header_v2.md) + [`2026_06_01_implemented_8b_wal_coverage_closure.md`](2026_06_01_implemented_8b_wal_coverage_closure.md) |
 | 9A | Compress-then-encrypt, authenticated compression flag, encrypted-store Pebble compression policy, storage benchmark (§6.4, §8.3) | shipped | `2026_07_18_implemented_9a_encryption_compression.md` |
-| 9B+ | KMS-backed wrappers, rotation/retire/rewrite, remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8) | open | — |
+| 9B | AWS KMS, GCP KMS, Vault Transit, and test/CI env KEK providers; mutually-exclusive source loader and loaded-provider mutator gate (§5.1, §6.1, §6.5) | shipped | `2026_07_18_implemented_9b_kek_providers.md` |
+| 9C+ | Rotation budget/rewrap/retire/rewrite, metrics, remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8, §9.2) | open | — |
 
 Stages 0–4 ship the entire byte-tag pipeline (storage envelope, raft
 envelope, FSM dispatch, halt-on-error) but leave it **production
@@ -1019,9 +1020,11 @@ Two-tier hierarchy:
 
 - **KEK (Key Encryption Key).** Held outside the cluster, never on
   the cluster's disks. Sources, in order of preference:
-  1. AWS KMS: `--kekUri=aws-kms://arn:aws:kms:...`. The KMS key never
+  1. AWS KMS: `--kekUri=aws-kms://arn:aws:kms:...:key/...` with an
+     immutable key ARN (mutable alias ARNs are rejected). The KMS key never
      leaves AWS; we call `Encrypt` / `Decrypt` to wrap/unwrap DEKs.
-  2. GCP KMS: `--kekUri=gcp-kms://projects/.../keys/...`. Same shape.
+  2. GCP KMS: `--kekUri=gcp-kms://projects/<project>/locations/<location>/keyRings/<ring>/cryptoKeys/<key>`.
+     Same shape.
   3. HashiCorp Vault Transit: `--kekUri=vault-transit://...`.
   4. Static file: `--kekFile=/etc/elastickv/kek.bin` — 32 bytes raw.
      Recommended only when the file lives on a tmpfs or sealed
@@ -1031,7 +1034,10 @@ Two-tier hierarchy:
      inspection); supported only for tests and CI.
 
   No default. If `--encryption-enabled` is set without a KEK source,
-  the process refuses to start.
+  the process refuses to start. Before encryption mutators are exposed, every
+  configured provider must also complete a random 32-byte DEK wrap/unwrap
+  preflight; this catches credentials, reachability, permission, key-binding,
+  and malformed-response failures before a Raft entry can commit.
 
 - **DEK (Data Encryption Key).** 32-byte AES key. Two DEKs are
   issued in v1: `dek_storage` (used by §4.1) and `dek_raft`

docs/design/2026_07_18_implemented_9b_kek_providers.md

@@ -0,0 +1,76 @@
+# Stage 9B: production KEK providers
+
+Status: Implemented
+Author: bootjp
+Date: 2026-07-18
+
+## Scope
+
+Stage 9B completes the §5.1 KEK source matrix:
+
+- `--kekUri=aws-kms://<key-arn>` uses an immutable AWS KMS `key/` ARN with
+  `Encrypt` / `Decrypt`, derives
+  the client region from the ARN, and binds every request to the fixed
+  `elastickv-purpose=dek-wrap-v1` encryption context.
+- `--kekUri=gcp-kms://<crypto-key-resource>` uses Google Cloud KMS with fixed
+  AAD. Request and response CRC32C values are supplied and verified before a
+  wrapped DEK or plaintext DEK is accepted.
+- `--kekUri=vault-transit://<mount>/<key>` uses Vault Transit through standard
+  `VAULT_*` client configuration, including nested mount paths with the final
+  path segment interpreted as the key name. The existing key is read before
+  encrypt to prevent implicit key creation, and encrypt/decrypt use fixed AAD.
+  Binary DEKs are base64 encoded for the API; only
+  `vault:v<positive-version>:<non-empty-payload>` ciphertexts are accepted.
+- `ELASTICKV_KEK_BASE64` supplies a 32-byte test/CI-only static KEK. The
+  variable is unset immediately after the decode attempt, including malformed
+  input. It is also unset when source ambiguity is rejected.
+- `--kekFile` keeps its existing owner-only regular-file contract.
+
+File, URI, and environment sources are mutually exclusive. Ambiguous
+configuration fails closed instead of selecting a source by precedence.
+
+## Runtime wiring
+
+The startup loader returns the actual loaded `kek.Wrapper`. Startup guards now
+derive `KEKConfigured` from that non-nil wrapper rather than from `--kekFile`
+text. The same loaded-state boolean is threaded to the EncryptionAdmin mutator
+gate, so URI and environment providers can bootstrap/rotate while a failed or
+absent provider cannot expose mutating RPCs.
+
+Before that gate can open, startup performs a real random 32-byte DEK
+wrap/unwrap round trip and compares the result in constant time. This proves
+provider reachability, credentials, encrypt/decrypt permission, key binding,
+and response shape on a fresh data directory where no sidecar DEK exists to
+exercise the provider. A failed preflight refuses process start before Raft or
+the mutating RPC surface is available.
+
+Remote wrappers use a 30-second operation deadline and reject nil, empty,
+integrity-invalid, or non-32-byte provider responses before sidecar state can be
+updated. Provider clients are safe for concurrent use and are hidden behind
+narrow interfaces for deterministic tests.
+
+## Compatibility and verification
+
+No envelope, sidecar, Raft opcode, or snapshot format changes. Existing wrapped
+DEKs remain provider-specific opaque bytes, and the `kek.Wrapper` contract is
+unchanged.
+
+Verification includes:
+
+- fake-client unit tests for AWS encryption context, GCP AAD/CRC32C, and Vault
+  key existence, nested mount, AAD, and binary request/response encoding;
+- malformed provider response, strict Vault ciphertext, immutable AWS key ARN,
+  wrong DEK length, invalid URI, source conflict, environment-unset, and
+  startup wrap/unwrap preflight tests;
+- a production-wiring integration test that loads the environment KEK through
+  the source selector, bootstraps both DEKs, enables the storage envelope,
+  writes encrypted data, snapshots it, restores it into an encrypted Pebble
+  store, and reads the original plaintext;
+- startup/mutator caller audit confirming the gate consumes loaded-wrapper
+  state rather than the legacy file flag.
+
+## Remaining work
+
+Stage 5E discovery batching and Stage 9C+ rotation-budget, rewrap, rewrite,
+retirement, metrics, remaining benchmark, and encrypted Jepsen requirements are
+not part of this milestone.

go.mod

@@ -5,11 +5,13 @@ go 1.26
 toolchain go1.26.5
 
 require (
+	cloud.google.com/go/kms v1.31.0
 	github.com/Jille/grpc-multi-resolver v1.3.0
 	github.com/aws/aws-sdk-go-v2 v1.42.1
 	github.com/aws/aws-sdk-go-v2/config v1.32.29
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.28
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.60.0
+	github.com/aws/aws-sdk-go-v2/service/kms v1.54.1
 	github.com/aws/smithy-go v1.27.3
 	github.com/cockroachdb/errors v1.14.0
 	github.com/cockroachdb/pebble/v2 v2.1.6
@@ -19,6 +21,7 @@ require (
 	github.com/goccy/go-json v0.10.6
 	github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e
 	github.com/hanwen/go-fuse/v2 v2.10.1
+	github.com/hashicorp/vault/api v1.23.0
 	github.com/klauspost/compress v1.19.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.23.2
@@ -41,6 +44,12 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.123.0 // indirect
+	cloud.google.com/go/auth v0.20.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
+	cloud.google.com/go/iam v1.7.0 // indirect
+	cloud.google.com/go/longrunning v0.9.0 // indirect
 	github.com/DataDog/zstd v1.5.7 // indirect
 	github.com/RaduBerinde/axisds v0.1.0 // indirect
 	github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect
@@ -56,6 +65,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.44.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
@@ -65,23 +75,40 @@ require (
 	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect
+	github.com/googleapis/gax-go/v2 v2.21.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.7 // indirect
+	github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
 	github.com/jonboulle/clockwork v0.5.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/tidwall/btree v1.1.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
@@ -92,6 +119,7 @@ require (
 	go.etcd.io/etcd/pkg/v3 v3.7.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
 	go.opentelemetry.io/otel v1.43.0 // indirect
 	go.opentelemetry.io/otel/metric v1.43.0 // indirect
 	go.opentelemetry.io/otel/trace v1.43.0 // indirect
@@ -101,7 +129,11 @@ require (
 	golang.org/x/crypto v0.52.0 // indirect
 	golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
 	golang.org/x/net v0.55.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/text v0.37.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/api v0.274.0 // indirect
+	google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect