From c6a9aa30bcf3502d178494ee488749f8044638e1 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 16 Jun 2026 15:50:19 -0700 Subject: [PATCH 1/6] feat(registry): add storage backend registry (KPEP-0001 phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the registry/ subpackage hosting the Backend interface, Factory type, and Backends instance-scoped registry. This is the first implementation step for KPEP-0001 (pluggable storage backends). The package is intentionally additive — nothing in the rest of kplane-dev/storage imports it, so existing consumers (the apiserver's StorageWithClusterIdentity decorator path) continue to work unchanged. The wiring lands in later PRs: - kplane-dev/spanner adds a Register() that satisfies registry.Backend. - kplane-dev/apiserver constructs *Backends in main and dispatches via --storage-backend (alongside the existing hardcoded if/else for one release, then the hardcoded branch is removed). The Factory signature matches upstream storagebackend/factory.Create exactly so existing backend implementations (the BackendFactory type that kplane-dev/spanner already exposes) plug in without adaptation. Tested: go test ./registry/... covers Register/Get/Names/AddFlags fan-out + duplicate-panic behavior. Existing tests (e2e_test.go, identity_test.go, keylayout_test.go) unaffected since nothing in those paths references the new package. See KPEP-0001 in kplane-dev/enhancements for the design rationale. --- go.mod | 2 +- registry/registry.go | 142 ++++++++++++++++++++++++++++++++++++++ registry/registry_test.go | 117 +++++++++++++++++++++++++++++++ 3 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 registry/registry.go create mode 100644 registry/registry_test.go diff --git a/go.mod b/go.mod index 347e0cb..45ad237 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ replace ( ) require ( + github.com/spf13/pflag v1.0.9 k8s.io/apimachinery v0.0.0 k8s.io/apiserver v0.0.0-00010101000000-000000000000 k8s.io/client-go v0.0.0 @@ -64,7 +65,6 @@ require ( github.com/prometheus/procfs v0.19.2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spf13/pflag v1.0.9 // indirect github.com/stretchr/testify v1.11.1 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/registry/registry.go b/registry/registry.go new file mode 100644 index 0000000..4a174f6 --- /dev/null +++ b/registry/registry.go @@ -0,0 +1,142 @@ +// Package registry hosts the kplane storage backend registry: an +// instance-scoped lookup of named backends (etcd3, spanner, future +// postgres, etc.) that the apiserver consults instead of the hardcoded +// per-backend if/else it carries today. +// +// Design lives in KPEP-0001 (kplane-dev/enhancements). The shape mirrors +// upstream admission.Plugins: backends self-describe via the Backend +// interface, register against a *Backends, and the apiserver picks one +// at startup based on --storage-backend. +// +// This package is intentionally additive — nothing in the rest of +// kplane-dev/storage imports it. Existing consumers continue to use +// DecoratorConfig / StorageWithClusterIdentity unchanged. Wiring happens +// in the apiserver once the per-backend Register hooks are in place. +package registry + +import ( + "fmt" + "sort" + "sync" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage" + "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/storage/storagebackend/factory" +) + +// Factory is the per-resource constructor a backend returns from Build(). +// The signature is a 1-to-1 match with upstream's +// storagebackend/factory.Create so any backend that already satisfies the +// existing kplane-dev/spanner BackendFactory shape works without +// adaptation. The apiserver's RESTOptionsGetter calls this once per +// GroupResource at REST registry construction time. +type Factory func( + config *storagebackend.ConfigForResource, + newFunc, newListFunc func() runtime.Object, + resourcePrefix string, +) (storage.Interface, factory.DestroyFunc, error) + +// Backend is what every registered storage backend implements. The +// lifecycle matches upstream's options-struct convention: bind flags, +// validate after parse, build the per-resource factory once. +// +// AddFlags is always called for every registered backend so the apiserver +// help output lists every backend's flag block, the same way upstream +// surfaces --etcd-* / --storage-media-type / --watch-cache regardless of +// which storage backend is selected. +// +// Validate is called only for the backend whose Name() matches the +// selected --storage-backend value. Build is called once after Validate +// passes and returns the per-resource Factory threaded into the apiserver's +// RESTOptions decorator chain. +type Backend interface { + // Name is the value matched against --storage-backend. The full set + // of registered names is shown in the flag help text. + Name() string + + // AddFlags binds the backend's CLI flags to fs. Called for every + // registered backend at startup so all flags appear in --help. + // In-tree backends are free to make this a no-op when their flags + // are already bound by another options block (the etcd wrapper does + // this because upstream EtcdOptions already owns --etcd-*). + AddFlags(fs *pflag.FlagSet) + + // Validate runs the backend's flag-level validation. Only called for + // the selected backend, after flag parsing. Errors short-circuit + // apiserver startup with a clear message naming the backend. + Validate() []error + + // Build returns the per-resource Factory. Called once after Validate. + // Any expensive setup (dialing a remote, opening a session pool) + // belongs here, not in AddFlags or Validate. + Build() (Factory, error) +} + +// Backends is the instance-scoped registry. It mirrors upstream's +// admission.Plugins shape — instance-scoped rather than a package-level +// global, so tests can construct fresh registries without leaking state +// and external consumers can build custom apiservers with their own +// backend selections. +type Backends struct { + mu sync.RWMutex + registered map[string]Backend +} + +// New returns an empty Backends registry. The apiserver constructs one +// in main, calls RegisterBuiltin (from kplane-dev/storage/backends), and +// optionally adds external backends before threading it into the options +// chain. +func New() *Backends { + return &Backends{registered: map[string]Backend{}} +} + +// Register installs backend into b, keyed by backend.Name(). Panics on +// duplicate name — same convention as database/sql.Register: a duplicate +// registration is a programming error, not a runtime condition we want +// to silently overwrite. +func (b *Backends) Register(backend Backend) { + name := backend.Name() + b.mu.Lock() + defer b.mu.Unlock() + if _, exists := b.registered[name]; exists { + panic(fmt.Sprintf("storage backend %q already registered", name)) + } + b.registered[name] = backend +} + +// Get returns the backend registered under name, or ok=false if no such +// backend was registered. The apiserver uses this after flag parse to +// dispatch Validate/Build. +func (b *Backends) Get(name string) (Backend, bool) { + b.mu.RLock() + defer b.mu.RUnlock() + backend, ok := b.registered[name] + return backend, ok +} + +// Names returns the sorted list of registered backend names. Used by the +// apiserver to populate the --storage-backend flag help text and to +// produce a useful error when an unknown backend name is requested. +func (b *Backends) Names() []string { + b.mu.RLock() + defer b.mu.RUnlock() + names := make([]string, 0, len(b.registered)) + for n := range b.registered { + names = append(names, n) + } + sort.Strings(names) + return names +} + +// AddFlags calls AddFlags on every registered backend, threading the same +// pflag set into each. Called once at startup before flag parse so the +// apiserver's --help lists every backend's flag block. +func (b *Backends) AddFlags(fs *pflag.FlagSet) { + b.mu.RLock() + defer b.mu.RUnlock() + for _, backend := range b.registered { + backend.AddFlags(fs) + } +} diff --git a/registry/registry_test.go b/registry/registry_test.go new file mode 100644 index 0000000..f349b1f --- /dev/null +++ b/registry/registry_test.go @@ -0,0 +1,117 @@ +package registry_test + +import ( + "errors" + "reflect" + "testing" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage" + "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/storage/storagebackend/factory" + + "github.com/kplane-dev/storage/registry" +) + +// fakeBackend is the minimal Backend impl used to exercise the registry. +// Real backends (etcd, spanner, postgres) live in +// kplane-dev/storage/backends// in later PRs. +type fakeBackend struct { + name string + flagsAdded bool + validateCalls int + validateErrors []error + buildCalls int + buildErr error +} + +func (f *fakeBackend) Name() string { return f.name } +func (f *fakeBackend) AddFlags(*pflag.FlagSet) { f.flagsAdded = true } +func (f *fakeBackend) Validate() []error { f.validateCalls++; return f.validateErrors } +func (f *fakeBackend) Build() (registry.Factory, error) { + f.buildCalls++ + if f.buildErr != nil { + return nil, f.buildErr + } + return func(*storagebackend.ConfigForResource, func() runtime.Object, func() runtime.Object, string) (storage.Interface, factory.DestroyFunc, error) { + return nil, func() {}, nil + }, nil +} + +func TestRegisterAndGet(t *testing.T) { + b := registry.New() + a := &fakeBackend{name: "alpha"} + b.Register(a) + + got, ok := b.Get("alpha") + if !ok { + t.Fatalf("Get(alpha) returned ok=false; want true") + } + if got != a { + t.Fatalf("Get(alpha) returned %v; want %v", got, a) + } + if _, ok := b.Get("missing"); ok { + t.Fatalf("Get(missing) returned ok=true; want false") + } +} + +func TestRegisterDuplicatePanics(t *testing.T) { + b := registry.New() + b.Register(&fakeBackend{name: "alpha"}) + defer func() { + if r := recover(); r == nil { + t.Fatalf("expected panic on duplicate register; got none") + } + }() + b.Register(&fakeBackend{name: "alpha"}) +} + +func TestNamesSorted(t *testing.T) { + b := registry.New() + b.Register(&fakeBackend{name: "zeta"}) + b.Register(&fakeBackend{name: "alpha"}) + b.Register(&fakeBackend{name: "mu"}) + + got := b.Names() + want := []string{"alpha", "mu", "zeta"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("Names() = %v; want %v", got, want) + } +} + +func TestAddFlagsFanOut(t *testing.T) { + b := registry.New() + a, c := &fakeBackend{name: "alpha"}, &fakeBackend{name: "charlie"} + b.Register(a) + b.Register(c) + + b.AddFlags(pflag.NewFlagSet("test", pflag.ContinueOnError)) + + if !a.flagsAdded || !c.flagsAdded { + t.Fatalf("AddFlags didn't fan out: alpha=%v charlie=%v", a.flagsAdded, c.flagsAdded) + } +} + +func TestValidateAndBuildReachBackend(t *testing.T) { + b := registry.New() + want := errors.New("bad config") + a := &fakeBackend{name: "alpha", validateErrors: []error{want}} + b.Register(a) + + got, _ := b.Get("alpha") + errs := got.Validate() + if a.validateCalls != 1 { + t.Fatalf("Validate not called on backend (calls=%d)", a.validateCalls) + } + if len(errs) != 1 || !errors.Is(errs[0], want) { + t.Fatalf("Validate returned %v; want [%v]", errs, want) + } + + if _, err := got.Build(); err != nil { + t.Fatalf("Build returned err=%v; want nil", err) + } + if a.buildCalls != 1 { + t.Fatalf("Build not called on backend (calls=%d)", a.buildCalls) + } +} From 2f8f16ad833833c7cfe0b8327bd4bafb63cf2528 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 16 Jun 2026 16:34:59 -0700 Subject: [PATCH 2/6] feat(decorator): add BackendFactory hook on DecoratorConfig (KPEP-0001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the seam the apiserver's RESTOptionsDecorator needs to install a registry-selected backend in place of the upstream etcd3 path. When DecoratorConfig.BackendFactory is non-nil, StorageWithClusterIdentity calls it instead of generic.NewRawStorage; the cacher wrapping above is unchanged. The BackendFactory type is declared at the top level (mirror of registry.Factory) so consumers of DecoratorConfig don't need a transitive import of the registry subpackage. Signature is 1-to-1 with upstream factory.Create — any backend already at that shape (etcd3, Spanner, future postgres) plugs in without adaptation. Nil preserves the pre-registry behavior so callers that haven't switched yet continue to hit etcd3 unchanged. --- decorator.go | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/decorator.go b/decorator.go index f34f9f9..7553db6 100644 --- a/decorator.go +++ b/decorator.go @@ -15,6 +15,20 @@ import ( "k8s.io/klog/v2" ) +// BackendFactory constructs the underlying raw storage for one resource. +// Signature matches upstream k8s.io/apiserver/pkg/storage/storagebackend/ +// factory.Create — so any backend that already satisfies that shape (etcd3, +// Spanner, future postgres) plugs in without adaptation. +// +// Mirrors registry.Factory; declared here as a top-level type so consumers +// of DecoratorConfig don't need a transitive import of the registry +// subpackage. +type BackendFactory func( + config *storagebackend.ConfigForResource, + newFunc, newListFunc func() runtime.Object, + resourcePrefix string, +) (storage.Interface, factory.DestroyFunc, error) + // DecoratorConfig configures the cluster-aware StorageDecorator. type DecoratorConfig struct { // KeyLayout defines how cluster identity is embedded in storage keys. @@ -23,6 +37,15 @@ type DecoratorConfig struct { // GroupResource identifies the resource type being stored. // Used for logging and metrics. GroupResource schema.GroupResource + + // BackendFactory, when non-nil, is used to construct the raw storage + // for each resource instead of the upstream etcd3 path + // (generic.NewRawStorage). This is the seam KPEP-0001's storage backend + // registry plugs into: the apiserver resolves --storage-backend to a + // registry.Factory and installs it here. A nil value preserves the + // pre-registry behavior so callers that don't yet use the registry + // continue to hit etcd3 unchanged. + BackendFactory BackendFactory } // StorageWithClusterIdentity returns a generic.StorageDecorator that creates @@ -88,7 +111,21 @@ func StorageWithClusterIdentity(cfg DecoratorConfig) generic.StorageDecorator { return callerKeyFunc(obj) } - s, d, err := generic.NewRawStorage(storageConfig, newFunc, newListFunc, resourcePrefix) + var s storage.Interface + var d factory.DestroyFunc + var err error + if cfg.BackendFactory != nil { + // Registered backend path (KPEP-0001). The Factory returns a + // storage.Interface + DestroyFunc with the same contract as + // upstream factory.Create. The cacher wraps it below exactly + // as it would wrap an etcd3-backed store. + s, d, err = cfg.BackendFactory(storageConfig, newFunc, newListFunc, resourcePrefix) + } else { + // Pre-registry default: upstream etcd3 path. Preserved so any + // caller that hasn't switched to setting BackendFactory keeps + // working unchanged. + s, d, err = generic.NewRawStorage(storageConfig, newFunc, newListFunc, resourcePrefix) + } if err != nil { return s, d, err } From c3b3554debc35ce863cbc70743b2427dc071d4d5 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 16 Jun 2026 16:52:35 -0700 Subject: [PATCH 3/6] chore: pin fork to feat/per-cluster-allocators (KPEP-0001 prereqs) That fork branch is the consolidated source of two prerequisite features for KPEP-0001: - 32f5e9075db: move DecodeCallback to storage package (lets non-etcd backends like Spanner honor cacher-installed decode callbacks). - 8744b93de42: per-cluster service allocator support (apiserver's multi-cluster bootstrap needs this when it threads BackendFactory through DecoratorConfig). Both consumers downstream of this PR (kplane-dev/spanner and kplane-dev/apiserver) pin against the same fork commit so the chain agrees on which symbols exist. When feat/per-cluster-allocators eventually merges, every consumer moves to the resulting main SHA in a follow-up bump. --- go.mod | 12 ++++++------ go.sum | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 45ad237..23bb5b9 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,12 @@ go 1.25.0 godebug default=go1.25 replace ( - k8s.io/api => github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260404055358-aac72e7d04ad - k8s.io/apimachinery => github.com/kplane-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260404055358-aac72e7d04ad - k8s.io/apiserver => github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260404055358-aac72e7d04ad - k8s.io/client-go => github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260404055358-aac72e7d04ad - k8s.io/component-base => github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260404055358-aac72e7d04ad - k8s.io/kms => github.com/kplane-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20260404055358-aac72e7d04ad + k8s.io/api => github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260311054814-32f5e9075db5 + k8s.io/apimachinery => github.com/kplane-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260311054814-32f5e9075db5 + k8s.io/apiserver => github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260311054814-32f5e9075db5 + k8s.io/client-go => github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260311054814-32f5e9075db5 + k8s.io/component-base => github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260311054814-32f5e9075db5 + k8s.io/kms => github.com/kplane-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20260311054814-32f5e9075db5 ) require ( diff --git a/go.sum b/go.sum index c273637..0470893 100644 --- a/go.sum +++ b/go.sum @@ -78,16 +78,16 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260404055358-aac72e7d04ad h1:0k/XM+mWOjNWWmc/HkYP5qYcIw72vssnGhS6i7jidBE= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260404055358-aac72e7d04ad/go.mod h1:KOrdwhDi3QHVXr50HAWBhO2r9uMtWbO46CiIHzRVtU8= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260404055358-aac72e7d04ad h1:hR3yKcG3v2+mjQEMME/9piLycd/Ry7JKtzVGB76satk= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260404055358-aac72e7d04ad/go.mod h1:7mgr/dli8ofwAbcIQXetFVX1fbOYsOYojq3AUbybVmQ= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260404055358-aac72e7d04ad h1:mWkFOi6/+Ttf9RzcReo+0Om1AFJDTrHzxnfIWueiTRQ= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260404055358-aac72e7d04ad/go.mod h1:GNWcUSRqjpm4i1hrLaGA7EQrl60YdahMic4aS+WUQVI= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260404055358-aac72e7d04ad h1:OhrfiUM1g6sHUohSTgiWoTwXfl1fg2He51+X0KneB8c= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260404055358-aac72e7d04ad/go.mod h1:7IM9p4c8CafSxF7ZY0F46WHylFn3o4mLVW5T1VZbaY8= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260404055358-aac72e7d04ad h1:msldiJV5olCNfzoFD7pMKFMa1D0DTMWbASyAS4fxMEs= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260404055358-aac72e7d04ad/go.mod h1:R6vYa1XRfX3PdQEGNkCaL3pt7NvLU2ti7FPzsEsA6GQ= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260311054814-32f5e9075db5 h1:VGSKAidEijlXCza8zQ6pdFZz9QczfT98wBox+qRTaxc= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260311054814-32f5e9075db5/go.mod h1:KOrdwhDi3QHVXr50HAWBhO2r9uMtWbO46CiIHzRVtU8= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260311054814-32f5e9075db5 h1:FdV1ooo4kmNti8PMOwjuzMgP+HaUjggCVW502sdMbV8= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260311054814-32f5e9075db5/go.mod h1:7mgr/dli8ofwAbcIQXetFVX1fbOYsOYojq3AUbybVmQ= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260311054814-32f5e9075db5 h1:B32q2wQIQLDhK8jhqRgRIBUZS2WoSEYYFau0dh9pl/g= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260311054814-32f5e9075db5/go.mod h1:GNWcUSRqjpm4i1hrLaGA7EQrl60YdahMic4aS+WUQVI= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260311054814-32f5e9075db5 h1:g1Cgzs09gUaCwXCtSzWJS5rFYaK1ycHKruIpwuCJJtI= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260311054814-32f5e9075db5/go.mod h1:7IM9p4c8CafSxF7ZY0F46WHylFn3o4mLVW5T1VZbaY8= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260311054814-32f5e9075db5 h1:l4RLyYStWu+BL8uKL++4byqPwLc2cv76rnHW/7A+Cvw= +github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260311054814-32f5e9075db5/go.mod h1:R6vYa1XRfX3PdQEGNkCaL3pt7NvLU2ti7FPzsEsA6GQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= From 3a873585321bf07659cec38635b1e248e8f59939 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 16 Jun 2026 17:36:08 -0700 Subject: [PATCH 4/6] feat(backends): migrate Spanner backend in-tree + add aggregator (KPEP-0001) Brings the Spanner backend into kplane-dev/storage as the first in-tree implementation, matching the KPEP-0001 end-state layout: kplane-dev/storage/ registry/ (added in previous commits) decorator.go BackendFactory = registry.Factory alias backends/ register.go RegisterBuiltin(b) aggregator spanner/ broadcast.go, config.go, factory.go, register.go, store.go, watcher.go, store_test.go, register_test.go What this collapses: - Drops the standalone kplane-dev/spanner repo entirely. It existed only because BackendFactory needed to be declared somewhere without importing kplane-dev/storage (the 'avoid an import cycle' comment in the old factory.go). Now that Spanner is a subpackage of kplane-dev/storage, it references storage.BackendFactory directly. - BackendFactory is now a type alias to registry.Factory, so the apiserver can pass registry.Backend.Build()'s return value into DecoratorConfig without a conversion. - The aggregator (backends/register.go) is one import + one Register() call per backend. Adding postgres or kine in the future means editing one file here, not the apiserver. Tested: - go test ./registry/... ./backends/... clean. - Spanner emulator-backed tests pass. - decorator.go alias compiles cleanly; the BackendFactory hook path in StorageWithClusterIdentity is unchanged. Follow-up: archive kplane-dev/spanner (or leave as a one-release re-export shim). kplane-dev/spanner#1 will be closed since its work is now here. See KPEP-0001 in kplane-dev/enhancements for the design. --- backends/register.go | 26 + backends/spanner/broadcast.go | 119 ++ backends/spanner/config.go | 177 +++ backends/spanner/factory.go | 59 + backends/spanner/register.go | 96 ++ backends/spanner/register_test.go | 99 ++ backends/spanner/store.go | 824 ++++++++++++++ backends/spanner/store_test.go | 1680 +++++++++++++++++++++++++++++ backends/spanner/watcher.go | 336 ++++++ decorator.go | 21 +- go.mod | 63 +- go.sum | 211 +++- 12 files changed, 3639 insertions(+), 72 deletions(-) create mode 100644 backends/register.go create mode 100644 backends/spanner/broadcast.go create mode 100644 backends/spanner/config.go create mode 100644 backends/spanner/factory.go create mode 100644 backends/spanner/register.go create mode 100644 backends/spanner/register_test.go create mode 100644 backends/spanner/store.go create mode 100644 backends/spanner/store_test.go create mode 100644 backends/spanner/watcher.go diff --git a/backends/register.go b/backends/register.go new file mode 100644 index 0000000..feaa28b --- /dev/null +++ b/backends/register.go @@ -0,0 +1,26 @@ +// Package backends is the aggregator: a single import the apiserver pulls +// in to register every in-tree storage backend against a *registry.Backends. +// +// Pattern mirrors upstream kubeapiserver/options/plugins.go: the apiserver +// imports this one package; the package imports each backend; each backend +// self-describes via its Options struct. Adding a backend (postgres, kine, +// etc.) means one new import + one Register() call here — no apiserver +// change required. +package backends + +import ( + "github.com/kplane-dev/storage/backends/spanner" + "github.com/kplane-dev/storage/registry" +) + +// RegisterBuiltin installs the in-tree backends into b. The apiserver +// constructs a *registry.Backends in main, calls RegisterBuiltin, then +// hands the registry to its options chain. External backends (out-of-tree) +// would call b.Register(.NewOptions()) directly from a custom +// apiserver main, after RegisterBuiltin. +func RegisterBuiltin(b *registry.Backends) { + b.Register(spanner.NewOptions()) + // Future backends: + // b.Register(postgres.NewOptions()) + // b.Register(kine.NewOptions()) +} diff --git a/backends/spanner/broadcast.go b/backends/spanner/broadcast.go new file mode 100644 index 0000000..c180d57 --- /dev/null +++ b/backends/spanner/broadcast.go @@ -0,0 +1,119 @@ +package spanner + +import ( + "sync" + + "k8s.io/klog/v2" +) + +// watchEvent is the internal representation of a storage mutation, +// published through the broadcaster to all active watchers. +type watchEvent struct { + key string + value []byte + prevValue []byte + rev int64 + + isCreated bool + isDeleted bool + isProgress bool +} + +// subscription is a channel-based subscription to broadcast events. +type subscription struct { + ch chan watchEvent + id uint64 + closed bool +} + +// Broadcaster fans out storage mutation events to all subscribed watchers. +// The write path (Create/Delete/GuaranteedUpdate) calls Publish() after each +// successful Spanner commit. This gives ~microsecond notification latency +// for in-process consumers (the cacher), avoiding the need for Change Streams +// on the hot path. +type Broadcaster struct { + mu sync.RWMutex + subscribers map[uint64]*subscription + nextID uint64 + + // highWaterMark tracks the highest revision seen, used for + // progress/bookmark events. + highWaterMark int64 +} + +// NewBroadcaster creates a new Broadcaster. +func NewBroadcaster() *Broadcaster { + return &Broadcaster{ + subscribers: make(map[uint64]*subscription), + } +} + +// Publish sends an event to all subscribers. Non-blocking: if a subscriber's +// channel is full, the subscription is closed so the watcher detects the gap +// and the cacher can re-list (same behavior as etcd closing a slow watch). +func (b *Broadcaster) Publish(e watchEvent) { + b.mu.Lock() + defer b.mu.Unlock() + + if e.rev > b.highWaterMark { + b.highWaterMark = e.rev + } + + klog.V(4).Infof("broadcaster.Publish: key=%q rev=%d isCreated=%v isDeleted=%v isProgress=%v subscribers=%d", + e.key, e.rev, e.isCreated, e.isDeleted, e.isProgress, len(b.subscribers)) + + for id, sub := range b.subscribers { + if sub.closed { + continue + } + select { + case sub.ch <- e: + default: + // Subscriber can't keep up — close it so the watcher + // exits cleanly and the cacher re-establishes the watch. + sub.closed = true + close(sub.ch) + delete(b.subscribers, id) + } + } +} + +// Subscribe creates a new subscription. The returned channel receives all +// events published after the subscription is created. bufSize controls the +// channel buffer depth. +func (b *Broadcaster) Subscribe(bufSize int) *subscription { + b.mu.Lock() + defer b.mu.Unlock() + + if bufSize < 64 { + bufSize = 64 + } + + sub := &subscription{ + ch: make(chan watchEvent, bufSize), + id: b.nextID, + } + b.nextID++ + b.subscribers[sub.id] = sub + return sub +} + +// Unsubscribe removes a subscription and closes its channel. +func (b *Broadcaster) Unsubscribe(sub *subscription) { + b.mu.Lock() + defer b.mu.Unlock() + + if sub.closed { + return + } + sub.closed = true + close(sub.ch) + delete(b.subscribers, sub.id) +} + +// HighWaterMark returns the highest revision seen by the broadcaster. +func (b *Broadcaster) HighWaterMark() int64 { + b.mu.RLock() + defer b.mu.RUnlock() + return b.highWaterMark +} diff --git a/backends/spanner/config.go b/backends/spanner/config.go new file mode 100644 index 0000000..1353203 --- /dev/null +++ b/backends/spanner/config.go @@ -0,0 +1,177 @@ +package spanner + +import ( + "context" + "fmt" + "time" + + "cloud.google.com/go/spanner" + database "cloud.google.com/go/spanner/admin/database/apiv1" + databasepb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb" + instance "cloud.google.com/go/spanner/admin/instance/apiv1" + instancepb "cloud.google.com/go/spanner/admin/instance/apiv1/instancepb" + "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "k8s.io/klog/v2" +) + +// SpannerConfig holds configuration for connecting to a Spanner instance. +type SpannerConfig struct { + // Project is the GCP project ID. + Project string + + // Instance is the Spanner instance ID. + Instance string + + // Database is the Spanner database name. + Database string + + // EmulatorHost overrides the Spanner endpoint for local development. + // When set, TLS is disabled and authentication is skipped. + // Format: "host:port" (e.g. "localhost:9010"). + EmulatorHost string +} + +// DatabasePath returns the fully qualified Spanner database path. +func (c SpannerConfig) DatabasePath() string { + return fmt.Sprintf("projects/%s/instances/%s/databases/%s", c.Project, c.Instance, c.Database) +} + +// InstancePath returns the fully qualified Spanner instance path. +func (c SpannerConfig) InstancePath() string { + return fmt.Sprintf("projects/%s/instances/%s", c.Project, c.Instance) +} + +// NewClient creates a Spanner client from the config. +func (c SpannerConfig) NewClient(ctx context.Context) (*spanner.Client, error) { + var opts []option.ClientOption + if c.EmulatorHost != "" { + opts = append(opts, + option.WithEndpoint(c.EmulatorHost), + option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), + option.WithoutAuthentication(), + ) + } + return spanner.NewClient(ctx, c.DatabasePath(), opts...) +} + +// schemaDDL returns the DDL statements for the kv table and change stream. +var schemaDDL = []string{ + `CREATE TABLE kv ( + key STRING(MAX) NOT NULL, + value BYTES(MAX) NOT NULL, + mod_ts TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp = true), + create_ts TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp = true), + lease_ttl INT64, +) PRIMARY KEY (key)`, + `CREATE CHANGE STREAM kv_changes FOR kv + OPTIONS (value_capture_type = 'OLD_AND_NEW_VALUES')`, +} + +// EnsureInstance creates the Spanner instance if it doesn't already exist. +// Intended for development / testing with the Spanner emulator. +func EnsureInstance(ctx context.Context, cfg SpannerConfig) error { + var opts []option.ClientOption + if cfg.EmulatorHost != "" { + opts = append(opts, + option.WithEndpoint(cfg.EmulatorHost), + option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), + option.WithoutAuthentication(), + ) + } + adminClient, err := instance.NewInstanceAdminClient(ctx, opts...) + if err != nil { + return fmt.Errorf("creating instance admin client: %w", err) + } + defer adminClient.Close() + + op, err := adminClient.CreateInstance(ctx, &instancepb.CreateInstanceRequest{ + Parent: fmt.Sprintf("projects/%s", cfg.Project), + InstanceId: cfg.Instance, + Instance: &instancepb.Instance{ + Config: "emulator-config", + DisplayName: cfg.Instance, + NodeCount: 1, + }, + }) + if err != nil { + // Instance may already exist — not an error. + return nil + } + _, _ = op.Wait(ctx) + return nil +} + +// EnsureSchema creates the database and applies the KV schema if it doesn't exist. +// Intended for development / testing with the Spanner emulator. +func EnsureSchema(ctx context.Context, cfg SpannerConfig) error { + var opts []option.ClientOption + if cfg.EmulatorHost != "" { + opts = append(opts, + option.WithEndpoint(cfg.EmulatorHost), + option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), + option.WithoutAuthentication(), + ) + } + + adminClient, err := database.NewDatabaseAdminClient(ctx, opts...) + if err != nil { + return fmt.Errorf("creating database admin client: %w", err) + } + defer adminClient.Close() + + // Create the database with schema in one shot. + op, err := adminClient.CreateDatabase(ctx, &databasepb.CreateDatabaseRequest{ + Parent: cfg.InstancePath(), + CreateStatement: fmt.Sprintf("CREATE DATABASE `%s`", cfg.Database), + ExtraStatements: schemaDDL, + }) + if err != nil { + return fmt.Errorf("creating database: %w", err) + } + + if _, err := op.Wait(ctx); err != nil { + return fmt.Errorf("waiting for database creation: %w", err) + } + + klog.V(2).InfoS("Spanner database created with schema", "database", cfg.DatabasePath()) + return nil +} + +// DropDatabase drops the specified Spanner database. +// Intended for test cleanup to avoid hitting emulator database limits. +func DropDatabase(ctx context.Context, cfg SpannerConfig) error { + var opts []option.ClientOption + if cfg.EmulatorHost != "" { + opts = append(opts, + option.WithEndpoint(cfg.EmulatorHost), + option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), + option.WithoutAuthentication(), + ) + } + + adminClient, err := database.NewDatabaseAdminClient(ctx, opts...) + if err != nil { + return fmt.Errorf("creating database admin client: %w", err) + } + defer adminClient.Close() + + return adminClient.DropDatabase(ctx, &databasepb.DropDatabaseRequest{ + Database: cfg.DatabasePath(), + }) +} + +// revisionFromCommitTimestamp converts a Spanner commit timestamp to a +// resource version. We use UnixNano which gives us ~292 years of headroom +// from epoch and natural monotonicity via TrueTime. +func revisionFromCommitTimestamp(ts time.Time) uint64 { + return uint64(ts.UnixNano()) +} + +// timestampFromRevision converts a resource version back to a time.Time +// for use in Spanner stale reads. +func timestampFromRevision(rv int64) time.Time { + return time.Unix(0, rv) +} diff --git a/backends/spanner/factory.go b/backends/spanner/factory.go new file mode 100644 index 0000000..aeaed7e --- /dev/null +++ b/backends/spanner/factory.go @@ -0,0 +1,59 @@ +package spanner + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage" + "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/storage/storagebackend/factory" + "k8s.io/apiserver/pkg/storage/value/encrypt/identity" + + kpstorage "github.com/kplane-dev/storage" +) + +// NewBackendFactory returns a storage.BackendFactory (defined on the top +// level of kplane-dev/storage) that creates Spanner-backed +// storage.Interface instances. Each call to the returned factory creates a +// new store sharing the same Spanner client. +// +// Pre-KPEP-0001, this lived in a standalone kplane-dev/spanner repo and +// kept its own duplicated BackendFactory type "to avoid an import cycle." +// Now that Spanner is a subpackage of kplane-dev/storage, we can reference +// storage.BackendFactory (== registry.Factory) directly. +func NewBackendFactory(cfg SpannerConfig) kpstorage.BackendFactory { + return func( + config *storagebackend.ConfigForResource, + newFunc, newListFunc func() runtime.Object, + resourcePrefix string, + ) (storage.Interface, factory.DestroyFunc, error) { + ctx := context.Background() + + client, err := cfg.NewClient(ctx) + if err != nil { + return nil, nil, err + } + + transformer := config.Transformer + if transformer == nil { + transformer = identity.NewEncryptCheckTransformer() + } + + s := NewStore( + client, + config.Codec, + newFunc, + newListFunc, + config.Prefix, + resourcePrefix, + transformer, + config.WrapDecodedObject, + ) + + destroyFunc := func() { + client.Close() + } + + return s, destroyFunc, nil + } +} diff --git a/backends/spanner/register.go b/backends/spanner/register.go new file mode 100644 index 0000000..e279a16 --- /dev/null +++ b/backends/spanner/register.go @@ -0,0 +1,96 @@ +package spanner + +import ( + "fmt" + + "github.com/spf13/pflag" + + "github.com/kplane-dev/storage/registry" +) + +// Options is the Spanner backend's flag block. It implements +// registry.Backend so an apiserver constructing a *registry.Backends can +// register Spanner via NewOptions() (and switch to it at runtime via +// --storage-backend=spanner) without the apiserver knowing anything +// Spanner-specific. +// +// Lifecycle: +// - AddFlags binds the --spanner-* flags. Called for every registered +// backend at startup so the help text lists every backend's flags. +// - Validate runs flag-level checks (required fields, sensible +// combinations). Called only when --storage-backend=spanner. +// - Build dials the shared Spanner client and returns a registry.Factory +// that creates per-resource stores against it. +// +// The Build factory delegates to NewBackendFactory (the existing +// constructor used by the apiserver's pre-registry hardcoded path), so the +// underlying store, broadcaster, and watcher behavior is identical between +// the legacy --spanner-project dispatch and the new --storage-backend +// dispatch. That lets us flip between paths during the cutover without +// behavior drift. +type Options struct { + cfg SpannerConfig +} + +// NewOptions returns a fresh Options. The apiserver registers this via +// b.Register(spanner.NewOptions()) inside its backends/RegisterBuiltin +// aggregator. +func NewOptions() *Options { + return &Options{} +} + +// Name is the value matched against --storage-backend. +func (o *Options) Name() string { return "spanner" } + +// AddFlags binds the --spanner-* flags. The flag set is shared across all +// registered backends so the apiserver's --help lists every backend's +// flags regardless of which one is selected. +func (o *Options) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.cfg.Project, "spanner-project", o.cfg.Project, + "Google Cloud project hosting the Spanner instance.") + fs.StringVar(&o.cfg.Instance, "spanner-instance", o.cfg.Instance, + "Spanner instance ID.") + fs.StringVar(&o.cfg.Database, "spanner-database", o.cfg.Database, + "Spanner database name within the instance.") + fs.StringVar(&o.cfg.EmulatorHost, "spanner-emulator-host", o.cfg.EmulatorHost, + "Optional host:port of a local Spanner emulator. When set, the project/instance/database flags still apply but credentials are skipped.") +} + +// Validate runs flag-level checks. Only invoked for the selected backend, +// so users running with --storage-backend=etcd3 (or the legacy default) +// aren't forced to set Spanner flags. +func (o *Options) Validate() []error { + var errs []error + if o.cfg.Project == "" { + errs = append(errs, fmt.Errorf("--spanner-project is required when --storage-backend=spanner")) + } + if o.cfg.Instance == "" { + errs = append(errs, fmt.Errorf("--spanner-instance is required when --storage-backend=spanner")) + } + if o.cfg.Database == "" { + errs = append(errs, fmt.Errorf("--spanner-database is required when --storage-backend=spanner")) + } + return errs +} + +// Build returns the per-resource Factory. The apiserver calls this once +// after Validate; the resulting Factory is invoked once per GroupResource +// at REST registry construction time. +// +// We reuse the existing NewBackendFactory so the legacy hardcoded path +// (apiserver's `if opts.SpannerProject != ""` branch) and the registry +// path produce identical store/broadcaster/watcher behavior. That keeps +// the Phase 3 cutover in the apiserver behavior-preserving. +func (o *Options) Build() (registry.Factory, error) { + bf := NewBackendFactory(o.cfg) + return registry.Factory(bf), nil +} + +// Compile-time assertion: Options satisfies registry.Backend. +var _ registry.Backend = (*Options)(nil) + +// Config exposes the resolved SpannerConfig for callers that need to +// inspect the backend's wiring (e.g. health-probe construction, debug +// endpoints). Returns the value, not a pointer, so callers can't mutate +// the live config after Build. +func (o *Options) Config() SpannerConfig { return o.cfg } diff --git a/backends/spanner/register_test.go b/backends/spanner/register_test.go new file mode 100644 index 0000000..5c36526 --- /dev/null +++ b/backends/spanner/register_test.go @@ -0,0 +1,99 @@ +package spanner_test + +import ( + "strings" + "testing" + + "github.com/spf13/pflag" + + "github.com/kplane-dev/storage/backends/spanner" + "github.com/kplane-dev/storage/registry" +) + +func TestOptionsImplementsBackend(t *testing.T) { + // Compile-time check that *Options implements registry.Backend. + // Repeated here as a runtime assertion so a future signature drift + // fails the test instead of silently breaking the registry contract. + var _ registry.Backend = spanner.NewOptions() + + if got := spanner.NewOptions().Name(); got != "spanner" { + t.Fatalf("Name() = %q; want %q", got, "spanner") + } +} + +func TestAddFlagsBindsSpannerPrefixed(t *testing.T) { + o := spanner.NewOptions() + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + o.AddFlags(fs) + + want := []string{ + "spanner-project", + "spanner-instance", + "spanner-database", + "spanner-emulator-host", + } + for _, name := range want { + if fs.Lookup(name) == nil { + t.Errorf("--%s not bound by AddFlags", name) + } + } +} + +func TestValidateRequiresCoreFields(t *testing.T) { + o := spanner.NewOptions() + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + o.AddFlags(fs) + + errs := o.Validate() + if len(errs) != 3 { + t.Fatalf("Validate() returned %d errors; want 3 (missing project/instance/database). errs=%v", len(errs), errs) + } + for _, want := range []string{"spanner-project", "spanner-instance", "spanner-database"} { + found := false + for _, e := range errs { + if strings.Contains(e.Error(), want) { + found = true + break + } + } + if !found { + t.Errorf("Validate() missing error mentioning --%s; got %v", want, errs) + } + } +} + +func TestValidatePassesWhenFlagsSet(t *testing.T) { + o := spanner.NewOptions() + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + o.AddFlags(fs) + + if err := fs.Parse([]string{ + "--spanner-project=p", + "--spanner-instance=i", + "--spanner-database=d", + }); err != nil { + t.Fatalf("flag parse: %v", err) + } + + if errs := o.Validate(); len(errs) != 0 { + t.Fatalf("Validate() returned %v; want none", errs) + } + + cfg := o.Config() + if cfg.Project != "p" || cfg.Instance != "i" || cfg.Database != "d" { + t.Fatalf("Config() did not capture flag values: %+v", cfg) + } +} + +func TestRegisterRoundtripsThroughRegistry(t *testing.T) { + b := registry.New() + b.Register(spanner.NewOptions()) + + got, ok := b.Get("spanner") + if !ok { + t.Fatalf("Get(spanner) ok=false; want true") + } + if _, isOptions := got.(*spanner.Options); !isOptions { + t.Fatalf("Get(spanner) returned %T; want *spanner.Options", got) + } +} diff --git a/backends/spanner/store.go b/backends/spanner/store.go new file mode 100644 index 0000000..850f62a --- /dev/null +++ b/backends/spanner/store.go @@ -0,0 +1,824 @@ +package spanner + +import ( + "bytes" + "context" + "fmt" + "path" + "reflect" + "strings" + "time" + + "cloud.google.com/go/spanner" + "google.golang.org/api/iterator" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/conversion" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/storage" + "k8s.io/apiserver/pkg/storage/value" + "k8s.io/klog/v2" +) + +// authenticatedDataString satisfies the value.Context interface. +// Uses the storage key to authenticate the stored data, same pattern as etcd3. +type authenticatedDataString string + +func (d authenticatedDataString) AuthenticatedData() []byte { + return []byte(string(d)) +} + +var _ value.Context = authenticatedDataString("") + +// store implements storage.Interface backed by Cloud Spanner. +type store struct { + client *spanner.Client + codec runtime.Codec + versioner storage.Versioner + transformer value.Transformer + + pathPrefix string + resourcePrefix string + groupResource string + newFunc func() runtime.Object + newListFunc func() runtime.Object + + broadcaster *Broadcaster + + // wrapDecodedObject wraps decoded objects with their storage key. + // Used by multicluster caching to carry key identity through the + // watch.Event boundary. nil for single-cluster deployments. + wrapDecodedObject func(obj runtime.Object, key string) runtime.Object +} + +var _ storage.Interface = (*store)(nil) + +// NewStore creates a new Spanner-backed storage.Interface. +func NewStore( + client *spanner.Client, + codec runtime.Codec, + newFunc, newListFunc func() runtime.Object, + prefix, resourcePrefix string, + transformer value.Transformer, + wrapDecodedObject func(obj runtime.Object, key string) runtime.Object, +) *store { + pathPrefix := path.Join("/", prefix) + if !strings.HasSuffix(pathPrefix, "/") { + pathPrefix += "/" + } + + return &store{ + client: client, + codec: codec, + versioner: storage.APIObjectVersioner{}, + transformer: transformer, + pathPrefix: pathPrefix, + resourcePrefix: resourcePrefix, + groupResource: resourcePrefix, + newFunc: newFunc, + newListFunc: newListFunc, + broadcaster: NewBroadcaster(), + wrapDecodedObject: wrapDecodedObject, + } +} + +func (s *store) Versioner() storage.Versioner { + return s.versioner +} + +// prepareKey validates and normalizes the storage key. +// Rejects path traversal attacks (.. and .), empty keys, and keys that +// don't start with the expected path prefix. +func (s *store) prepareKey(key string) (string, error) { + if key == ".." || + strings.HasPrefix(key, "../") || + strings.HasSuffix(key, "/..") || + strings.Contains(key, "/../") { + return "", fmt.Errorf("invalid key: %q", key) + } + if key == "." || + strings.HasPrefix(key, "./") || + strings.HasSuffix(key, "/.") || + strings.Contains(key, "/./") { + return "", fmt.Errorf("invalid key: %q", key) + } + if key == "" || key == "/" { + return "", fmt.Errorf("empty key: %q", key) + } + if strings.HasPrefix(key, s.pathPrefix) { + return key, nil + } + return s.pathPrefix + key, nil +} + +func (s *store) storageKeyFromSpannerKey(spannerKey string) string { + return strings.TrimPrefix(spannerKey, s.pathPrefix) +} + +// Create adds a new object at the given key. +func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error { + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } + + if version, err := s.versioner.ObjectResourceVersion(obj); err == nil && version != 0 { + return storage.ErrResourceVersionSetOnCreate + } + if err := s.versioner.PrepareObjectForStorage(obj); err != nil { + return fmt.Errorf("PrepareObjectForStorage failed: %v", err) + } + + data, err := runtime.Encode(s.codec, obj) + if err != nil { + return err + } + + newData, err := s.transformer.TransformToStorage(ctx, data, authenticatedDataString(preparedKey)) + if err != nil { + return storage.NewInternalError(err) + } + + cols := map[string]interface{}{ + "key": preparedKey, + "value": newData, + "mod_ts": spanner.CommitTimestamp, + "create_ts": spanner.CommitTimestamp, + } + if ttl != 0 { + cols["lease_ttl"] = int64(ttl) + } + // Use Apply with Insert (not InsertOrUpdate) so Spanner rejects + // duplicates via PRIMARY KEY constraint — avoids a ReadRow round trip. + commitTs, err := s.client.Apply(ctx, []*spanner.Mutation{spanner.InsertMap("kv", cols)}) + if err != nil { + if status.Code(err) == codes.AlreadyExists { + return storage.NewKeyExistsError(preparedKey, 0) + } + return err + } + + if out != nil { + if err := decode(s.codec, s.versioner, data, out, revisionFromCommitTimestamp(commitTs)); err != nil { + return err + } + } + + // Publish watch event. + s.broadcaster.Publish(watchEvent{ + key: preparedKey, + value: newData, + rev: int64(revisionFromCommitTimestamp(commitTs)), + isCreated: true, + }) + + return nil +} + +// Get retrieves the object at the given key. +func (s *store) Get(ctx context.Context, key string, opts storage.GetOptions, out runtime.Object) error { + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } + + // Honor ResourceVersion: stale read at exact timestamp, strong read otherwise. + txn := s.client.Single() + if opts.ResourceVersion != "" { + if parsed, err := s.versioner.ParseResourceVersion(opts.ResourceVersion); err == nil && parsed > 0 { + txn = s.client.Single().WithTimestampBound(spanner.ReadTimestamp(timestampFromRevision(int64(parsed)))) + } + } + defer txn.Close() + + row, err := txn.ReadRow(ctx, "kv", spanner.Key{preparedKey}, []string{"value", "mod_ts"}) + if err != nil { + if spanner.ErrCode(err) == 5 { // NOT_FOUND + if opts.IgnoreNotFound { + return runtime.SetZeroValue(out) + } + return storage.NewKeyNotFoundError(preparedKey, 0) + } + return err + } + + var val []byte + var modTs time.Time + if err := row.Columns(&val, &modTs); err != nil { + return err + } + + data, _, err := s.transformer.TransformFromStorage(ctx, val, authenticatedDataString(preparedKey)) + if err != nil { + return storage.NewInternalError(err) + } + + return decode(s.codec, s.versioner, data, out, revisionFromCommitTimestamp(modTs)) +} + +// Delete removes the object at the given key. +func (s *store) Delete( + ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions, + validateDeletion storage.ValidateObjectFunc, cachedExistingObject runtime.Object, opts storage.DeleteOptions) error { + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } + + v, err := conversion.EnforcePtr(out) + if err != nil { + return fmt.Errorf("unable to convert output object to pointer: %v", err) + } + _ = v + + var oldData []byte // decrypted, for decoding into out + var oldEncData []byte // encrypted, for watch event prevValue + + commitTs, err := s.client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { + row, err := txn.ReadRow(ctx, "kv", spanner.Key{preparedKey}, []string{"value", "mod_ts"}) + if err != nil { + if spanner.ErrCode(err) == 5 { // NOT_FOUND + return storage.NewKeyNotFoundError(preparedKey, 0) + } + return err + } + + var val []byte + var modTs time.Time + if err := row.Columns(&val, &modTs); err != nil { + return err + } + + // Use cachedExistingObject to skip decrypt+decode when the RV matches. + var existing runtime.Object + var data []byte + if cachedExistingObject != nil { + if cachedRV, rvErr := s.versioner.ObjectResourceVersion(cachedExistingObject); rvErr == nil && cachedRV == revisionFromCommitTimestamp(modTs) { + existing = cachedExistingObject + } + } + if existing == nil { + data, _, err = s.transformer.TransformFromStorage(ctx, val, authenticatedDataString(preparedKey)) + if err != nil { + return storage.NewInternalError(err) + } + existing = s.newFunc() + if err := decode(s.codec, s.versioner, data, existing, revisionFromCommitTimestamp(modTs)); err != nil { + return err + } + } + + if preconditions != nil { + if err := preconditions.Check(preparedKey, existing); err != nil { + return err + } + } + if err := validateDeletion(ctx, existing); err != nil { + return err + } + + // We always need the decrypted data for the out object. + if data == nil { + data, _, err = s.transformer.TransformFromStorage(ctx, val, authenticatedDataString(preparedKey)) + if err != nil { + return storage.NewInternalError(err) + } + } + oldData = data + oldEncData = val + txn.BufferWrite([]*spanner.Mutation{spanner.Delete("kv", spanner.Key{preparedKey})}) + return nil + }) + if err != nil { + return err + } + + rv := revisionFromCommitTimestamp(commitTs) + + // Decode the deleted object into out. + if err := decode(s.codec, s.versioner, oldData, out, rv); err != nil { + return err + } + + s.broadcaster.Publish(watchEvent{ + key: preparedKey, + prevValue: oldEncData, + rev: int64(rv), + isDeleted: true, + }) + + return nil +} + +// Watch starts watching at the given key. +func (s *store) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) { + preparedKey, err := s.prepareKey(key) + if err != nil { + return nil, err + } + if opts.Recursive && !strings.HasSuffix(preparedKey, "/") { + preparedKey += "/" + } + + rev := int64(0) + if opts.ResourceVersion != "" { + parsed, err := s.versioner.ParseResourceVersion(opts.ResourceVersion) + if err != nil { + return nil, err + } + rev = int64(parsed) + } + + w := newWatcher(s, preparedKey, opts, rev) + return w, nil +} + +// GetList retrieves a list of objects matching the key prefix. +func (s *store) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } + if opts.Recursive && !strings.HasSuffix(preparedKey, "/") { + preparedKey += "/" + } + + listPtr, err := meta.GetItemsPtr(listObj) + if err != nil { + return err + } + v, err := conversion.EnforcePtr(listPtr) + if err != nil || v.Kind() != reflect.Slice { + return fmt.Errorf("need ptr to slice: %v", err) + } + + withRev, continueKey, err := storage.ValidateListOptions(preparedKey, s.versioner, opts) + if err != nil { + return err + } + + // Build the read transaction (strong or stale). + var txn *spanner.ReadOnlyTransaction + if withRev > 0 { + txn = s.client.Single().WithTimestampBound(spanner.ReadTimestamp(timestampFromRevision(withRev))) + } else { + txn = s.client.Single() + } + defer txn.Close() + + // Build key range for prefix scan. + var keyRange spanner.KeyRange + if continueKey != "" { + // Continue from the last key (exclusive). + keyRange = spanner.KeyRange{ + Start: spanner.Key{continueKey}, + End: spanner.Key{prefixEnd(preparedKey)}, + Kind: spanner.ClosedOpen, + } + } else if opts.Recursive { + keyRange = spanner.KeyRange{ + Start: spanner.Key{preparedKey}, + End: spanner.Key{prefixEnd(preparedKey)}, + Kind: spanner.ClosedOpen, + } + } else { + // Non-recursive: exact key match. + row, err := txn.ReadRow(ctx, "kv", spanner.Key{preparedKey}, []string{"key", "value", "mod_ts"}) + if err != nil { + if spanner.ErrCode(err) == 5 { // NOT_FOUND + if v.IsNil() { + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) + } + return s.versioner.UpdateList(listObj, uint64(withRev), "", nil) + } + return err + } + var rowKey string + var val []byte + var modTs time.Time + if err := row.Columns(&rowKey, &val, &modTs); err != nil { + return err + } + data, _, err := s.transformer.TransformFromStorage(ctx, val, authenticatedDataString(rowKey)) + if err != nil { + return storage.NewInternalError(err) + } + obj := s.newFunc() + if err := decode(s.codec, s.versioner, data, obj, revisionFromCommitTimestamp(modTs)); err != nil { + return err + } + + // Notify decode callback for identity resolution. + if cb := storage.DecodeCallbackFromContext(ctx); cb != nil { + cb(obj, s.storageKeyFromSpannerKey(rowKey), int64(revisionFromCommitTimestamp(modTs))) + } + + if matched, err := opts.Predicate.Matches(obj); err == nil && matched { + v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem())) + } + // Use transaction read timestamp for consistency with recursive path. + readRV := revisionFromCommitTimestamp(modTs) + if ts, tsErr := txn.Timestamp(); tsErr == nil { + readRV = revisionFromCommitTimestamp(ts) + } + return s.versioner.UpdateList(listObj, readRV, "", nil) + } + + limit := opts.Predicate.Limit + paging := limit > 0 + + iter := txn.Read(ctx, "kv", spanner.KeySets(keyRange), []string{"key", "value", "mod_ts"}) + defer iter.Stop() + + var lastKey string + var count int64 + + for { + // Check if the request context has been cancelled or timed out. + select { + case <-ctx.Done(): + return storage.NewTimeoutError(preparedKey, "request did not complete within requested timeout") + default: + } + + row, err := iter.Next() + if err != nil { + if err == iterator.Done { + break + } + return err + } + + var rowKey string + var val []byte + var modTs time.Time + if err := row.Columns(&rowKey, &val, &modTs); err != nil { + return err + } + + rv := revisionFromCommitTimestamp(modTs) + + data, _, err := s.transformer.TransformFromStorage(ctx, val, authenticatedDataString(rowKey)) + if err != nil { + return storage.NewInternalError(err) + } + + obj := s.newFunc() + if err := decode(s.codec, s.versioner, data, obj, rv); err != nil { + klog.Errorf("failed to decode object at key %s: %v", rowKey, err) + continue + } + + // Notify decode callback for identity resolution. + if cb := storage.DecodeCallbackFromContext(ctx); cb != nil { + cb(obj, s.storageKeyFromSpannerKey(rowKey), int64(rv)) + } + + if matched, matchErr := opts.Predicate.Matches(obj); matchErr == nil && matched { + v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem())) + } + + count++ + lastKey = rowKey + + if paging && int64(v.Len()) >= limit { + break + } + } + + if v.IsNil() { + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) + } + + // Use the transaction's read timestamp as the list RV. This ensures + // paginated continuations read at the same snapshot, even if some items + // have later mod_ts than the ones returned on this page. + var listRV uint64 + if withRev > 0 { + listRV = uint64(withRev) + } else { + readTs, err := txn.Timestamp() + if err == nil { + listRV = revisionFromCommitTimestamp(readTs) + } + // Fallback: if no rows were read, txn.Timestamp() may fail. + // Use current timestamp to ensure a non-zero RV. + if listRV == 0 { + if ts, tsErr := s.getCurrentTimestamp(ctx); tsErr == nil { + listRV = revisionFromCommitTimestamp(ts) + } + } + } + + var continueValue string + var remainingItemCount *int64 + if paging && int64(v.Len()) >= limit && lastKey != "" { + hasMore := true + continueValue, remainingItemCount, err = storage.PrepareContinueToken(lastKey, preparedKey, int64(listRV), count, hasMore, opts) + if err != nil { + return err + } + } + + return s.versioner.UpdateList(listObj, listRV, continueValue, remainingItemCount) +} + +// GuaranteedUpdate implements optimistic concurrency control via Spanner transactions. +func (s *store) GuaranteedUpdate( + ctx context.Context, key string, destination runtime.Object, ignoreNotFound bool, + preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, cachedExistingObject runtime.Object) error { + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } + + v, err := conversion.EnforcePtr(destination) + if err != nil { + return fmt.Errorf("unable to convert output object to pointer: %v", err) + } + _ = v + + transformContext := authenticatedDataString(preparedKey) + + for { + var newData []byte // codec-encoded (unencrypted), for decoding into destination + var newEncData []byte // encrypted, for watch event value + var origEncData []byte // encrypted, for watch event prevValue + var noopRev int64 // set when data is unchanged (no-op), holds existing mod_ts + var created bool // true when object didn't exist (upsert) + var conflict bool // set when tryUpdate returns a retriable conflict + + commitTs, err := s.client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { + row, readErr := txn.ReadRow(ctx, "kv", spanner.Key{preparedKey}, []string{"value", "mod_ts"}) + + var origObj runtime.Object + var origRev int64 + var origData []byte // decrypted, for no-op comparison + + if readErr != nil { + if spanner.ErrCode(readErr) != 5 { // NOT_FOUND + return readErr + } + if !ignoreNotFound { + return storage.NewKeyNotFoundError(preparedKey, 0) + } + origObj = s.newFunc() + } else { + var val []byte + var modTs time.Time + if err := row.Columns(&val, &modTs); err != nil { + return err + } + origRev = int64(revisionFromCommitTimestamp(modTs)) + origEncData = val + + // Use cachedExistingObject to skip decrypt+decode when RV matches. + // origData stays nil, which disables no-op detection for this attempt — + // acceptable tradeoff since we already saved the decrypt+decode cost. + if cachedExistingObject != nil { + if cachedRV, rvErr := s.versioner.ObjectResourceVersion(cachedExistingObject); rvErr == nil && cachedRV == uint64(origRev) { + origObj = cachedExistingObject + } + } + if origObj == nil { + data, stale, err := s.transformer.TransformFromStorage(ctx, val, transformContext) + if err != nil { + return storage.NewInternalError(err) + } + origObj = s.newFunc() + if err := decode(s.codec, s.versioner, data, origObj, revisionFromCommitTimestamp(modTs)); err != nil { + return err + } + // Only enable no-op detection when the stored data is fresh. + // If stale (e.g. key rotation), force a re-write even if + // the decoded content is identical. + if !stale { + origData = data + } + } + } + + if preconditions != nil { + if err := preconditions.Check(preparedKey, origObj); err != nil { + return err + } + } + + ret, ttl, err := tryUpdate(origObj, storage.ResponseMeta{ResourceVersion: uint64(origRev)}) + if err != nil { + if apierrors.IsConflict(err) { + // Signal the outer loop to retry with a fresh read. + conflict = true + return err + } + return err + } + + if err := s.versioner.PrepareObjectForStorage(ret); err != nil { + return fmt.Errorf("PrepareObjectForStorage failed: %v", err) + } + + data, err := runtime.Encode(s.codec, ret) + if err != nil { + return err + } + + // No-op detection: if data is identical, skip write. + if origData != nil && bytes.Equal(data, origData) { + newData = data + noopRev = origRev + return nil + } + + encrypted, err := s.transformer.TransformToStorage(ctx, data, transformContext) + if err != nil { + return storage.NewInternalError(err) + } + + cols := map[string]interface{}{ + "key": preparedKey, + "value": encrypted, + "mod_ts": spanner.CommitTimestamp, + } + if origRev == 0 { + // Object didn't exist — insert with create_ts. + cols["create_ts"] = spanner.CommitTimestamp + } + if ttl != nil && *ttl != 0 { + cols["lease_ttl"] = int64(*ttl) + } + + var m *spanner.Mutation + if origRev == 0 { + m = spanner.InsertOrUpdateMap("kv", cols) + } else { + m = spanner.UpdateMap("kv", cols) + } + txn.BufferWrite([]*spanner.Mutation{m}) + + newData = data + newEncData = encrypted + created = origRev == 0 + return nil + }) + if err != nil { + if conflict { + // Retry: re-read current state and call tryUpdate again. + conflict = false + continue + } + return err + } + + // Use existing mod_ts for no-op, commit timestamp for actual writes. + var rv uint64 + if noopRev > 0 { + rv = uint64(noopRev) + } else { + rv = revisionFromCommitTimestamp(commitTs) + } + + if err := decode(s.codec, s.versioner, newData, destination, rv); err != nil { + return err + } + + // Only publish watch event if data actually changed. + if noopRev == 0 { + s.broadcaster.Publish(watchEvent{ + key: preparedKey, + value: newEncData, + prevValue: origEncData, + rev: int64(rv), + isCreated: created, + }) + } + + return nil + } +} + +// Stats returns storage statistics. +func (s *store) Stats(ctx context.Context) (storage.Stats, error) { + stmt := spanner.Statement{ + SQL: "SELECT COUNT(*) as cnt FROM kv WHERE STARTS_WITH(key, @prefix)", + Params: map[string]interface{}{"prefix": s.pathPrefix + s.resourcePrefix}, + } + iter := s.client.Single().Query(ctx, stmt) + defer iter.Stop() + + row, err := iter.Next() + if err != nil { + return storage.Stats{}, err + } + var count int64 + if err := row.Columns(&count); err != nil { + return storage.Stats{}, err + } + return storage.Stats{ObjectCount: count}, nil +} + +// ReadinessCheck verifies Spanner connectivity. +func (s *store) ReadinessCheck() error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + iter := s.client.Single().Query(ctx, spanner.Statement{SQL: "SELECT 1"}) + defer iter.Stop() + _, err := iter.Next() + return err +} + +// RequestWatchProgress publishes a synthetic bookmark event. +func (s *store) RequestWatchProgress(ctx context.Context) error { + ts, err := s.getCurrentTimestamp(ctx) + if err != nil { + return err + } + s.broadcaster.Publish(watchEvent{ + rev: int64(revisionFromCommitTimestamp(ts)), + isProgress: true, + }) + return nil +} + +// GetCurrentResourceVersion returns the latest resource version. +func (s *store) GetCurrentResourceVersion(ctx context.Context) (uint64, error) { + ts, err := s.getCurrentTimestamp(ctx) + if err != nil { + return 0, err + } + rv := revisionFromCommitTimestamp(ts) + if rv == 0 { + return 0, fmt.Errorf("the current resource version must be greater than 0") + } + return rv, nil +} + +// EnableResourceSizeEstimation is a no-op for Spanner. +func (s *store) EnableResourceSizeEstimation(storage.KeysFunc) error { + return nil +} + +// CompactRevision returns 0 — Spanner handles data lifecycle via row deletion policies. +func (s *store) CompactRevision() int64 { + return 0 +} + +// getModTimestamp reads the current mod_ts for a key. +func (s *store) getModTimestamp(ctx context.Context, key string) (time.Time, error) { + row, err := s.client.Single().ReadRow(ctx, "kv", spanner.Key{key}, []string{"mod_ts"}) + if err != nil { + return time.Time{}, err + } + var ts time.Time + if err := row.Column(0, &ts); err != nil { + return time.Time{}, err + } + return ts, nil +} + +// getCurrentTimestamp gets the current Spanner timestamp. +func (s *store) getCurrentTimestamp(ctx context.Context) (time.Time, error) { + iter := s.client.Single().Query(ctx, spanner.Statement{SQL: "SELECT CURRENT_TIMESTAMP()"}) + defer iter.Stop() + row, err := iter.Next() + if err != nil { + return time.Time{}, err + } + var ts time.Time + if err := row.Column(0, &ts); err != nil { + return time.Time{}, err + } + return ts, nil +} + +// decode decodes data into out and sets the resource version. +func decode(codec runtime.Codec, versioner storage.Versioner, data []byte, out runtime.Object, rv uint64) error { + if _, _, err := codec.Decode(data, nil, out); err != nil { + return err + } + if rv != 0 { + if err := versioner.UpdateObject(out, rv); err != nil { + return err + } + } + return nil +} + +// prefixEnd returns the key that is just past all keys with the given prefix. +func prefixEnd(prefix string) string { + if len(prefix) == 0 { + return "" + } + end := []byte(prefix) + for i := len(end) - 1; i >= 0; i-- { + if end[i] < 0xff { + end[i]++ + return string(end[:i+1]) + } + } + return "" // prefix is all 0xff +} diff --git a/backends/spanner/store_test.go b/backends/spanner/store_test.go new file mode 100644 index 0000000..96f3d07 --- /dev/null +++ b/backends/spanner/store_test.go @@ -0,0 +1,1680 @@ +package spanner + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + instance "cloud.google.com/go/spanner/admin/instance/apiv1" + instancepb "cloud.google.com/go/spanner/admin/instance/apiv1/instancepb" + "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/storage" + "k8s.io/apiserver/pkg/storage/value/encrypt/identity" +) + +// testObj is a minimal runtime.Object for testing. +type testObj struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Data string `json:"data,omitempty"` +} + +func (t *testObj) DeepCopyObject() runtime.Object { + return &testObj{ + TypeMeta: t.TypeMeta, + ObjectMeta: *t.ObjectMeta.DeepCopy(), + Data: t.Data, + } +} + +type testObjList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []testObj `json:"items"` +} + +func (t *testObjList) DeepCopyObject() runtime.Object { + out := &testObjList{ + TypeMeta: t.TypeMeta, + ListMeta: *t.ListMeta.DeepCopy(), + } + for _, item := range t.Items { + out.Items = append(out.Items, *item.DeepCopyObject().(*testObj)) + } + return out +} + +var ( + testScheme = runtime.NewScheme() + testCodecs serializer.CodecFactory + testGVK = schema.GroupVersionKind{Group: "test.io", Version: "v1", Kind: "TestObj"} + testGVR = schema.GroupVersionResource{Group: "test.io", Version: "v1", Resource: "testobjs"} +) + +func init() { + schemeBuilder := runtime.NewSchemeBuilder(func(s *runtime.Scheme) error { + s.AddKnownTypeWithName(testGVK, &testObj{}) + s.AddKnownTypeWithName(schema.GroupVersionKind{Group: "test.io", Version: "v1", Kind: "TestObjList"}, &testObjList{}) + metav1.AddToGroupVersion(s, schema.GroupVersion{Group: "test.io", Version: "v1"}) + return nil + }) + utilruntime.Must(schemeBuilder.AddToScheme(testScheme)) + testCodecs = serializer.NewCodecFactory(testScheme) +} + +func emulatorHost() string { + if h := os.Getenv("SPANNER_EMULATOR_HOST"); h != "" { + return h + } + return "localhost:9010" +} + +func skipIfNoEmulator(t *testing.T) { + t.Helper() + host := emulatorHost() + // Quick connection test. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + conn, err := grpc.DialContext(ctx, host, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock()) + if err != nil { + t.Skipf("Spanner emulator not available at %s: %v", host, err) + } + conn.Close() +} + +func setupTestStore(t *testing.T) *store { + t.Helper() + skipIfNoEmulator(t) + + ctx := context.Background() + host := emulatorHost() + dbName := fmt.Sprintf("testdb_%d", time.Now().UnixNano()) + + cfg := SpannerConfig{ + Project: "test-project", + Instance: "test-instance", + Database: dbName, + EmulatorHost: host, + } + + // Create instance if not exists. + opts := []option.ClientOption{ + option.WithEndpoint(host), + option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), + option.WithoutAuthentication(), + } + adminClient, err := instance.NewInstanceAdminClient(ctx, opts...) + if err != nil { + t.Fatalf("creating instance admin client: %v", err) + } + defer adminClient.Close() + + op, err := adminClient.CreateInstance(ctx, &instancepb.CreateInstanceRequest{ + Parent: "projects/test-project", + InstanceId: "test-instance", + Instance: &instancepb.Instance{ + Config: "emulator-config", + DisplayName: "Test Instance", + NodeCount: 1, + }, + }) + if err == nil { + if _, err := op.Wait(ctx); err != nil { + // Instance may already exist, ignore. + _ = err + } + } + + // Create database with schema. + if err := EnsureSchema(ctx, cfg); err != nil { + t.Fatalf("EnsureSchema: %v", err) + } + + client, err := cfg.NewClient(ctx) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + t.Cleanup(func() { client.Close() }) + + codec := testCodecs.LegacyCodec(schema.GroupVersion{Group: "test.io", Version: "v1"}) + + s := NewStore( + client, + codec, + func() runtime.Object { return &testObj{} }, + func() runtime.Object { return &testObjList{} }, + "/registry", + "/testobjs", + identity.NewEncryptCheckTransformer(), + nil, + ) + + return s +} + +func TestCreate(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test1", + Namespace: "default", + }, + Data: "hello", + } + + out := &testObj{} + err := s.Create(ctx, "/testobjs/default/test1", obj, out, 0) + if err != nil { + t.Fatalf("Create: %v", err) + } + + if out.Name != "test1" { + t.Errorf("expected name test1, got %s", out.Name) + } + if out.ResourceVersion == "" { + t.Error("expected non-empty resource version") + } + if out.Data != "hello" { + t.Errorf("expected data hello, got %s", out.Data) + } + + // Create duplicate should fail. + err = s.Create(ctx, "/testobjs/default/test1", obj, out, 0) + if !storage.IsExist(err) { + t.Errorf("expected key exists error, got: %v", err) + } +} + +func TestCreateRejectsNonZeroRV(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "rv-test", + Namespace: "default", + ResourceVersion: "12345", + }, + } + + out := &testObj{} + err := s.Create(ctx, "/testobjs/default/rv-test", obj, out, 0) + if err != storage.ErrResourceVersionSetOnCreate { + t.Errorf("expected ErrResourceVersionSetOnCreate, got: %v", err) + } +} + +func TestGet(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create an object first. + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "get-test", + Namespace: "default", + }, + Data: "world", + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/get-test", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // Get it back. + got := &testObj{} + if err := s.Get(ctx, "/testobjs/default/get-test", storage.GetOptions{}, got); err != nil { + t.Fatalf("Get: %v", err) + } + if got.Name != "get-test" { + t.Errorf("expected name get-test, got %s", got.Name) + } + if got.Data != "world" { + t.Errorf("expected data world, got %s", got.Data) + } + + // Get non-existent key. + notFound := &testObj{} + err := s.Get(ctx, "/testobjs/default/nonexistent", storage.GetOptions{}, notFound) + if !storage.IsNotFound(err) { + t.Errorf("expected not found error, got: %v", err) + } + + // Get non-existent key with IgnoreNotFound. + notFound2 := &testObj{} + err = s.Get(ctx, "/testobjs/default/nonexistent", storage.GetOptions{IgnoreNotFound: true}, notFound2) + if err != nil { + t.Errorf("expected no error with IgnoreNotFound, got: %v", err) + } +} + +func TestGetWithResourceVersion(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "rv-get", + Namespace: "default", + }, + Data: "v1", + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/rv-get", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + createRV := created.ResourceVersion + + // Update the object. + updated := &testObj{} + if err := s.GuaranteedUpdate(ctx, "/testobjs/default/rv-get", updated, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Data = "v2" + return existing, nil, nil + }, nil); err != nil { + t.Fatalf("GuaranteedUpdate: %v", err) + } + + // Get at the old RV should return v1. + got := &testObj{} + if err := s.Get(ctx, "/testobjs/default/rv-get", storage.GetOptions{ResourceVersion: createRV}, got); err != nil { + t.Fatalf("Get at old RV: %v", err) + } + if got.Data != "v1" { + t.Errorf("expected data v1 at old RV, got %s", got.Data) + } + + // Get without RV (strong read) should return v2. + got2 := &testObj{} + if err := s.Get(ctx, "/testobjs/default/rv-get", storage.GetOptions{}, got2); err != nil { + t.Fatalf("Get strong: %v", err) + } + if got2.Data != "v2" { + t.Errorf("expected data v2 on strong read, got %s", got2.Data) + } + + // Get with RV "0" should do strong read (v2). + got3 := &testObj{} + if err := s.Get(ctx, "/testobjs/default/rv-get", storage.GetOptions{ResourceVersion: "0"}, got3); err != nil { + t.Fatalf("Get RV=0: %v", err) + } + if got3.Data != "v2" { + t.Errorf("expected data v2 for RV=0, got %s", got3.Data) + } +} + +func TestDeleteWithCachedObject(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "cached-del", + Namespace: "default", + UID: "the-uid", + }, + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/cached-del", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // Delete with cachedExistingObject — should use it for precondition check. + correctUID := types.UID("the-uid") + deleted := &testObj{} + err := s.Delete(ctx, "/testobjs/default/cached-del", deleted, &storage.Preconditions{UID: &correctUID}, + storage.ValidateAllObjectFunc, created, storage.DeleteOptions{}) + if err != nil { + t.Fatalf("Delete with cached object: %v", err) + } + if deleted.Name != "cached-del" { + t.Errorf("expected name cached-del, got %s", deleted.Name) + } +} + +func TestGuaranteedUpdateWithCachedObject(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "cached-update", + Namespace: "default", + }, + Data: "original", + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/cached-update", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // Update with cachedExistingObject — should use it for tryUpdate. + dest := &testObj{} + err := s.GuaranteedUpdate(ctx, "/testobjs/default/cached-update", dest, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Data = "via-cached" + return existing, nil, nil + }, created) + if err != nil { + t.Fatalf("GuaranteedUpdate with cached: %v", err) + } + if dest.Data != "via-cached" { + t.Errorf("expected data via-cached, got %s", dest.Data) + } + if dest.ResourceVersion == created.ResourceVersion { + t.Error("expected RV to change after update") + } + + // Verify via Get. + got := &testObj{} + if err := s.Get(ctx, "/testobjs/default/cached-update", storage.GetOptions{}, got); err != nil { + t.Fatalf("Get: %v", err) + } + if got.Data != "via-cached" { + t.Errorf("expected data via-cached, got %s", got.Data) + } +} + +func TestDelete(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "del-test", + Namespace: "default", + }, + Data: "to-delete", + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/del-test", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + deleted := &testObj{} + err := s.Delete(ctx, "/testobjs/default/del-test", deleted, nil, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}) + if err != nil { + t.Fatalf("Delete: %v", err) + } + if deleted.Name != "del-test" { + t.Errorf("expected deleted object name del-test, got %s", deleted.Name) + } + if deleted.ResourceVersion == "" { + t.Error("expected non-empty resource version on deleted object") + } + + // Verify it's gone. + got := &testObj{} + err = s.Get(ctx, "/testobjs/default/del-test", storage.GetOptions{}, got) + if !storage.IsNotFound(err) { + t.Errorf("expected not found after delete, got: %v", err) + } +} + +func TestDeleteNotFound(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + deleted := &testObj{} + err := s.Delete(ctx, "/testobjs/default/nonexistent", deleted, nil, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}) + if !storage.IsNotFound(err) { + t.Errorf("expected not found error, got: %v", err) + } +} + +func TestDeletePreconditions(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "del-precond", + Namespace: "default", + UID: "correct-uid", + }, + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/del-precond", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // Wrong UID should fail. + wrongUID := types.UID("wrong-uid") + deleted := &testObj{} + err := s.Delete(ctx, "/testobjs/default/del-precond", deleted, &storage.Preconditions{UID: &wrongUID}, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}) + if err == nil { + t.Fatal("expected precondition error for wrong UID") + } + + // Correct UID should succeed. + correctUID := types.UID("correct-uid") + deleted = &testObj{} + err = s.Delete(ctx, "/testobjs/default/del-precond", deleted, &storage.Preconditions{UID: &correctUID}, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}) + if err != nil { + t.Fatalf("Delete with correct UID: %v", err) + } +} + +func TestGuaranteedUpdate(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "update-test", + Namespace: "default", + }, + Data: "original", + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/update-test", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + updated := &testObj{} + err := s.GuaranteedUpdate(ctx, "/testobjs/default/update-test", updated, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Data = "updated" + return existing, nil, nil + }, nil) + if err != nil { + t.Fatalf("GuaranteedUpdate: %v", err) + } + if updated.Data != "updated" { + t.Errorf("expected data updated, got %s", updated.Data) + } + if updated.ResourceVersion == "" { + t.Error("expected non-empty resource version after update") + } + if updated.ResourceVersion == created.ResourceVersion { + t.Error("expected resource version to change after update") + } +} + +func TestGuaranteedUpdateNotFoundIgnored(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // ignoreNotFound=true should create the object. + dest := &testObj{} + err := s.GuaranteedUpdate(ctx, "/testobjs/default/upsert-test", dest, true, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + obj := input.(*testObj) + obj.TypeMeta = metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"} + obj.Name = "upsert-test" + obj.Namespace = "default" + obj.Data = "created-via-update" + return obj, nil, nil + }, nil) + if err != nil { + t.Fatalf("GuaranteedUpdate (upsert): %v", err) + } + if dest.Data != "created-via-update" { + t.Errorf("expected data created-via-update, got %s", dest.Data) + } + if dest.ResourceVersion == "" { + t.Error("expected non-empty resource version") + } + + // Verify it exists via Get. + got := &testObj{} + if err := s.Get(ctx, "/testobjs/default/upsert-test", storage.GetOptions{}, got); err != nil { + t.Fatalf("Get after upsert: %v", err) + } + if got.Data != "created-via-update" { + t.Errorf("expected data created-via-update, got %s", got.Data) + } +} + +func TestGuaranteedUpdateNotFoundError(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // ignoreNotFound=false should return not found. + dest := &testObj{} + err := s.GuaranteedUpdate(ctx, "/testobjs/default/no-exist", dest, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + return input, nil, nil + }, nil) + if !storage.IsNotFound(err) { + t.Errorf("expected not found error, got: %v", err) + } +} + +func TestGuaranteedUpdateNoOp(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "noop-test", + Namespace: "default", + }, + Data: "unchanged", + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/noop-test", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // Start a watcher to verify no event is emitted. + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: true, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + time.Sleep(50 * time.Millisecond) + + // Update with no actual change. + dest := &testObj{} + err = s.GuaranteedUpdate(ctx, "/testobjs/default/noop-test", dest, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + // Return unchanged object. + return input, nil, nil + }, nil) + if err != nil { + t.Fatalf("GuaranteedUpdate (no-op): %v", err) + } + + // RV should stay the same. + if dest.ResourceVersion != created.ResourceVersion { + t.Errorf("expected resource version %s to stay unchanged, got %s", created.ResourceVersion, dest.ResourceVersion) + } + + // No watch event should arrive. + select { + case event := <-w.ResultChan(): + t.Errorf("expected no watch event for no-op, got %s", event.Type) + case <-time.After(200 * time.Millisecond): + // Good — no event. + } +} + +func TestGuaranteedUpdatePreconditions(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "precond-test", + Namespace: "default", + UID: "the-uid", + }, + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/precond-test", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // Wrong UID precondition should fail. + wrongUID := types.UID("wrong-uid") + dest := &testObj{} + err := s.GuaranteedUpdate(ctx, "/testobjs/default/precond-test", dest, false, + &storage.Preconditions{UID: &wrongUID}, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + return input, nil, nil + }, nil) + if err == nil { + t.Fatal("expected precondition error for wrong UID") + } +} + +func TestGuaranteedUpdateConflictRetry(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "conflict-test", + Namespace: "default", + }, + Data: "v1", + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/conflict-test", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // tryUpdate that fails with conflict on the first call, succeeds on retry. + callCount := 0 + dest := &testObj{} + err := s.GuaranteedUpdate(ctx, "/testobjs/default/conflict-test", dest, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + callCount++ + existing := input.(*testObj) + if callCount == 1 { + return nil, nil, apierrors.NewConflict( + schema.GroupResource{Group: "test.io", Resource: "testobjs"}, + "conflict-test", + fmt.Errorf("simulated conflict"), + ) + } + existing.Data = "v2-after-retry" + return existing, nil, nil + }, nil) + if err != nil { + t.Fatalf("GuaranteedUpdate: %v (callCount=%d)", err, callCount) + } + if callCount < 2 { + t.Errorf("expected tryUpdate to be called at least 2 times, got %d", callCount) + } + if dest.Data != "v2-after-retry" { + t.Errorf("expected data v2-after-retry, got %s", dest.Data) + } +} + +func TestGetList(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create a few objects. + for i := 0; i < 3; i++ { + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("list-%d", i), + Namespace: "default", + }, + Data: fmt.Sprintf("item-%d", i), + } + out := &testObj{} + if err := s.Create(ctx, fmt.Sprintf("/testobjs/default/list-%d", i), obj, out, 0); err != nil { + t.Fatalf("Create %d: %v", i, err) + } + } + + // List all. + listObj := &testObjList{} + err := s.GetList(ctx, "/testobjs/", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: true, + }, listObj) + if err != nil { + t.Fatalf("GetList: %v", err) + } + if len(listObj.Items) != 3 { + t.Errorf("expected 3 items, got %d", len(listObj.Items)) + } + if listObj.ResourceVersion == "" { + t.Error("expected non-empty list resource version") + } +} + +func TestWatch(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Start watching. + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: true, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + + // Give the watcher time to start. + time.Sleep(50 * time.Millisecond) + + // Create an object. + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "watch-test", + Namespace: "default", + }, + Data: "watched", + } + out := &testObj{} + if err := s.Create(ctx, "/testobjs/default/watch-test", obj, out, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // Read the watch event. + select { + case event := <-w.ResultChan(): + if event.Type != watch.Added { + t.Errorf("expected Added event, got %s", event.Type) + } + watchedObj, ok := event.Object.(*testObj) + if !ok { + t.Fatalf("expected *testObj, got %T", event.Object) + } + if watchedObj.Name != "watch-test" { + t.Errorf("expected name watch-test, got %s", watchedObj.Name) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for watch event") + } +} + +func TestStats(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create an object. + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "stats-test", + Namespace: "default", + }, + } + out := &testObj{} + if err := s.Create(ctx, "/testobjs/default/stats-test", obj, out, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + stats, err := s.Stats(ctx) + if err != nil { + t.Fatalf("Stats: %v", err) + } + if stats.ObjectCount < 1 { + t.Errorf("expected at least 1 object, got %d", stats.ObjectCount) + } +} + +func TestGetCurrentResourceVersion(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + rv, err := s.GetCurrentResourceVersion(ctx) + if err != nil { + t.Fatalf("GetCurrentResourceVersion: %v", err) + } + if rv == 0 { + t.Error("expected non-zero resource version") + } +} + +func TestReadinessCheck(t *testing.T) { + s := setupTestStore(t) + if err := s.ReadinessCheck(); err != nil { + t.Fatalf("ReadinessCheck: %v", err) + } +} + +func TestDecodeCallback(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create objects. + for i := 0; i < 2; i++ { + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("cb-%d", i), + Namespace: "default", + }, + } + out := &testObj{} + if err := s.Create(ctx, fmt.Sprintf("/testobjs/default/cb-%d", i), obj, out, 0); err != nil { + t.Fatalf("Create %d: %v", i, err) + } + } + + // List with decode callback. + var callbackKeys []string + ctx = storage.WithDecodeCallback(ctx, func(obj runtime.Object, storageKey string, modRevision int64) { + callbackKeys = append(callbackKeys, storageKey) + }) + + listObj := &testObjList{} + err := s.GetList(ctx, "/testobjs/", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: true, + }, listObj) + if err != nil { + t.Fatalf("GetList: %v", err) + } + + if len(callbackKeys) != 2 { + t.Errorf("expected 2 callback calls, got %d: %v", len(callbackKeys), callbackKeys) + } +} + +func TestRequestWatchProgress(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Start watching with progress notifications. + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: true, + ProgressNotify: true, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + + time.Sleep(50 * time.Millisecond) + + // Request progress. + if err := s.RequestWatchProgress(ctx); err != nil { + t.Fatalf("RequestWatchProgress: %v", err) + } + + // Should receive a bookmark event. + select { + case event := <-w.ResultChan(): + if event.Type != watch.Bookmark { + t.Errorf("expected Bookmark event, got %s", event.Type) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for progress event") + } +} + +func TestResourceVersionMonotonicity(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + var rvs []uint64 + + // Create multiple objects and track RVs. + for i := 0; i < 5; i++ { + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("mono-%d", i), + Namespace: "default", + }, + Data: fmt.Sprintf("data-%d", i), + } + out := &testObj{} + if err := s.Create(ctx, fmt.Sprintf("/testobjs/default/mono-%d", i), obj, out, 0); err != nil { + t.Fatalf("Create %d: %v", i, err) + } + parsed, err := s.versioner.ParseResourceVersion(out.ResourceVersion) + if err != nil { + t.Fatalf("parse RV: %v", err) + } + rvs = append(rvs, parsed) + } + + // Now update to generate more RVs. + for i := 0; i < 3; i++ { + updated := &testObj{} + err := s.GuaranteedUpdate(ctx, "/testobjs/default/mono-0", updated, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Data = fmt.Sprintf("updated-%d", i) + return existing, nil, nil + }, nil) + if err != nil { + t.Fatalf("GuaranteedUpdate %d: %v", i, err) + } + parsed, err := s.versioner.ParseResourceVersion(updated.ResourceVersion) + if err != nil { + t.Fatalf("parse RV: %v", err) + } + rvs = append(rvs, parsed) + } + + // Verify strict monotonicity. + for i := 1; i < len(rvs); i++ { + if rvs[i] <= rvs[i-1] { + t.Errorf("resource versions not strictly monotonic: rv[%d]=%d <= rv[%d]=%d", i, rvs[i], i-1, rvs[i-1]) + } + } +} + +func TestGetListPagination(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create 5 objects. + for i := 0; i < 5; i++ { + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("page-%d", i), + Namespace: "default", + }, + } + out := &testObj{} + if err := s.Create(ctx, fmt.Sprintf("/testobjs/default/page-%d", i), obj, out, 0); err != nil { + t.Fatalf("Create %d: %v", i, err) + } + } + + // Page 1: limit 2. + listObj := &testObjList{} + pred := storage.Everything + pred.Limit = 2 + err := s.GetList(ctx, "/testobjs/", storage.ListOptions{ + Predicate: pred, + Recursive: true, + }, listObj) + if err != nil { + t.Fatalf("GetList page 1: %v", err) + } + if len(listObj.Items) != 2 { + t.Fatalf("expected 2 items on page 1, got %d", len(listObj.Items)) + } + if listObj.Continue == "" { + t.Fatal("expected non-empty continue token") + } + + // Page 2: use continue token (RV is encoded in the token, not passed separately). + listObj2 := &testObjList{} + pred2 := storage.Everything + pred2.Limit = 2 + pred2.Continue = listObj.Continue + err = s.GetList(ctx, "/testobjs/", storage.ListOptions{ + Predicate: pred2, + Recursive: true, + }, listObj2) + if err != nil { + t.Fatalf("GetList page 2: %v", err) + } + if len(listObj2.Items) != 2 { + t.Fatalf("expected 2 items on page 2, got %d", len(listObj2.Items)) + } + + // Collect all names to verify no duplicates. + seen := map[string]bool{} + for _, item := range listObj.Items { + if seen[item.Name] { + t.Errorf("duplicate name in pages: %s", item.Name) + } + seen[item.Name] = true + } + for _, item := range listObj2.Items { + if seen[item.Name] { + t.Errorf("duplicate name across pages: %s", item.Name) + } + seen[item.Name] = true + } +} + +func TestWatchSendInitialEvents(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create objects before starting the watch. + for i := 0; i < 3; i++ { + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("init-%d", i), + Namespace: "default", + }, + } + out := &testObj{} + if err := s.Create(ctx, fmt.Sprintf("/testobjs/default/init-%d", i), obj, out, 0); err != nil { + t.Fatalf("Create %d: %v", i, err) + } + } + + // Watch with SendInitialEvents. + sendInitial := true + pred := storage.Everything + pred.AllowWatchBookmarks = true + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Predicate: pred, + Recursive: true, + SendInitialEvents: &sendInitial, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + + // Should receive 3 Added events + 1 Bookmark. + addedCount := 0 + bookmarkCount := 0 + timeout := time.After(5 * time.Second) + for addedCount < 3 || bookmarkCount < 1 { + select { + case event := <-w.ResultChan(): + switch event.Type { + case watch.Added: + addedCount++ + case watch.Bookmark: + bookmarkCount++ + default: + t.Errorf("unexpected event type: %s", event.Type) + } + case <-timeout: + t.Fatalf("timed out: got %d added, %d bookmarks", addedCount, bookmarkCount) + } + } + if addedCount != 3 { + t.Errorf("expected 3 Added events, got %d", addedCount) + } +} + +func TestWatchNonRecursive(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Watch a specific key (non-recursive). + w, err := s.Watch(ctx, "/testobjs/default/specific", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: false, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + time.Sleep(50 * time.Millisecond) + + // Create an object at a different key — should NOT trigger. + other := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: "default"}, + } + out := &testObj{} + if err := s.Create(ctx, "/testobjs/default/other", other, out, 0); err != nil { + t.Fatalf("Create other: %v", err) + } + + // Create the watched object — should trigger. + specific := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{Name: "specific", Namespace: "default"}, + } + if err := s.Create(ctx, "/testobjs/default/specific", specific, out, 0); err != nil { + t.Fatalf("Create specific: %v", err) + } + + // Should receive only the specific key event. + select { + case event := <-w.ResultChan(): + if event.Type != watch.Added { + t.Errorf("expected Added, got %s", event.Type) + } + obj := event.Object.(*testObj) + if obj.Name != "specific" { + t.Errorf("expected name specific, got %s", obj.Name) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for event") + } + + // No other event should arrive. + select { + case event := <-w.ResultChan(): + t.Errorf("unexpected extra event: %s %v", event.Type, event.Object) + case <-time.After(200 * time.Millisecond): + // Good. + } +} + +func TestDeleteReturnsRV(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "del-rv", + Namespace: "default", + }, + } + created := &testObj{} + if err := s.Create(ctx, "/testobjs/default/del-rv", obj, created, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + deleted := &testObj{} + if err := s.Delete(ctx, "/testobjs/default/del-rv", deleted, nil, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}); err != nil { + t.Fatalf("Delete: %v", err) + } + + // Deleted object must have an RV, and it must be >= the create RV (delete happens after create). + if deleted.ResourceVersion == "" { + t.Fatal("expected non-empty resource version on deleted object") + } + createRV, _ := s.versioner.ParseResourceVersion(created.ResourceVersion) + deleteRV, _ := s.versioner.ParseResourceVersion(deleted.ResourceVersion) + if deleteRV < createRV { + t.Errorf("delete RV (%d) should be >= create RV (%d)", deleteRV, createRV) + } +} + +func TestWatchDeleteHasObject(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "watch-del-obj", + Namespace: "default", + }, + Data: "before-delete", + } + out := &testObj{} + if err := s.Create(ctx, "/testobjs/default/watch-del-obj", obj, out, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: true, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + time.Sleep(50 * time.Millisecond) + + deleted := &testObj{} + if err := s.Delete(ctx, "/testobjs/default/watch-del-obj", deleted, nil, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}); err != nil { + t.Fatalf("Delete: %v", err) + } + + // The delete watch event should contain the previous object state. + select { + case event := <-w.ResultChan(): + if event.Type != watch.Deleted { + t.Fatalf("expected Deleted, got %s", event.Type) + } + delObj := event.Object.(*testObj) + if delObj.Name != "watch-del-obj" { + t.Errorf("expected name watch-del-obj, got %s", delObj.Name) + } + if delObj.Data != "before-delete" { + t.Errorf("expected data before-delete, got %s", delObj.Data) + } + if delObj.ResourceVersion == "" { + t.Error("expected non-empty RV on watched delete event object") + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for delete event") + } +} + +func TestWatchUpsertIsAdded(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Watch before the upsert. + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: true, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + time.Sleep(50 * time.Millisecond) + + // GuaranteedUpdate with ignoreNotFound=true creates a new object. + dest := &testObj{} + err = s.GuaranteedUpdate(ctx, "/testobjs/default/upsert-watch", dest, true, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + obj := input.(*testObj) + obj.TypeMeta = metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"} + obj.Name = "upsert-watch" + obj.Namespace = "default" + obj.Data = "created" + return obj, nil, nil + }, nil) + if err != nil { + t.Fatalf("GuaranteedUpdate (upsert): %v", err) + } + + // The watch event should be Added (not Modified). + select { + case event := <-w.ResultChan(): + if event.Type != watch.Added { + t.Errorf("expected Added event for upsert, got %s", event.Type) + } + obj := event.Object.(*testObj) + if obj.Data != "created" { + t.Errorf("expected data created, got %s", obj.Data) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for upsert watch event") + } +} + +func TestSendInitialEventsNoDuplicates(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create objects before watching. + for i := 0; i < 3; i++ { + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("dedup-%d", i), + Namespace: "default", + }, + } + out := &testObj{} + if err := s.Create(ctx, fmt.Sprintf("/testobjs/default/dedup-%d", i), obj, out, 0); err != nil { + t.Fatalf("Create %d: %v", i, err) + } + } + + // Watch with SendInitialEvents. + sendInitial := true + pred := storage.Everything + pred.AllowWatchBookmarks = true + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Predicate: pred, + Recursive: true, + SendInitialEvents: &sendInitial, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + + // Collect initial events: 3 Added + 1 Bookmark. + addedNames := map[string]int{} + timeout := time.After(5 * time.Second) + for i := 0; i < 4; i++ { + select { + case event := <-w.ResultChan(): + switch event.Type { + case watch.Added: + obj := event.Object.(*testObj) + addedNames[obj.Name]++ + case watch.Bookmark: + // expected + } + case <-timeout: + t.Fatalf("timed out after %d events", i) + } + } + + // Verify no duplicates. + for name, count := range addedNames { + if count > 1 { + t.Errorf("duplicate Added event for %s: %d times", name, count) + } + } + + // No extra events should arrive (no duplicates from broadcaster). + select { + case event := <-w.ResultChan(): + t.Errorf("unexpected extra event after initial: %s %T", event.Type, event.Object) + case <-time.After(300 * time.Millisecond): + // Good — no duplicates. + } +} + +func TestWatchDelete(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create object first. + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "watch-del", + Namespace: "default", + }, + Data: "will-delete", + } + out := &testObj{} + if err := s.Create(ctx, "/testobjs/default/watch-del", obj, out, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // Start watching. + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: true, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + + time.Sleep(50 * time.Millisecond) + + // Delete the object. + deleted := &testObj{} + if err := s.Delete(ctx, "/testobjs/default/watch-del", deleted, nil, storage.ValidateAllObjectFunc, nil, storage.DeleteOptions{}); err != nil { + t.Fatalf("Delete: %v", err) + } + + // Should receive a Deleted event. + select { + case event := <-w.ResultChan(): + if event.Type != watch.Deleted { + t.Errorf("expected Deleted event, got %s", event.Type) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for delete event") + } +} + +func TestWatchModify(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create object. + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "watch-mod", + Namespace: "default", + }, + Data: "original", + } + out := &testObj{} + if err := s.Create(ctx, "/testobjs/default/watch-mod", obj, out, 0); err != nil { + t.Fatalf("Create: %v", err) + } + + // Start watching. + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Predicate: storage.Everything, + Recursive: true, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + + time.Sleep(50 * time.Millisecond) + + // Update the object. + updated := &testObj{} + if err := s.GuaranteedUpdate(ctx, "/testobjs/default/watch-mod", updated, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Data = "modified" + return existing, nil, nil + }, nil); err != nil { + t.Fatalf("GuaranteedUpdate: %v", err) + } + + // Should receive a Modified event. + select { + case event := <-w.ResultChan(): + if event.Type != watch.Modified { + t.Errorf("expected Modified event, got %s", event.Type) + } + modObj, ok := event.Object.(*testObj) + if !ok { + t.Fatalf("expected *testObj, got %T", event.Object) + } + if modObj.Data != "modified" { + t.Errorf("expected data modified, got %s", modObj.Data) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for modified event") + } +} + +func TestPrepareKeyValidation(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + tests := []struct { + name string + key string + }{ + {"path traversal with ..", "../etc/passwd"}, + {"path traversal mid-key", "/testobjs/../../secrets"}, + {"path traversal suffix", "/testobjs/default/.."}, + {"dot traversal", "./something"}, + {"dot mid-key", "/testobjs/./default"}, + {"dot suffix", "/testobjs/default/."}, + {"empty key", ""}, + {"slash only", "/"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{Name: "bad", Namespace: "default"}, + } + out := &testObj{} + err := s.Create(ctx, tc.key, obj, out, 0) + if err == nil { + t.Errorf("expected error for key %q, got nil", tc.key) + } + + err = s.Get(ctx, tc.key, storage.GetOptions{}, out) + if err == nil { + t.Errorf("expected error for Get with key %q, got nil", tc.key) + } + + err = s.Delete(ctx, tc.key, out, nil, func(_ context.Context, _ runtime.Object) error { return nil }, nil, storage.DeleteOptions{}) + if err == nil { + t.Errorf("expected error for Delete with key %q, got nil", tc.key) + } + + err = s.GuaranteedUpdate(ctx, tc.key, out, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + return input, nil, nil + }, nil) + if err == nil { + t.Errorf("expected error for GuaranteedUpdate with key %q, got nil", tc.key) + } + }) + } +} + +func TestGetListContextCancellation(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create several objects so the list has items to iterate. + for i := 0; i < 10; i++ { + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("item-%d", i), Namespace: "default"}, + Data: fmt.Sprintf("data-%d", i), + } + out := &testObj{} + if err := s.Create(ctx, fmt.Sprintf("/testobjs/default/item-%d", i), obj, out, 0); err != nil { + t.Fatalf("Create: %v", err) + } + } + + // Cancel the context before listing. + cancelCtx, cancel := context.WithCancel(ctx) + cancel() // immediately cancelled + + listObj := &testObjList{} + err := s.GetList(cancelCtx, "/testobjs/", storage.ListOptions{ + Recursive: true, + Predicate: storage.SelectionPredicate{}, + }, listObj) + if err == nil { + t.Fatal("expected error from cancelled context, got nil") + } +} + +// testObjGetAttrs returns labels, fields, and an error for a testObj. +// Required for SelectionPredicate.Matches() to work. +func testObjGetAttrs(obj runtime.Object) (labels.Set, fields.Set, error) { + t, ok := obj.(*testObj) + if !ok { + return nil, nil, fmt.Errorf("not a testObj: %T", obj) + } + return t.Labels, fields.Set{"metadata.name": t.Name}, nil +} + +func TestWatchPredicateFilterTransitions(t *testing.T) { + s := setupTestStore(t) + ctx := context.Background() + + // Create an object that matches the predicate (has label "color=blue"). + obj := &testObj{ + TypeMeta: metav1.TypeMeta{APIVersion: "test.io/v1", Kind: "TestObj"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "filter-test", + Namespace: "default", + Labels: map[string]string{"color": "blue"}, + }, + Data: "initial", + } + out := &testObj{} + if err := s.Create(ctx, "/testobjs/default/filter-test", obj, out, 0); err != nil { + t.Fatalf("Create: %v", err) + } + createdRV := out.ResourceVersion + + // Start a watch with a label predicate selecting color=blue. + pred := storage.SelectionPredicate{ + Label: labels.SelectorFromSet(labels.Set{"color": "blue"}), + Field: fields.Everything(), + GetAttrs: testObjGetAttrs, + } + w, err := s.Watch(ctx, "/testobjs/", storage.ListOptions{ + Recursive: true, + Predicate: pred, + ResourceVersion: createdRV, + }) + if err != nil { + t.Fatalf("Watch: %v", err) + } + defer w.Stop() + time.Sleep(50 * time.Millisecond) + + // --- Test 1: Object leaves predicate window (blue → red) --- + // Should produce a synthetic Deleted event with the OLD object. + updated := &testObj{} + if err := s.GuaranteedUpdate(ctx, "/testobjs/default/filter-test", updated, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Labels = map[string]string{"color": "red"} + existing.Data = "now-red" + return existing, nil, nil + }, nil); err != nil { + t.Fatalf("GuaranteedUpdate (blue→red): %v", err) + } + + select { + case event := <-w.ResultChan(): + if event.Type != watch.Deleted { + t.Errorf("expected synthetic Deleted when leaving predicate window, got %s", event.Type) + } + delObj := event.Object.(*testObj) + // The deleted event should carry the OLD object (color=blue). + if delObj.Labels["color"] != "blue" { + t.Errorf("expected old object with color=blue in synthetic Deleted, got color=%s", delObj.Labels["color"]) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for synthetic Deleted event") + } + + // --- Test 2: Object re-enters predicate window (red → blue) --- + // Should produce a synthetic Added event with the NEW object. + if err := s.GuaranteedUpdate(ctx, "/testobjs/default/filter-test", updated, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Labels = map[string]string{"color": "blue"} + existing.Data = "blue-again" + return existing, nil, nil + }, nil); err != nil { + t.Fatalf("GuaranteedUpdate (red→blue): %v", err) + } + + select { + case event := <-w.ResultChan(): + if event.Type != watch.Added { + t.Errorf("expected synthetic Added when entering predicate window, got %s", event.Type) + } + addObj := event.Object.(*testObj) + if addObj.Data != "blue-again" { + t.Errorf("expected data blue-again, got %s", addObj.Data) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for synthetic Added event") + } + + // --- Test 3: Object stays in predicate window (blue → blue, different data) --- + // Should produce a normal Modified event. + if err := s.GuaranteedUpdate(ctx, "/testobjs/default/filter-test", updated, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Data = "still-blue" + return existing, nil, nil + }, nil); err != nil { + t.Fatalf("GuaranteedUpdate (blue→blue): %v", err) + } + + select { + case event := <-w.ResultChan(): + if event.Type != watch.Modified { + t.Errorf("expected Modified when staying in predicate window, got %s", event.Type) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for Modified event") + } + + // --- Test 4: Object modified outside predicate window (red → green) --- + // First move it out of the window. + if err := s.GuaranteedUpdate(ctx, "/testobjs/default/filter-test", updated, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Labels = map[string]string{"color": "red"} + return existing, nil, nil + }, nil); err != nil { + t.Fatalf("GuaranteedUpdate (blue→red): %v", err) + } + // Drain the Deleted event. + select { + case <-w.ResultChan(): + case <-time.After(5 * time.Second): + t.Fatal("timed out draining Deleted event") + } + + // Now modify while still outside — should produce no event. + if err := s.GuaranteedUpdate(ctx, "/testobjs/default/filter-test", updated, false, nil, + func(input runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) { + existing := input.(*testObj) + existing.Labels = map[string]string{"color": "green"} + existing.Data = "green-data" + return existing, nil, nil + }, nil); err != nil { + t.Fatalf("GuaranteedUpdate (red→green): %v", err) + } + + select { + case event := <-w.ResultChan(): + t.Errorf("expected no event for modification outside predicate window, got %s", event.Type) + case <-time.After(200 * time.Millisecond): + // Good — no event received. + } +} diff --git a/backends/spanner/watcher.go b/backends/spanner/watcher.go new file mode 100644 index 0000000..0020099 --- /dev/null +++ b/backends/spanner/watcher.go @@ -0,0 +1,336 @@ +package spanner + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/storage" + "k8s.io/klog/v2" +) + +// spannerWatcher implements watch.Interface backed by a broadcast subscription. +type spannerWatcher struct { + store *store + prefix string + opts storage.ListOptions + startRev int64 + + result chan watch.Event + done chan struct{} + once sync.Once + + sub *subscription +} + +var _ watch.Interface = (*spannerWatcher)(nil) + +func newWatcher(s *store, prefix string, opts storage.ListOptions, startRev int64) *spannerWatcher { + w := &spannerWatcher{ + store: s, + prefix: prefix, + opts: opts, + startRev: startRev, + result: make(chan watch.Event, 100), + done: make(chan struct{}), + } + + w.sub = s.broadcaster.Subscribe(256) + go w.run() + return w +} + +func (w *spannerWatcher) ResultChan() <-chan watch.Event { + return w.result +} + +func (w *spannerWatcher) Stop() { + w.once.Do(func() { + w.store.broadcaster.Unsubscribe(w.sub) + close(w.done) + }) +} + +func (w *spannerWatcher) run() { + defer close(w.result) + + klog.V(4).Infof("spannerWatcher.run: prefix=%q startRev=%d sendInitialEvents=%v", w.prefix, w.startRev, w.opts.SendInitialEvents) + + // If SendInitialEvents is requested, send a snapshot first, + // then a bookmark, then switch to streaming. + if w.opts.SendInitialEvents != nil && *w.opts.SendInitialEvents { + if err := w.sendInitialEvents(); err != nil { + klog.Errorf("failed to send initial events: %v", err) + return + } + } + + klog.V(4).Infof("spannerWatcher.run: streaming from broadcaster, prefix=%q startRev=%d", w.prefix, w.startRev) + + // Stream events from the broadcaster. + for { + select { + case <-w.done: + return + case e, ok := <-w.sub.ch: + if !ok { + klog.V(4).Infof("spannerWatcher.run: subscription closed, prefix=%q", w.prefix) + return + } + if err := w.processEvent(e); err != nil { + klog.V(4).Infof("watcher: failed to process event: %v", err) + } + } + } +} + +// sendInitialEvents lists all existing objects and sends them as ADDED events, +// followed by a bookmark at the current resource version. It advances startRev +// so that the subsequent broadcast stream doesn't re-emit events already sent. +func (w *spannerWatcher) sendInitialEvents() error { + ctx := context.Background() + + // Capture storage keys via decode callback so we can wrap each item + // with wrapDecodedObject (same identity annotation as processEvent). + var keyMap map[string]string + if w.store.wrapDecodedObject != nil { + keyMap = make(map[string]string) + ctx = storage.WithDecodeCallback(ctx, func(obj runtime.Object, storageKey string, modRev int64) { + accessor, err := meta.Accessor(obj) + if err != nil { + return + } + id := strconv.FormatInt(modRev, 10) + "|" + accessor.GetNamespace() + "|" + accessor.GetName() + keyMap[id] = storageKey + }) + } + + listObj := w.store.newListFunc() + + listOpts := storage.ListOptions{ + Predicate: w.opts.Predicate, + Recursive: w.opts.Recursive, + } + if w.startRev > 0 { + listOpts.ResourceVersion = strconv.FormatUint(uint64(w.startRev), 10) + } + + if err := w.store.GetList(ctx, w.prefix, listOpts, listObj); err != nil { + klog.Errorf("spannerWatcher.sendInitialEvents: GetList failed: %v (prefix=%q)", err, w.prefix) + return err + } + + items, err := extractListItems(listObj) + if err != nil { + return err + } + + klog.V(4).Infof("spannerWatcher.sendInitialEvents: prefix=%q items=%d", w.prefix, len(items)) + + for _, item := range items { + obj := item + if w.store.wrapDecodedObject != nil && keyMap != nil { + if accessor, aErr := meta.Accessor(item); aErr == nil { + id := accessor.GetResourceVersion() + "|" + accessor.GetNamespace() + "|" + accessor.GetName() + if storageKey, ok := keyMap[id]; ok { + obj = w.store.wrapDecodedObject(item, storageKey) + } + } + } + select { + case <-w.done: + return nil + case w.result <- watch.Event{Type: watch.Added, Object: obj}: + } + } + + // Get the current RV to use as the bookmark and the dedup watermark. + bookmarkRV, rvErr := w.store.GetCurrentResourceVersion(ctx) + if rvErr != nil { + klog.Errorf("spannerWatcher.sendInitialEvents: GetCurrentResourceVersion failed: %v (prefix=%q)", rvErr, w.prefix) + } + klog.V(4).Infof("spannerWatcher.sendInitialEvents: bookmarkRV=%d allowBookmarks=%v prefix=%q", bookmarkRV, w.opts.Predicate.AllowWatchBookmarks, w.prefix) + + // Send a bookmark to signal the end of initial events. + // The annotation is required by the WatchList contract so that the + // reflector knows the initial snapshot is complete. + if w.opts.Predicate.AllowWatchBookmarks && bookmarkRV > 0 { + bookmarkObj := w.store.newFunc() + _ = w.store.versioner.UpdateObject(bookmarkObj, bookmarkRV) + if err := storage.AnnotateInitialEventsEndBookmark(bookmarkObj); err != nil { + return fmt.Errorf("failed to annotate initial events end bookmark: %w", err) + } + select { + case <-w.done: + return nil + case w.result <- watch.Event{Type: watch.Bookmark, Object: bookmarkObj}: + } + } + + // Advance startRev so the broadcast stream doesn't re-emit events + // that were already covered by the initial list. + if int64(bookmarkRV) > w.startRev { + w.startRev = int64(bookmarkRV) + } + + return nil +} + +func (w *spannerWatcher) processEvent(e watchEvent) error { + // Handle progress/bookmark events before key filtering — + // progress events have no key and apply to all watchers. + if e.isProgress { + if w.opts.ProgressNotify { + bookmarkObj := w.store.newFunc() + if e.rev > 0 { + _ = w.store.versioner.UpdateObject(bookmarkObj, uint64(e.rev)) + } + select { + case <-w.done: + case w.result <- watch.Event{Type: watch.Bookmark, Object: bookmarkObj}: + } + } + return nil + } + + // Filter by key prefix for recursive watches. + if w.opts.Recursive { + if !strings.HasPrefix(e.key, w.prefix) { + klog.V(5).Infof("spannerWatcher.processEvent: key prefix mismatch key=%q prefix=%q", e.key, w.prefix) + return nil + } + } else { + // Non-recursive: exact key match (strip trailing /). + trimmed := strings.TrimSuffix(w.prefix, "/") + if e.key != trimmed { + return nil + } + } + + // Filter by revision. + if w.startRev > 0 && e.rev <= w.startRev { + klog.V(5).Infof("spannerWatcher.processEvent: rev filtered key=%q rev=%d startRev=%d", e.key, e.rev, w.startRev) + return nil + } + + klog.V(4).Infof("spannerWatcher.processEvent: DELIVERING key=%q rev=%d isCreated=%v isDeleted=%v prefix=%q", e.key, e.rev, e.isCreated, e.isDeleted, w.prefix) + + // Decode current and previous objects as needed. + ctx := context.Background() + var curObj, oldObj runtime.Object + + if !e.isDeleted && e.value != nil { + data, _, err := w.store.transformer.TransformFromStorage(ctx, e.value, authenticatedDataString(e.key)) + if err != nil { + return err + } + curObj = w.store.newFunc() + if err := decode(w.store.codec, w.store.versioner, data, curObj, uint64(e.rev)); err != nil { + return err + } + } + + // Decode prevValue for deletes, and for modifications when a predicate + // filter is present (needed to detect predicate window transitions). + if len(e.prevValue) > 0 && (e.isDeleted || !w.opts.Predicate.Empty()) { + data, _, err := w.store.transformer.TransformFromStorage(ctx, e.prevValue, authenticatedDataString(e.key)) + if err != nil { + return err + } + oldObj = w.store.newFunc() + if err := decode(w.store.codec, w.store.versioner, data, oldObj, uint64(e.rev)); err != nil { + return err + } + } + + // Determine event type, applying predicate filter transitions. + // Matches etcd3 behavior: when a predicate is present, modifications + // that cause an object to enter/leave the predicate window emit + // synthetic Added/Deleted events. + var eventType watch.EventType + var obj runtime.Object + + switch { + case e.isCreated: + eventType = watch.Added + obj = curObj + case e.isDeleted: + eventType = watch.Deleted + obj = oldObj + default: + // Modification — check predicate transitions. + if w.opts.Predicate.Empty() { + eventType = watch.Modified + obj = curObj + } else { + curPasses := curObj != nil && w.matchesPredicate(curObj) + oldPasses := oldObj != nil && w.matchesPredicate(oldObj) + switch { + case curPasses && oldPasses: + eventType = watch.Modified + obj = curObj + case curPasses && !oldPasses: + // Object entered predicate window. + eventType = watch.Added + obj = curObj + case !curPasses && oldPasses: + // Object left predicate window. + eventType = watch.Deleted + obj = oldObj + default: + // Neither matches — skip entirely. + return nil + } + } + } + + if obj == nil { + return nil + } + + // Apply wrapDecodedObject hook if configured. + // Strip the storage pathPrefix so the key matches the cacher's + // namespace (same as etcd3's storageKeyFromPreparedKey). + if w.store.wrapDecodedObject != nil { + storageKey := w.store.storageKeyFromSpannerKey(e.key) + obj = w.store.wrapDecodedObject(obj, storageKey) + } + + // Apply predicate filter for create/delete events. + if !w.opts.Predicate.Empty() && (e.isCreated || e.isDeleted) { + if !w.matchesPredicate(obj) { + return nil + } + } + + select { + case <-w.done: + case w.result <- watch.Event{Type: eventType, Object: obj}: + } + + return nil +} + +// matchesPredicate returns true if the object matches the watcher's predicate. +func (w *spannerWatcher) matchesPredicate(obj runtime.Object) bool { + matched, err := w.opts.Predicate.Matches(obj) + return err == nil && matched +} + +// extractListItems extracts individual items from a list object. +func extractListItems(listObj runtime.Object) ([]runtime.Object, error) { + items, err := meta.ExtractList(listObj) + if err != nil { + return nil, err + } + result := make([]runtime.Object, 0, len(items)) + for _, item := range items { + result = append(result, item) + } + return result, nil +} diff --git a/decorator.go b/decorator.go index 7553db6..7017617 100644 --- a/decorator.go +++ b/decorator.go @@ -13,21 +13,18 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + + "github.com/kplane-dev/storage/registry" ) // BackendFactory constructs the underlying raw storage for one resource. -// Signature matches upstream k8s.io/apiserver/pkg/storage/storagebackend/ -// factory.Create — so any backend that already satisfies that shape (etcd3, -// Spanner, future postgres) plugs in without adaptation. -// -// Mirrors registry.Factory; declared here as a top-level type so consumers -// of DecoratorConfig don't need a transitive import of the registry -// subpackage. -type BackendFactory func( - config *storagebackend.ConfigForResource, - newFunc, newListFunc func() runtime.Object, - resourcePrefix string, -) (storage.Interface, factory.DestroyFunc, error) +// Aliased to registry.Factory so the apiserver can pass the same value +// returned by registry.Backend.Build() directly into DecoratorConfig +// without a type conversion. Signature matches upstream +// k8s.io/apiserver/pkg/storage/storagebackend/factory.Create — so any +// backend that already satisfies that shape (etcd3, Spanner, future +// postgres) plugs in without adaptation. +type BackendFactory = registry.Factory // DecoratorConfig configures the cluster-aware StorageDecorator. type DecoratorConfig struct { diff --git a/go.mod b/go.mod index 23bb5b9..0c6ba2f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kplane-dev/storage -go 1.25.0 +go 1.25.8 godebug default=go1.25 @@ -14,26 +14,44 @@ replace ( ) require ( + cloud.google.com/go/spanner v1.92.0 + github.com/kplane-dev/spanner v0.0.0-20260311053803-bee01b5e2306 github.com/spf13/pflag v1.0.9 + google.golang.org/api v0.285.0 + google.golang.org/grpc v1.81.1 k8s.io/apimachinery v0.0.0 - k8s.io/apiserver v0.0.0-00010101000000-000000000000 + k8s.io/apiserver v0.0.0 k8s.io/client-go v0.0.0 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 ) require ( + cel.dev/expr v0.25.1 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.11.0 // indirect + cloud.google.com/go/longrunning v1.0.0 // indirect + cloud.google.com/go/monitoring v1.29.0 // indirect + github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -45,7 +63,10 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect @@ -58,6 +79,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -65,6 +87,7 @@ require ( github.com/prometheus/procfs v0.19.2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/testify v1.11.1 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect github.com/x448/float16 v0.8.4 // indirect @@ -76,30 +99,34 @@ require ( go.etcd.io/etcd/pkg/v3 v3.6.7 // indirect go.etcd.io/etcd/server/v3 v3.6.7 // indirect go.etcd.io/raft/v3 v3.6.0 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.47.0 // indirect - golang.org/x/net v0.49.0 // indirect - golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sys v0.40.0 // indirect - golang.org/x/term v0.39.0 // indirect - golang.org/x/text v0.33.0 // indirect - golang.org/x/time v0.14.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/grpc v1.78.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 0470893..30d9f1c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,27 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/longrunning v1.0.0 h1:lwzWEYD8+NkYV7dhexOz6kmlvajZA70+bW/xMhRVVdY= +cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= +cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= +cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= +cloud.google.com/go/spanner v1.92.0 h1:cfeMNmtFjz+OYzQVCIuGBw4Cik4CbF2ptXMuRQcUar0= +cloud.google.com/go/spanner v1.92.0/go.mod h1:rCDPfWXNX0h+t484r+crCEaaMKbJfoWkHRDKU3H3+oY= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 h1:BzsL0qE7LvtTEtXG7Dt5NS1EP0CQwI21HZfj9aGghhw= +github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0/go.mod h1:I7kE2kM3qCr9QPT4cU4cCFYkEpVyVr16YOGUHzy+nR0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -6,8 +30,13 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= @@ -23,10 +52,24 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -46,19 +89,48 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= +github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.16 h1:F/VPrx0YPBdksZJQdCAp0WUsqnNmZpUZszzfYt0M5Dw= +github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= @@ -88,6 +160,8 @@ github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260311054 github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260311054814-32f5e9075db5/go.mod h1:7IM9p4c8CafSxF7ZY0F46WHylFn3o4mLVW5T1VZbaY8= github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260311054814-32f5e9075db5 h1:l4RLyYStWu+BL8uKL++4byqPwLc2cv76rnHW/7A+Cvw= github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260311054814-32f5e9075db5/go.mod h1:R6vYa1XRfX3PdQEGNkCaL3pt7NvLU2ti7FPzsEsA6GQ= +github.com/kplane-dev/spanner v0.0.0-20260311053803-bee01b5e2306 h1:AVp7RjXcgXrhFhw4RPgw8sSPUX/LY38aBUOmP8EI924= +github.com/kplane-dev/spanner v0.0.0-20260311053803-bee01b5e2306/go.mod h1:aswcrXDx33W3tgV36qWgut8ziOIqAopK5YPDjqVuIU4= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -111,11 +185,14 @@ github.com/onsi/ginkgo/v2 v2.27.4 h1:fcEcQW/A++6aZAZQNUmNjvA9PSOzefMJBerHJ4t8v8Y github.com/onsi/ginkgo/v2 v2.27.4/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= @@ -131,6 +208,8 @@ github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -166,26 +245,32 @@ go.etcd.io/etcd/server/v3 v3.6.7 h1:8dEGQ877tj0cQJFEfD2bDoZDA76qbS2OkvCNjwAyrSo= go.etcd.io/etcd/server/v3 v3.6.7/go.mod h1:LEM328bPA2uVMhN0+Ht/vAsADW127QS1oM7EuHrOTy0= go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -201,27 +286,40 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= -golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= -golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= -golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -230,36 +328,63 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.285.0 h1:B7eHHoKGAX/LrPkQvhQqnGwjgWxofbdGwCTQvpm8FkM= +google.golang.org/api v0.285.0/go.mod h1:NlOlUIr8MPoIhT9Bb/oUnRuHbJOLwxb6JSYJM8Yz+jQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad h1:45WmJvIV6C2+O/jjLkPUH+F3aOj/1miDoU2DD0+NWbg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -274,6 +399,8 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 h1:HhDfevmPS+OalTjQRKbTHppRIz01AWi8s45TMXStgYY= From dccace0ff6c3c5d746e6871a10919e6a63bd4eb9 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 16 Jun 2026 17:40:11 -0700 Subject: [PATCH 5/6] chore: bump go directive to 1.25.8 (cloud.google.com/go/spanner v1.92.0 requirement) Auto-bumped by go mod tidy after pulling in Spanner deps from the backends/spanner migration. Local dev needs Go 1.25.8 (or auto-toolchain download). CI bumps will follow. --- go.mod | 3 +-- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0c6ba2f..9b3c69d 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,7 @@ replace ( ) require ( - cloud.google.com/go/spanner v1.92.0 - github.com/kplane-dev/spanner v0.0.0-20260311053803-bee01b5e2306 + cloud.google.com/go/spanner v1.88.0 github.com/spf13/pflag v1.0.9 google.golang.org/api v0.285.0 google.golang.org/grpc v1.81.1 diff --git a/go.sum b/go.sum index 30d9f1c..eebd487 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,8 @@ cloud.google.com/go/longrunning v1.0.0 h1:lwzWEYD8+NkYV7dhexOz6kmlvajZA70+bW/xMh cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= -cloud.google.com/go/spanner v1.92.0 h1:cfeMNmtFjz+OYzQVCIuGBw4Cik4CbF2ptXMuRQcUar0= -cloud.google.com/go/spanner v1.92.0/go.mod h1:rCDPfWXNX0h+t484r+crCEaaMKbJfoWkHRDKU3H3+oY= +cloud.google.com/go/spanner v1.88.0 h1:HS+5TuEYZOVOXj9K+0EtrbTw7bKBLrMe3vgGsbnehmU= +cloud.google.com/go/spanner v1.88.0/go.mod h1:MzulBwuuYwQUVdkZXBBFapmXee3N+sQrj2T/yup6uEE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 h1:BzsL0qE7LvtTEtXG7Dt5NS1EP0CQwI21HZfj9aGghhw= github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0/go.mod h1:I7kE2kM3qCr9QPT4cU4cCFYkEpVyVr16YOGUHzy+nR0= @@ -160,8 +160,6 @@ github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260311054 github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260311054814-32f5e9075db5/go.mod h1:7IM9p4c8CafSxF7ZY0F46WHylFn3o4mLVW5T1VZbaY8= github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260311054814-32f5e9075db5 h1:l4RLyYStWu+BL8uKL++4byqPwLc2cv76rnHW/7A+Cvw= github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260311054814-32f5e9075db5/go.mod h1:R6vYa1XRfX3PdQEGNkCaL3pt7NvLU2ti7FPzsEsA6GQ= -github.com/kplane-dev/spanner v0.0.0-20260311053803-bee01b5e2306 h1:AVp7RjXcgXrhFhw4RPgw8sSPUX/LY38aBUOmP8EI924= -github.com/kplane-dev/spanner v0.0.0-20260311053803-bee01b5e2306/go.mod h1:aswcrXDx33W3tgV36qWgut8ziOIqAopK5YPDjqVuIU4= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= From b3b272e67d677b31eea8cb318e52cb84010184e8 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 16 Jun 2026 17:53:30 -0700 Subject: [PATCH 6/6] feat(spanner): auto-apply schema in Options.Build() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires EnsureSchema into Options.Build() so a fresh Spanner backend doesn't require an out-of-band schema-apply step. EnsureSchema is now idempotent (gRPC AlreadyExists is treated as success) so the call is safe on every apiserver startup, not just first-run. Adds a 30s timeout context so a misconfigured emulator endpoint can't hang apiserver startup indefinitely. Operator UX is now: ./kplane-apiserver --storage-backend=spanner --spanner-* ... instead of: go run ./cmd/spanner-schema --... # one-time ./kplane-apiserver --storage-backend=spanner --spanner-* ... KPEP-0001 local e2e recipe in kplane-dev/apiserver no longer needs the 'apply the schema manually' caveat — Build() handles it. --- backends/spanner/config.go | 33 ++++++++++++++++++++++++++++++--- backends/spanner/register.go | 20 ++++++++++++++++---- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/backends/spanner/config.go b/backends/spanner/config.go index 1353203..e55be4b 100644 --- a/backends/spanner/config.go +++ b/backends/spanner/config.go @@ -12,7 +12,9 @@ import ( instancepb "cloud.google.com/go/spanner/admin/instance/apiv1/instancepb" "google.golang.org/api/option" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" "k8s.io/klog/v2" ) @@ -104,8 +106,11 @@ func EnsureInstance(ctx context.Context, cfg SpannerConfig) error { return nil } -// EnsureSchema creates the database and applies the KV schema if it doesn't exist. -// Intended for development / testing with the Spanner emulator. +// EnsureSchema creates the database and applies the KV schema if it doesn't +// exist. Idempotent: a pre-existing database returns nil instead of an +// error, so the function is safe to call on every apiserver startup. This +// is the seam Options.Build() uses so a fresh Spanner backend doesn't +// require a separate schema-apply step out of band. func EnsureSchema(ctx context.Context, cfg SpannerConfig) error { var opts []option.ClientOption if cfg.EmulatorHost != "" { @@ -122,17 +127,25 @@ func EnsureSchema(ctx context.Context, cfg SpannerConfig) error { } defer adminClient.Close() - // Create the database with schema in one shot. + // Try to create the database with schema in one shot. op, err := adminClient.CreateDatabase(ctx, &databasepb.CreateDatabaseRequest{ Parent: cfg.InstancePath(), CreateStatement: fmt.Sprintf("CREATE DATABASE `%s`", cfg.Database), ExtraStatements: schemaDDL, }) if err != nil { + if isAlreadyExists(err) { + klog.V(2).InfoS("Spanner database already exists, skipping schema apply", "database", cfg.DatabasePath()) + return nil + } return fmt.Errorf("creating database: %w", err) } if _, err := op.Wait(ctx); err != nil { + if isAlreadyExists(err) { + klog.V(2).InfoS("Spanner database raced to exist, skipping schema apply", "database", cfg.DatabasePath()) + return nil + } return fmt.Errorf("waiting for database creation: %w", err) } @@ -140,6 +153,20 @@ func EnsureSchema(ctx context.Context, cfg SpannerConfig) error { return nil } +// isAlreadyExists returns true if err carries a gRPC AlreadyExists status — +// the signal the Spanner database admin API uses when CreateDatabase is +// called against a database that already exists. Extracted so EnsureSchema +// can treat re-runs as a no-op without parsing string messages. +func isAlreadyExists(err error) bool { + if err == nil { + return false + } + if s, ok := status.FromError(err); ok && s.Code() == codes.AlreadyExists { + return true + } + return false +} + // DropDatabase drops the specified Spanner database. // Intended for test cleanup to avoid hitting emulator database limits. func DropDatabase(ctx context.Context, cfg SpannerConfig) error { diff --git a/backends/spanner/register.go b/backends/spanner/register.go index e279a16..41916b5 100644 --- a/backends/spanner/register.go +++ b/backends/spanner/register.go @@ -1,7 +1,9 @@ package spanner import ( + "context" "fmt" + "time" "github.com/spf13/pflag" @@ -77,11 +79,21 @@ func (o *Options) Validate() []error { // after Validate; the resulting Factory is invoked once per GroupResource // at REST registry construction time. // -// We reuse the existing NewBackendFactory so the legacy hardcoded path -// (apiserver's `if opts.SpannerProject != ""` branch) and the registry -// path produce identical store/broadcaster/watcher behavior. That keeps -// the Phase 3 cutover in the apiserver behavior-preserving. +// Build also applies the kv schema to the configured Spanner database via +// EnsureSchema. EnsureSchema is idempotent — a pre-existing database is +// a no-op — so this is safe on every startup and avoids a separate +// out-of-band schema apply step in the operator workflow. func (o *Options) Build() (registry.Factory, error) { + // EnsureSchema dials the database admin API, attempts CreateDatabase + // with the kv DDL, and treats AlreadyExists as success. A modest + // timeout keeps a misconfigured emulator endpoint from hanging the + // apiserver startup indefinitely. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := EnsureSchema(ctx, o.cfg); err != nil { + return nil, fmt.Errorf("ensuring spanner schema: %w", err) + } + bf := NewBackendFactory(o.cfg) return registry.Factory(bf), nil }