From 78e0ae12a500ddb309554d21474f3915da02e772 Mon Sep 17 00:00:00 2001 From: Luca Miccini Date: Tue, 23 Jun 2026 07:27:22 +0200 Subject: [PATCH] Add transport URL secret rotation with consumer finalizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When infra-operator rotates a RabbitMQ transport URL (creating a new secret and user), consumer operators must hold a consumer finalizer on the old secret until all their pods have rolled out with the new credentials. Without this, infra-operator cleans up the old RabbitMQ user while pods are still connected with old credentials, causing message bus outages. Design: 1. Add consumer finalizer to the current transport URL secret early in reconcile. Set instance.Status.TransportURLSecret for first-time setup only (when empty or unchanged); during rotation the status is updated solely by FinalizeSecretRotation at end of reconcile. 2. Pass transportURL.Status.SecretName directly to sub-CR creation functions and config generation as a parameter — never read from instance.Status.TransportURLSecret for sub-CR specs. 3. Use statefulset.IsReady() / deployment.IsReady() from lib-common in all sub-CR controllers for accurate rollout status. 4. Use object.ManageRotationGracePeriod() to enforce a 60-second grace period before evaluating the rotation guard. This gives sub-CRs time to detect config changes, update their workloads, and roll pods — without relying on informer cache freshness. 5. Guard: CredentialRotationGuardReady(true, conditions) — evaluates AllSubConditionIsTrue after the grace period expires. Only when all sub-CR conditions are True does FinalizeSecretRotation remove the consumer finalizer from the old secret. The same pattern applies to notification transport URL secrets and application credential secrets where applicable. Co-Authored-By: Claude Opus 4.6 (1M context) --- api/go.mod | 7 +- api/go.sum | 8 +- api/v1beta1/keystoneapplicationcredential.go | 75 --------- go.mod | 7 +- go.sum | 8 +- internal/controller/keystoneapi_controller.go | 52 +++++- internal/keystone/const.go | 2 + .../functional/keystoneapi_controller_test.go | 152 ++++++++++++++++++ 8 files changed, 218 insertions(+), 93 deletions(-) diff --git a/api/go.mod b/api/go.mod index bc0e524e..577ac61c 100644 --- a/api/go.mod +++ b/api/go.mod @@ -74,6 +74,10 @@ require ( sigs.k8s.io/yaml v1.6.0 // indirect ) +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/lmiccini/lib-common/modules/common v0.0.0-20260625081740-8dfcc3e3c06c + +replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 + // mschuppert: map to latest commit from release-4.18 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e //allow-merging @@ -97,7 +101,4 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging replace sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.19.7 //allow-merging -// custom RabbitmqClusterSpecCore for OpenStackControlplane (v2.16.0_patches) -replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging - replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging diff --git a/api/go.sum b/api/go.sum index 9c3b4905..e8e02bc5 100644 --- a/api/go.sum +++ b/api/go.sum @@ -68,6 +68,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 h1:CiQFCVrdzGgeEQqt3C39e6s9Vpi1784HIm+LNYm9py4= +github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8= +github.com/lmiccini/lib-common/modules/common v0.0.0-20260625081740-8dfcc3e3c06c h1:yp6cz5FNiYRAwlQ6FhODK7Vjj3FyebynlR9cDY8Rahk= +github.com/lmiccini/lib-common/modules/common v0.0.0-20260625081740-8dfcc3e3c06c/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -85,10 +89,6 @@ github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4= github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyUt0GEdoAE+r5TXy7YS21yNEo+2U= github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36 h1:nGpBRRuWJbxiH9Vv5ir0TUWmL3XFChvqvXX8We5Lvnc= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:aIc5ECO3dubv265jjUZ66oi56kf5iUt8Y1DWmCPrzOc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:OVFoNXzinsI0rq8gbegu8TnlDPkO409iyVoWhU4nEdQ= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:7yqbVpg0k0vW+kZks+TMU/cd1ovoejyHfVPWcyGYLHI= github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:3cyU3HUhCoV7vscqea6ZUbkwxNSAJd1Rwk0P15vsUZw= diff --git a/api/v1beta1/keystoneapplicationcredential.go b/api/v1beta1/keystoneapplicationcredential.go index 884a7ba9..99ea1ff8 100644 --- a/api/v1beta1/keystoneapplicationcredential.go +++ b/api/v1beta1/keystoneapplicationcredential.go @@ -17,17 +17,8 @@ limitations under the License. package v1beta1 import ( - "context" "fmt" "strings" - - corev1 "k8s.io/api/core/v1" - k8s_errors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/openstack-k8s-operators/lib-common/modules/common/helper" - "github.com/openstack-k8s-operators/lib-common/modules/common/object" ) // ApplicationCredentialData contains AC ID/Secret extracted from a Secret @@ -66,69 +57,3 @@ const ( func (ac *KeystoneApplicationCredential) IsEDPMService() bool { return ac.GetAnnotations()[EDPMServiceAnnotation] != "false" } - -// ManageACSecretFinalizer ensures consumerFinalizer is present on the AC secret -// identified by newSecretName and absent from the one identified by -// oldSecretName. It is a no-op when both names are equal. -func ManageACSecretFinalizer( - ctx context.Context, - h *helper.Helper, - namespace string, - newSecretName string, - oldSecretName string, - consumerFinalizer string, -) error { - if newSecretName == oldSecretName { - return nil - } - - var newObj, oldObj client.Object - - if newSecretName != "" { - secret := &corev1.Secret{} - key := types.NamespacedName{Name: newSecretName, Namespace: namespace} - if err := h.GetClient().Get(ctx, key, secret); err != nil { - return fmt.Errorf("failed to get new AC secret %s: %w", newSecretName, err) - } - newObj = secret - } - - if oldSecretName != "" { - secret := &corev1.Secret{} - key := types.NamespacedName{Name: oldSecretName, Namespace: namespace} - if err := h.GetClient().Get(ctx, key, secret); err != nil { - if !k8s_errors.IsNotFound(err) { - return fmt.Errorf("failed to get old AC secret %s: %w", oldSecretName, err) - } - } else { - oldObj = secret - } - } - - return object.ManageConsumerFinalizer(ctx, h, newObj, oldObj, consumerFinalizer) -} - -// RemoveACSecretConsumerFinalizer removes consumerFinalizer from the AC secret -// identified by secretName. It is a no-op when secretName is empty or the -// secret no longer exists. -func RemoveACSecretConsumerFinalizer( - ctx context.Context, - h *helper.Helper, - namespace string, - secretName string, - consumerFinalizer string, -) error { - if secretName == "" { - return nil - } - - secret := &corev1.Secret{} - key := types.NamespacedName{Name: secretName, Namespace: namespace} - if err := h.GetClient().Get(ctx, key, secret); err != nil { - if k8s_errors.IsNotFound(err) { - return nil - } - return err - } - return object.RemoveConsumerFinalizer(ctx, h, secret, consumerFinalizer) -} diff --git a/go.mod b/go.mod index 1318b881..e298c6d8 100644 --- a/go.mod +++ b/go.mod @@ -118,6 +118,10 @@ require ( replace github.com/openstack-k8s-operators/keystone-operator/api => ./api +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/lmiccini/lib-common/modules/common v0.0.0-20260625081740-8dfcc3e3c06c + +replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 + // mschuppert: map to latest commit from release-4.18 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e //allow-merging @@ -141,7 +145,4 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging replace sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.19.7 //allow-merging -// custom RabbitmqClusterSpecCore for OpenStackControlplane (v2.16.0_patches) -replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging - replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging diff --git a/go.sum b/go.sum index a9593b74..85dbb2f4 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 h1:CiQFCVrdzGgeEQqt3C39e6s9Vpi1784HIm+LNYm9py4= +github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8= +github.com/lmiccini/lib-common/modules/common v0.0.0-20260625081740-8dfcc3e3c06c h1:yp6cz5FNiYRAwlQ6FhODK7Vjj3FyebynlR9cDY8Rahk= +github.com/lmiccini/lib-common/modules/common v0.0.0-20260625081740-8dfcc3e3c06c/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= @@ -120,10 +124,6 @@ github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4= github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyUt0GEdoAE+r5TXy7YS21yNEo+2U= github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36 h1:nGpBRRuWJbxiH9Vv5ir0TUWmL3XFChvqvXX8We5Lvnc= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:aIc5ECO3dubv265jjUZ66oi56kf5iUt8Y1DWmCPrzOc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0= github.com/openstack-k8s-operators/lib-common/modules/edpm v0.0.0-20260618132757-fe8e60d1d8a6 h1:Iq7reRH6OtQ3MCqA/Tiz6e8fqfYi+h1o5cyZv0RNv0E= github.com/openstack-k8s-operators/lib-common/modules/edpm v0.0.0-20260618132757-fe8e60d1d8a6/go.mod h1:xsKeDFU3/xEObaVDqd6XEYV3MzvFswbWMlnr2Z3q3ZI= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:OVFoNXzinsI0rq8gbegu8TnlDPkO409iyVoWhU4nEdQ= diff --git a/internal/controller/keystoneapi_controller.go b/internal/controller/keystoneapi_controller.go index 53b638c9..bc9f4f69 100644 --- a/internal/controller/keystoneapi_controller.go +++ b/internal/controller/keystoneapi_controller.go @@ -42,6 +42,7 @@ import ( job "github.com/openstack-k8s-operators/lib-common/modules/common/job" labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/object" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" @@ -522,6 +523,11 @@ func (r *KeystoneAPIReconciler) reconcileDelete(ctx context.Context, instance *k } } + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, + instance.Status.TransportURLSecret, keystone.TransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info("Reconciled Service delete successfully") @@ -1069,9 +1075,9 @@ func (r *KeystoneAPIReconciler) reconcileNormal( Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) } - instance.Status.TransportURLSecret = transportURL.Status.SecretName + currentTransportSecret := transportURL.Status.SecretName - if instance.Status.TransportURLSecret == "" { + if currentTransportSecret == "" { // Since the TransportURL secret is automatically created by the Infra operator, // we treat this as an info (because the user is not responsible for manually creating it). Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) @@ -1084,6 +1090,19 @@ func (r *KeystoneAPIReconciler) reconcileNormal( } Log.Info(fmt.Sprintf("TransportURL secret name %s", transportURL.Status.SecretName)) instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) + + // Set status early for first-time setup so PatchInstance persists it + // even on early returns. During rotation (old != current), the status + // is only updated by FinalizeSecretRotation at end of reconcile. + if instance.Status.TransportURLSecret == "" || + instance.Status.TransportURLSecret == currentTransportSecret { + instance.Status.TransportURLSecret = currentTransportSecret + } + + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, + currentTransportSecret, keystone.TransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } // run check rabbitmq - end // @@ -1152,7 +1171,7 @@ func (r *KeystoneAPIReconciler) reconcileNormal( // - %-config configmap holding minimal keystone config required to get the service up, user can add additional files to be added to the service // - parameters which has passwords gets added from the OpenStack secret via the init container // - err = r.generateServiceConfigMaps(ctx, instance, helper, &configMapVars, memcached, db) + err = r.generateServiceConfigMaps(ctx, instance, helper, &configMapVars, memcached, db, currentTransportSecret) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -1468,6 +1487,30 @@ func (r *KeystoneAPIReconciler) reconcileNormal( return ctrl.Result{}, err } + rotationPending := instance.Status.TransportURLSecret != "" && + instance.Status.TransportURLSecret != currentTransportSecret + result, graceActive, err := object.ManageRotationGracePeriod( + ctx, r.Client, instance, rotationPending, 60*time.Second) + if err != nil { + return ctrl.Result{}, err + } + if graceActive { + return result, nil + } + + guardReady := condition.CredentialRotationGuardReady(true, &instance.Status.Conditions) + transportSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.TransportURLSecret, + currentTransportSecret, + keystone.TransportConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.TransportURLSecret = transportSecretName + Log.Info("Reconciled Service successfully") return ctrl.Result{}, nil } @@ -1520,6 +1563,7 @@ func (r *KeystoneAPIReconciler) generateServiceConfigMaps( envVars *map[string]env.Setter, mc *memcachedv1.Memcached, db *mariadbv1.Database, + transportURLSecretName string, ) error { // // create Configmap/Secret required for keystone input @@ -1545,7 +1589,7 @@ func (r *KeystoneAPIReconciler) generateServiceConfigMaps( } maps.Copy(customData, instance.Spec.DefaultConfigOverwrite) - transportURLSecret, _, err := oko_secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + transportURLSecret, _, err := oko_secret.GetSecret(ctx, h, transportURLSecretName, instance.Namespace) if err != nil { return err } diff --git a/internal/keystone/const.go b/internal/keystone/const.go index 1e8902a4..e896b74d 100644 --- a/internal/keystone/const.go +++ b/internal/keystone/const.go @@ -53,6 +53,8 @@ const ( FederationMultiRealmSecret = "keystone-multirealm-federation-secret" // FederationDefaultMountPath - if user doesn't specify otherwise, this location is used FederationDefaultMountPath = "/var/lib/config-data/default/multirealm-federation" + // TransportConsumerFinalizer is the finalizer added to transport URL secrets + TransportConsumerFinalizer = "openstack.org/keystone-transport-consumer" ) // KeystonePropagation is the definition of the Keystone propagation service diff --git a/test/functional/keystoneapi_controller_test.go b/test/functional/keystoneapi_controller_test.go index 421a808e..a18947e0 100644 --- a/test/functional/keystoneapi_controller_test.go +++ b/test/functional/keystoneapi_controller_test.go @@ -35,6 +35,7 @@ import ( memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + keystoneinternal "github.com/openstack-k8s-operators/keystone-operator/internal/keystone" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" mariadb_test "github.com/openstack-k8s-operators/mariadb-operator/api/test/helpers" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -2787,4 +2788,155 @@ OIDCRedirectURI "{{ .KeystoneEndpointPublic }}/v3/auth/OS-FEDERATION/websso/open Expect(err.Error()).To(ContainSubstring("gracePeriodDays must be smaller than expirationDays")) }) }) + + When("TransportURL consumer finalizer is managed", func() { + BeforeEach(func() { + DeferCleanup(th.DeleteInstance, CreateKeystoneAPI(keystoneAPIName, GetDefaultKeystoneAPISpec())) + DeferCleanup( + k8sClient.Delete, ctx, CreateKeystoneMessageBusSecret(namespace, "rabbitmq-secret")) + DeferCleanup( + k8sClient.Delete, ctx, CreateKeystoneAPISecret(namespace, SecretName)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + namespace, + GetKeystoneAPI(keystoneAPIName).Spec.DatabaseInstance, + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + mariadb.SimulateMariaDBAccountCompleted(keystoneAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(keystoneDatabaseName) + infra.SimulateTransportURLReady(types.NamespacedName{ + Name: fmt.Sprintf("%s-keystone-transport", keystoneAPIName.Name), + Namespace: namespace, + }) + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(namespace, "memcached", memcachedSpec)) + infra.SimulateMemcachedReady(types.NamespacedName{ + Name: "memcached", + Namespace: namespace, + }) + th.SimulateJobSuccess(dbSyncJobName) + th.SimulateJobSuccess(bootstrapJobName) + th.SimulateDeploymentReplicaReady(deploymentName) + }) + + It("should add the consumer finalizer to the transport secret", func() { + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: namespace, + Name: "rabbitmq-secret", + }) + g.Expect(secret.Finalizers).To( + ContainElement(keystoneinternal.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + }) + + It("should remove the consumer finalizer from transport secret on CR deletion", func() { + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: namespace, + Name: "rabbitmq-secret", + }) + g.Expect(secret.Finalizers).To( + ContainElement(keystoneinternal.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + th.DeleteInstance(GetKeystoneAPI(keystoneAPIName)) + + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: namespace, + Name: "rabbitmq-secret", + }) + g.Expect(secret.Finalizers).NotTo( + ContainElement(keystoneinternal.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + }) + + It("should move the finalizer from the old to the new secret on transport rotation", func() { + oldSecretName := "rabbitmq-secret" + + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: namespace, + Name: oldSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(keystoneinternal.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Wait for keystone to reach fully ready state + Eventually(func(g Gomega) { + instance := GetKeystoneAPI(keystoneAPIName) + g.Expect(instance.Status.Conditions.IsTrue(condition.ReadyCondition)).To(BeTrue()) + g.Expect(instance.Status.TransportURLSecret).To(Equal(oldSecretName)) + }, timeout, interval).Should(Succeed()) + + newSecretName := "rabbitmq-secret-rotated" + + // Create the new rotated secret with different content + newSecret := th.CreateSecret( + types.NamespacedName{ + Namespace: namespace, + Name: newSecretName, + }, + map[string][]byte{ + "transport_url": []byte("rabbit://rotated-user:rotated-pass@rabbitmq/fake"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, newSecret) + + // Simulate transport rotation: update TransportURL status with new secret name + transportURLName := types.NamespacedName{ + Name: fmt.Sprintf("%s-keystone-transport", keystoneAPIName.Name), + Namespace: namespace, + } + Eventually(func(g Gomega) { + transport := infra.GetTransportURL(transportURLName) + transport.Status.SecretName = newSecretName + g.Expect(k8sClient.Status().Update(ctx, transport)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Verify finalizer is added to the new secret + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: namespace, + Name: newSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(keystoneinternal.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // The old secret's finalizer should NOT be removed yet -- deployment + // is re-deploying with new credentials and is not ready + Consistently(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: namespace, + Name: oldSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(keystoneinternal.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Simulate deployment becoming ready with the new credentials + Eventually(func(g Gomega) { + th.SimulateDeploymentReplicaReady(deploymentName) + // Verify old finalizer is removed + secret := th.GetSecret(types.NamespacedName{ + Namespace: namespace, + Name: oldSecretName, + }) + g.Expect(secret.Finalizers).NotTo( + ContainElement(keystoneinternal.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Verify status tracks the new secret + Eventually(func(g Gomega) { + instance := GetKeystoneAPI(keystoneAPIName) + g.Expect(instance.Status.TransportURLSecret).To(Equal(newSecretName)) + }, timeout, interval).Should(Succeed()) + }) + }) })