From 18d5eded0e97955e99aa1d5590e4fb3d225f98d7 Mon Sep 17 00:00:00 2001 From: Cesar Wong Date: Thu, 21 May 2026 17:21:04 -0400 Subject: [PATCH] feat(ho): add HostedClusterDeleting condition to track deletion progress Add a new HostedClusterDeleting status condition that surfaces the current phase of HostedCluster teardown. The condition is set to True with a phase-specific reason during deletion and reconciled to False when the cluster is not being deleted. Include NodePool names in the deletion progress message, truncate long namespace condition messages to 1024 characters, and use the return value of meta.SetStatusCondition to skip redundant status updates. Co-Authored-By: Claude Opus 4.6 --- .../v1beta1/hostedcluster_conditions.go | 17 + docs/content/reference/aggregated-docs.md | 6 + docs/content/reference/api.md | 6 + .../hostedcluster/hostedcluster_controller.go | 115 +++- .../hostedcluster_controller_test.go | 495 ++++++++++++++++++ .../v1beta1/hostedcluster_conditions.go | 17 + 6 files changed, 655 insertions(+), 1 deletion(-) diff --git a/api/hypershift/v1beta1/hostedcluster_conditions.go b/api/hypershift/v1beta1/hostedcluster_conditions.go index 2a00effd35d0..ceca209597c5 100644 --- a/api/hypershift/v1beta1/hostedcluster_conditions.go +++ b/api/hypershift/v1beta1/hostedcluster_conditions.go @@ -266,6 +266,12 @@ const ( // cluster's shared ingress. Status reflects observed state: True means // public endpoints are reachable, False means they are not. PublicEndpointExposed ConditionType = "PublicEndpointExposed" + + // HostedClusterDeleting indicates whether the HostedCluster is being deleted and + // provides first-class visibility into which phase of deletion the cluster is in. + // **False / AsExpected** means the cluster is not being deleted. + // **True** means deletion is in progress; the Reason and Message indicate the current phase. + HostedClusterDeleting ConditionType = "HostedClusterDeleting" ) // Reasons for PublicEndpointExposed condition. @@ -370,6 +376,17 @@ const ( ReEncryptionCompletedReason = "ReEncryptionCompleted" ReEncryptionFailedReason = "ReEncryptionFailed" ReEncryptionWaitingForKASReason = "ReEncryptionWaitingForKASConvergence" + + // HostedClusterDeleting reasons track progress through each phase of deletion. + DeletionWaitingForNodePoolDeletionReason = "WaitingForNodePoolDeletion" + DeletionWaitingForCAPIClusterDeletionReason = "WaitingForCAPIClusterDeletion" + DeletionWaitingForEndpointServiceDeletionReason = "WaitingForEndpointServiceDeletion" + DeletionWaitingForPrivateConnectDeletionReason = "WaitingForPrivateConnectDeletion" + DeletionWaitingForControlPlaneDeletionReason = "WaitingForControlPlaneDeletion" + DeletionWaitingForNamespaceDeletionReason = "WaitingForNamespaceDeletion" + // DeletionCompletedReason indicates all hosted cluster resources have been torn down. + // The HostedCluster object itself may still exist briefly until the finalizer is removed. + DeletionCompletedReason = "DeletionCompleted" ) // Messages. diff --git a/docs/content/reference/aggregated-docs.md b/docs/content/reference/aggregated-docs.md index 394dca417c44..143196656c73 100644 --- a/docs/content/reference/aggregated-docs.md +++ b/docs/content/reference/aggregated-docs.md @@ -42985,6 +42985,12 @@ When this is false for too long and there’s no clear indication in the &ld

HostedClusterDegraded indicates whether the HostedCluster is encountering an error that may require user intervention to resolve.

+

"HostedClusterDeleting"

+

HostedClusterDeleting indicates whether the HostedCluster is being deleted and +provides first-class visibility into which phase of deletion the cluster is in. +False / AsExpected means the cluster is not being deleted. +True means deletion is in progress; the Reason and Message indicate the current phase.

+

"HostedClusterDestroyed"

HostedClusterDestroyed indicates that a hosted has finished destroying and that it is waiting for a destroy grace period to go away. The grace period is determined by the hypershift.openshift.io/destroy-grace-period annotation in the HostedCluster if present.

diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index cd14f57b98ca..8e450a74e2b8 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -6207,6 +6207,12 @@ When this is false for too long and there’s no clear indication in the &ld

HostedClusterDegraded indicates whether the HostedCluster is encountering an error that may require user intervention to resolve.

+

"HostedClusterDeleting"

+

HostedClusterDeleting indicates whether the HostedCluster is being deleted and +provides first-class visibility into which phase of deletion the cluster is in. +False / AsExpected means the cluster is not being deleted. +True means deletion is in progress; the Reason and Message indicate the current phase.

+

"HostedClusterDestroyed"

HostedClusterDestroyed indicates that a hosted has finished destroying and that it is waiting for a destroy grace period to go away. The grace period is determined by the hypershift.openshift.io/destroy-grace-period annotation in the HostedCluster if present.

diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index fe05b7b0d149..b706fe73668e 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -24,6 +24,7 @@ import ( "net/netip" "os" "reflect" + "sort" "strconv" "strings" "time" @@ -497,6 +498,14 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques } } + // Keep the steady-state non-deleting condition current. + if hcluster.DeletionTimestamp.IsZero() { + if err := r.reconcileDeletingConditionSteadyState(ctx, hcluster); err != nil { + return ctrl.Result{}, err + } + } + + // If deleted, clean up and return early. if !hcluster.DeletionTimestamp.IsZero() { // This new condition is necessary for OCM personnel to report any cloud dangling objects to the user. // The grace period is customizable using an annotation called HCDestroyGracePeriodAnnotation. It's a time.Duration annotation. @@ -3736,11 +3745,39 @@ func deleteControlPlaneOperatorRBAC(ctx context.Context, c client.Client, rbacNa return nil } +func (r *HostedClusterReconciler) reconcileDeletingConditionSteadyState(ctx context.Context, hc *hyperv1.HostedCluster) error { + if updated := meta.SetStatusCondition(&hc.Status.Conditions, metav1.Condition{ + Type: string(hyperv1.HostedClusterDeleting), + Status: metav1.ConditionFalse, + Reason: hyperv1.AsExpectedReason, + Message: "HostedCluster is not being deleted", + ObservedGeneration: hc.Generation, + }); updated { + if err := r.Client.Status().Update(ctx, hc); err != nil { + return fmt.Errorf("failed to reconcile HostedClusterDeleting condition: %w", err) + } + } + return nil +} + //nolint:gocyclo func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.HostedCluster) (bool, error) { controlPlaneNamespace := manifests.HostedControlPlaneNamespace(hc.Namespace, hc.Name) log := ctrl.LoggerFrom(ctx) + setDeletionProgress := func(reason, message string) error { + if updated := meta.SetStatusCondition(&hc.Status.Conditions, metav1.Condition{ + Type: string(hyperv1.HostedClusterDeleting), + Status: metav1.ConditionTrue, + Reason: reason, + Message: message, + ObservedGeneration: hc.Generation, + }); !updated { + return nil + } + return r.Client.Status().Update(ctx, hc) + } + // Unpause CAPI cluster to allow deletion to proceed if err := pauseCAPICluster(ctx, r.Client, hc, false); err != nil { return false, err @@ -3774,6 +3811,24 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted return false, err } + remainingNodePools, err := listNodePools(ctx, r.Client, hc.Namespace, hc.Name) + if err != nil { + return false, fmt.Errorf("failed to list NodePools: %w", err) + } + if len(remainingNodePools) > 0 { + npNames := make([]string, len(remainingNodePools)) + for i := range remainingNodePools { + npNames[i] = remainingNodePools[i].Name + } + sort.Strings(npNames) + log.Info("Waiting for NodePool deletion", "remaining", len(remainingNodePools), "nodepools", npNames) + if err := setDeletionProgress(hyperv1.DeletionWaitingForNodePoolDeletionReason, + fmt.Sprintf("Waiting for %d NodePool(s) to be deleted: %s", len(remainingNodePools), strings.Join(npNames, ", "))); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } + return false, nil + } + p, err := platform.GetPlatform(ctx, hc, nil, "", nil) if err != nil { return false, err @@ -3797,6 +3852,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted if exists { log.Info("Waiting for cluster deletion", "clusterName", hc.Spec.InfraID, "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForCAPIClusterDeletionReason, + fmt.Sprintf("Waiting for CAPI cluster %s/%s to be deleted", controlPlaneNamespace, hc.Spec.InfraID)); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } else { // once infra is deleted remove finalizers. @@ -3836,6 +3895,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted } if exists { log.Info("Waiting for awsendpointservice deletion", "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForEndpointServiceDeletionReason, + fmt.Sprintf("Waiting for AWS endpoint services in %s to be deleted", controlPlaneNamespace)); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } } @@ -3847,6 +3910,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted } if exists { log.Info("Waiting for gcpprivateserviceconnect deletion", "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForPrivateConnectDeletionReason, + fmt.Sprintf("Waiting for GCP Private Service Connect resources in %s to be deleted", controlPlaneNamespace)); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } } @@ -3902,6 +3969,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted } if exists { log.Info("Waiting for hostedcontrolplane deletion", "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForControlPlaneDeletionReason, + fmt.Sprintf("Waiting for HostedControlPlane %s/%s to be deleted", controlPlaneNamespace, hc.Name)); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } @@ -3916,6 +3987,9 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted r.KubevirtInfraClients.Delete(hc.Spec.InfraID) if skipNSDeletion := hc.Annotations[hyperv1.SkipControlPlaneNamespaceDeletionAnnotation]; skipNSDeletion == "true" { + if err := setDeletionProgress(hyperv1.DeletionCompletedReason, "Deletion completed (namespace deletion skipped)"); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return true, nil } @@ -3928,10 +4002,49 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted return false, err } if exists { - log.Info("Waiting for namespace deletion", "controlPlaneNamespace", controlPlaneNamespace) + message := fmt.Sprintf("Waiting for namespace %s to be deleted", controlPlaneNamespace) + + // Fetch the namespace to inspect its phase and conditions + ns := &corev1.Namespace{} + if getErr := r.Client.Get(ctx, types.NamespacedName{Name: controlPlaneNamespace}, ns); getErr == nil { + message = fmt.Sprintf("Waiting for namespace %s to be deleted (phase: %s)", controlPlaneNamespace, ns.Status.Phase) + var details []string + for _, cond := range ns.Status.Conditions { + switch cond.Type { + case corev1.NamespaceContentRemaining, + corev1.NamespaceFinalizersRemaining, + corev1.NamespaceDeletionContentFailure: + if cond.Status == corev1.ConditionTrue { + details = append(details, fmt.Sprintf("%s: %s", cond.Type, cond.Message)) + log.Info("Namespace deletion blocked", + "controlPlaneNamespace", controlPlaneNamespace, + "conditionType", cond.Type, + "reason", cond.Reason, + "message", cond.Message, + ) + } + } + } + if len(details) > 0 { + message = fmt.Sprintf("Waiting for namespace %s to be deleted (phase: %s): %s", + controlPlaneNamespace, ns.Status.Phase, strings.Join(details, "; ")) + const maxMessageLen = 1024 + if len(message) > maxMessageLen { + message = message[:maxMessageLen-3] + "..." + } + } + } + + log.Info(message, "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForNamespaceDeletionReason, message); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } + if err := setDeletionProgress(hyperv1.DeletionCompletedReason, "Deletion completed"); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return true, nil } diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go index fcb01733e739..7d45eeea7903 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go @@ -8832,3 +8832,498 @@ func TestReconcileSREMetricsConfig_EffectiveMetricsSet(t *testing.T) { }) } } + +func TestDeleteSetDeletionProgressIdempotency(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "test-infra-id", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{ + Conditions: []metav1.Condition{ + { + Type: string(hyperv1.HostedClusterDeleting), + Status: metav1.ConditionTrue, + Reason: hyperv1.DeletionWaitingForNodePoolDeletionReason, + Message: "Waiting for 1 NodePool(s) to be deleted: np-1", + ObservedGeneration: 1, + }, + }, + }, + } + + np := &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "np-1", + Namespace: "clusters", + Finalizers: []string{"hypershift.openshift.io/test-finalizer"}, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "test-cluster", + Platform: hyperv1.NodePoolPlatform{Type: hyperv1.NonePlatform}, + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Management: hyperv1.NodePoolManagement{UpgradeType: hyperv1.UpgradeTypeReplace}, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc, np). + WithStatusSubresource(hc). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + + // First call sets the condition. + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + // Fetch the resource version after first call. + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + rvAfterFirst := updatedHC.ResourceVersion + + // Second call with same state should be idempotent (no status update). + hc.ResourceVersion = updatedHC.ResourceVersion + hc.Status = updatedHC.Status + completed, err = r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + g.Expect(updatedHC.ResourceVersion).To(Equal(rvAfterFirst)) +} + +func TestDeleteNodePoolTeardownPhase(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + + tests := []struct { + name string + existingNodePools []hyperv1.NodePool + expectedConditionReason string + expectedMessage string + }{ + { + name: "When NodePools remain after deletion it should set WaitingForNodePoolDeletion and return incomplete", + existingNodePools: []hyperv1.NodePool{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "np-1", + Namespace: "clusters", + Finalizers: []string{"hypershift.openshift.io/test-finalizer"}, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "test-cluster", + Platform: hyperv1.NodePoolPlatform{Type: hyperv1.NonePlatform}, + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Management: hyperv1.NodePoolManagement{UpgradeType: hyperv1.UpgradeTypeReplace}, + }, + }, + }, + expectedConditionReason: hyperv1.DeletionWaitingForNodePoolDeletionReason, + expectedMessage: "Waiting for 1 NodePool(s) to be deleted: np-1", + }, + { + name: "When multiple NodePools remain it should report the count in the message", + existingNodePools: []hyperv1.NodePool{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "np-1", + Namespace: "clusters", + Finalizers: []string{"hypershift.openshift.io/test-finalizer"}, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "test-cluster", + Platform: hyperv1.NodePoolPlatform{Type: hyperv1.NonePlatform}, + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Management: hyperv1.NodePoolManagement{UpgradeType: hyperv1.UpgradeTypeReplace}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "np-2", + Namespace: "clusters", + Finalizers: []string{"hypershift.openshift.io/test-finalizer"}, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "test-cluster", + Platform: hyperv1.NodePoolPlatform{Type: hyperv1.NonePlatform}, + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Management: hyperv1.NodePoolManagement{UpgradeType: hyperv1.UpgradeTypeReplace}, + }, + }, + }, + expectedConditionReason: hyperv1.DeletionWaitingForNodePoolDeletionReason, + expectedMessage: "Waiting for 2 NodePool(s) to be deleted: np-1, np-2", + }, + { + name: "When no NodePools exist it should proceed past the NodePool phase", + existingNodePools: nil, + expectedConditionReason: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "test-infra-id", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + objs := []crclient.Object{hc} + for i := range tc.existingNodePools { + objs = append(objs, &tc.existingNodePools[i]) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(objs...). + WithStatusSubresource(&hyperv1.HostedCluster{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + + if tc.expectedConditionReason != "" { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(tc.expectedConditionReason)) + g.Expect(cond.Message).To(Equal(tc.expectedMessage)) + g.Expect(cond.ObservedGeneration).To(Equal(int64(1))) + } else { + // No NodePools case: it proceeds past the NodePool phase into later + // deletion stages. We verify it did not set WaitingForNodePoolDeletion. + g.Expect(err).ToNot(HaveOccurred()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + if cond != nil { + g.Expect(cond.Reason).ToNot(Equal(hyperv1.DeletionWaitingForNodePoolDeletionReason)) + } + } + }) + } +} + +func TestDeleteCAPIClusterTeardownPhase(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + controlPlaneNamespace := "clusters-test-cluster" + + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "test-infra-id", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + capiCluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-infra-id", + Namespace: controlPlaneNamespace, + DeletionTimestamp: &now, + Finalizers: []string{"capi.cluster.x-k8s.io/test-finalizer"}, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc, capiCluster). + WithStatusSubresource(&hyperv1.HostedCluster{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(hyperv1.DeletionWaitingForCAPIClusterDeletionReason)) + g.Expect(cond.Message).To(ContainSubstring("Waiting for CAPI cluster")) +} + +func TestDeleteNamespaceTeardownPhase(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + controlPlaneNamespace := "clusters-test-cluster" + + t.Run("When namespace has blocking conditions it should include them in the message", func(t *testing.T) { + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: controlPlaneNamespace, + DeletionTimestamp: &now, + Finalizers: []string{"kubernetes"}, + }, + Status: corev1.NamespaceStatus{ + Phase: corev1.NamespaceTerminating, + Conditions: []corev1.NamespaceCondition{ + { + Type: corev1.NamespaceContentRemaining, + Status: corev1.ConditionTrue, + Message: "Some resources still exist in the namespace", + }, + { + Type: corev1.NamespaceFinalizersRemaining, + Status: corev1.ConditionTrue, + Message: "Some content in the namespace has finalizers", + }, + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc, ns). + WithStatusSubresource(&hyperv1.HostedCluster{}, &corev1.Namespace{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(hyperv1.DeletionWaitingForNamespaceDeletionReason)) + g.Expect(cond.Message).To(ContainSubstring("NamespaceContentRemaining")) + g.Expect(cond.Message).To(ContainSubstring("NamespaceFinalizersRemaining")) + g.Expect(cond.Message).To(ContainSubstring("Terminating")) + }) + + t.Run("When namespace has no blocking conditions it should report waiting with phase only", func(t *testing.T) { + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: controlPlaneNamespace, + DeletionTimestamp: &now, + Finalizers: []string{"kubernetes"}, + }, + Status: corev1.NamespaceStatus{ + Phase: corev1.NamespaceTerminating, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc, ns). + WithStatusSubresource(&hyperv1.HostedCluster{}, &corev1.Namespace{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(hyperv1.DeletionWaitingForNamespaceDeletionReason)) + g.Expect(cond.Message).To(ContainSubstring("Terminating")) + g.Expect(cond.Message).ToNot(ContainSubstring("NamespaceContentRemaining")) + }) +} + +func TestDeleteCompleted(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + Annotations: map[string]string{ + hyperv1.SkipControlPlaneNamespaceDeletionAnnotation: "true", + }, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc). + WithStatusSubresource(&hyperv1.HostedCluster{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeTrue()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(hyperv1.DeletionCompletedReason)) + g.Expect(cond.Message).To(ContainSubstring("namespace deletion skipped")) +} diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_conditions.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_conditions.go index 2a00effd35d0..ceca209597c5 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_conditions.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_conditions.go @@ -266,6 +266,12 @@ const ( // cluster's shared ingress. Status reflects observed state: True means // public endpoints are reachable, False means they are not. PublicEndpointExposed ConditionType = "PublicEndpointExposed" + + // HostedClusterDeleting indicates whether the HostedCluster is being deleted and + // provides first-class visibility into which phase of deletion the cluster is in. + // **False / AsExpected** means the cluster is not being deleted. + // **True** means deletion is in progress; the Reason and Message indicate the current phase. + HostedClusterDeleting ConditionType = "HostedClusterDeleting" ) // Reasons for PublicEndpointExposed condition. @@ -370,6 +376,17 @@ const ( ReEncryptionCompletedReason = "ReEncryptionCompleted" ReEncryptionFailedReason = "ReEncryptionFailed" ReEncryptionWaitingForKASReason = "ReEncryptionWaitingForKASConvergence" + + // HostedClusterDeleting reasons track progress through each phase of deletion. + DeletionWaitingForNodePoolDeletionReason = "WaitingForNodePoolDeletion" + DeletionWaitingForCAPIClusterDeletionReason = "WaitingForCAPIClusterDeletion" + DeletionWaitingForEndpointServiceDeletionReason = "WaitingForEndpointServiceDeletion" + DeletionWaitingForPrivateConnectDeletionReason = "WaitingForPrivateConnectDeletion" + DeletionWaitingForControlPlaneDeletionReason = "WaitingForControlPlaneDeletion" + DeletionWaitingForNamespaceDeletionReason = "WaitingForNamespaceDeletion" + // DeletionCompletedReason indicates all hosted cluster resources have been torn down. + // The HostedCluster object itself may still exist briefly until the finalizer is removed. + DeletionCompletedReason = "DeletionCompleted" ) // Messages.