diff --git a/api/hypershift/v1beta1/hostedcluster_conditions.go b/api/hypershift/v1beta1/hostedcluster_conditions.go index 2a00effd35d0..ceca209597c5 100644 --- a/api/hypershift/v1beta1/hostedcluster_conditions.go +++ b/api/hypershift/v1beta1/hostedcluster_conditions.go @@ -266,6 +266,12 @@ const ( // cluster's shared ingress. Status reflects observed state: True means // public endpoints are reachable, False means they are not. PublicEndpointExposed ConditionType = "PublicEndpointExposed" + + // HostedClusterDeleting indicates whether the HostedCluster is being deleted and + // provides first-class visibility into which phase of deletion the cluster is in. + // **False / AsExpected** means the cluster is not being deleted. + // **True** means deletion is in progress; the Reason and Message indicate the current phase. + HostedClusterDeleting ConditionType = "HostedClusterDeleting" ) // Reasons for PublicEndpointExposed condition. @@ -370,6 +376,17 @@ const ( ReEncryptionCompletedReason = "ReEncryptionCompleted" ReEncryptionFailedReason = "ReEncryptionFailed" ReEncryptionWaitingForKASReason = "ReEncryptionWaitingForKASConvergence" + + // HostedClusterDeleting reasons track progress through each phase of deletion. + DeletionWaitingForNodePoolDeletionReason = "WaitingForNodePoolDeletion" + DeletionWaitingForCAPIClusterDeletionReason = "WaitingForCAPIClusterDeletion" + DeletionWaitingForEndpointServiceDeletionReason = "WaitingForEndpointServiceDeletion" + DeletionWaitingForPrivateConnectDeletionReason = "WaitingForPrivateConnectDeletion" + DeletionWaitingForControlPlaneDeletionReason = "WaitingForControlPlaneDeletion" + DeletionWaitingForNamespaceDeletionReason = "WaitingForNamespaceDeletion" + // DeletionCompletedReason indicates all hosted cluster resources have been torn down. + // The HostedCluster object itself may still exist briefly until the finalizer is removed. + DeletionCompletedReason = "DeletionCompleted" ) // Messages. diff --git a/docs/content/reference/aggregated-docs.md b/docs/content/reference/aggregated-docs.md index 394dca417c44..143196656c73 100644 --- a/docs/content/reference/aggregated-docs.md +++ b/docs/content/reference/aggregated-docs.md @@ -42985,6 +42985,12 @@ When this is false for too long and there’s no clear indication in the &ld

HostedClusterDegraded indicates whether the HostedCluster is encountering an error that may require user intervention to resolve.

+

"HostedClusterDeleting"

+

HostedClusterDeleting indicates whether the HostedCluster is being deleted and +provides first-class visibility into which phase of deletion the cluster is in. +False / AsExpected means the cluster is not being deleted. +True means deletion is in progress; the Reason and Message indicate the current phase.

+

"HostedClusterDestroyed"

HostedClusterDestroyed indicates that a hosted has finished destroying and that it is waiting for a destroy grace period to go away. The grace period is determined by the hypershift.openshift.io/destroy-grace-period annotation in the HostedCluster if present.

diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index cd14f57b98ca..8e450a74e2b8 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -6207,6 +6207,12 @@ When this is false for too long and there’s no clear indication in the &ld

HostedClusterDegraded indicates whether the HostedCluster is encountering an error that may require user intervention to resolve.

+

"HostedClusterDeleting"

+

HostedClusterDeleting indicates whether the HostedCluster is being deleted and +provides first-class visibility into which phase of deletion the cluster is in. +False / AsExpected means the cluster is not being deleted. +True means deletion is in progress; the Reason and Message indicate the current phase.

+

"HostedClusterDestroyed"

HostedClusterDestroyed indicates that a hosted has finished destroying and that it is waiting for a destroy grace period to go away. The grace period is determined by the hypershift.openshift.io/destroy-grace-period annotation in the HostedCluster if present.

diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index fe05b7b0d149..b706fe73668e 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -24,6 +24,7 @@ import ( "net/netip" "os" "reflect" + "sort" "strconv" "strings" "time" @@ -497,6 +498,14 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques } } + // Keep the steady-state non-deleting condition current. + if hcluster.DeletionTimestamp.IsZero() { + if err := r.reconcileDeletingConditionSteadyState(ctx, hcluster); err != nil { + return ctrl.Result{}, err + } + } + + // If deleted, clean up and return early. if !hcluster.DeletionTimestamp.IsZero() { // This new condition is necessary for OCM personnel to report any cloud dangling objects to the user. // The grace period is customizable using an annotation called HCDestroyGracePeriodAnnotation. It's a time.Duration annotation. @@ -3736,11 +3745,39 @@ func deleteControlPlaneOperatorRBAC(ctx context.Context, c client.Client, rbacNa return nil } +func (r *HostedClusterReconciler) reconcileDeletingConditionSteadyState(ctx context.Context, hc *hyperv1.HostedCluster) error { + if updated := meta.SetStatusCondition(&hc.Status.Conditions, metav1.Condition{ + Type: string(hyperv1.HostedClusterDeleting), + Status: metav1.ConditionFalse, + Reason: hyperv1.AsExpectedReason, + Message: "HostedCluster is not being deleted", + ObservedGeneration: hc.Generation, + }); updated { + if err := r.Client.Status().Update(ctx, hc); err != nil { + return fmt.Errorf("failed to reconcile HostedClusterDeleting condition: %w", err) + } + } + return nil +} + //nolint:gocyclo func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.HostedCluster) (bool, error) { controlPlaneNamespace := manifests.HostedControlPlaneNamespace(hc.Namespace, hc.Name) log := ctrl.LoggerFrom(ctx) + setDeletionProgress := func(reason, message string) error { + if updated := meta.SetStatusCondition(&hc.Status.Conditions, metav1.Condition{ + Type: string(hyperv1.HostedClusterDeleting), + Status: metav1.ConditionTrue, + Reason: reason, + Message: message, + ObservedGeneration: hc.Generation, + }); !updated { + return nil + } + return r.Client.Status().Update(ctx, hc) + } + // Unpause CAPI cluster to allow deletion to proceed if err := pauseCAPICluster(ctx, r.Client, hc, false); err != nil { return false, err @@ -3774,6 +3811,24 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted return false, err } + remainingNodePools, err := listNodePools(ctx, r.Client, hc.Namespace, hc.Name) + if err != nil { + return false, fmt.Errorf("failed to list NodePools: %w", err) + } + if len(remainingNodePools) > 0 { + npNames := make([]string, len(remainingNodePools)) + for i := range remainingNodePools { + npNames[i] = remainingNodePools[i].Name + } + sort.Strings(npNames) + log.Info("Waiting for NodePool deletion", "remaining", len(remainingNodePools), "nodepools", npNames) + if err := setDeletionProgress(hyperv1.DeletionWaitingForNodePoolDeletionReason, + fmt.Sprintf("Waiting for %d NodePool(s) to be deleted: %s", len(remainingNodePools), strings.Join(npNames, ", "))); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } + return false, nil + } + p, err := platform.GetPlatform(ctx, hc, nil, "", nil) if err != nil { return false, err @@ -3797,6 +3852,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted if exists { log.Info("Waiting for cluster deletion", "clusterName", hc.Spec.InfraID, "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForCAPIClusterDeletionReason, + fmt.Sprintf("Waiting for CAPI cluster %s/%s to be deleted", controlPlaneNamespace, hc.Spec.InfraID)); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } else { // once infra is deleted remove finalizers. @@ -3836,6 +3895,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted } if exists { log.Info("Waiting for awsendpointservice deletion", "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForEndpointServiceDeletionReason, + fmt.Sprintf("Waiting for AWS endpoint services in %s to be deleted", controlPlaneNamespace)); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } } @@ -3847,6 +3910,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted } if exists { log.Info("Waiting for gcpprivateserviceconnect deletion", "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForPrivateConnectDeletionReason, + fmt.Sprintf("Waiting for GCP Private Service Connect resources in %s to be deleted", controlPlaneNamespace)); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } } @@ -3902,6 +3969,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted } if exists { log.Info("Waiting for hostedcontrolplane deletion", "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForControlPlaneDeletionReason, + fmt.Sprintf("Waiting for HostedControlPlane %s/%s to be deleted", controlPlaneNamespace, hc.Name)); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } @@ -3916,6 +3987,9 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted r.KubevirtInfraClients.Delete(hc.Spec.InfraID) if skipNSDeletion := hc.Annotations[hyperv1.SkipControlPlaneNamespaceDeletionAnnotation]; skipNSDeletion == "true" { + if err := setDeletionProgress(hyperv1.DeletionCompletedReason, "Deletion completed (namespace deletion skipped)"); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return true, nil } @@ -3928,10 +4002,49 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted return false, err } if exists { - log.Info("Waiting for namespace deletion", "controlPlaneNamespace", controlPlaneNamespace) + message := fmt.Sprintf("Waiting for namespace %s to be deleted", controlPlaneNamespace) + + // Fetch the namespace to inspect its phase and conditions + ns := &corev1.Namespace{} + if getErr := r.Client.Get(ctx, types.NamespacedName{Name: controlPlaneNamespace}, ns); getErr == nil { + message = fmt.Sprintf("Waiting for namespace %s to be deleted (phase: %s)", controlPlaneNamespace, ns.Status.Phase) + var details []string + for _, cond := range ns.Status.Conditions { + switch cond.Type { + case corev1.NamespaceContentRemaining, + corev1.NamespaceFinalizersRemaining, + corev1.NamespaceDeletionContentFailure: + if cond.Status == corev1.ConditionTrue { + details = append(details, fmt.Sprintf("%s: %s", cond.Type, cond.Message)) + log.Info("Namespace deletion blocked", + "controlPlaneNamespace", controlPlaneNamespace, + "conditionType", cond.Type, + "reason", cond.Reason, + "message", cond.Message, + ) + } + } + } + if len(details) > 0 { + message = fmt.Sprintf("Waiting for namespace %s to be deleted (phase: %s): %s", + controlPlaneNamespace, ns.Status.Phase, strings.Join(details, "; ")) + const maxMessageLen = 1024 + if len(message) > maxMessageLen { + message = message[:maxMessageLen-3] + "..." + } + } + } + + log.Info(message, "controlPlaneNamespace", controlPlaneNamespace) + if err := setDeletionProgress(hyperv1.DeletionWaitingForNamespaceDeletionReason, message); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return false, nil } + if err := setDeletionProgress(hyperv1.DeletionCompletedReason, "Deletion completed"); err != nil { + return false, fmt.Errorf("failed to update deletion progress: %w", err) + } return true, nil } diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go index fcb01733e739..7d45eeea7903 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go @@ -8832,3 +8832,498 @@ func TestReconcileSREMetricsConfig_EffectiveMetricsSet(t *testing.T) { }) } } + +func TestDeleteSetDeletionProgressIdempotency(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "test-infra-id", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{ + Conditions: []metav1.Condition{ + { + Type: string(hyperv1.HostedClusterDeleting), + Status: metav1.ConditionTrue, + Reason: hyperv1.DeletionWaitingForNodePoolDeletionReason, + Message: "Waiting for 1 NodePool(s) to be deleted: np-1", + ObservedGeneration: 1, + }, + }, + }, + } + + np := &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "np-1", + Namespace: "clusters", + Finalizers: []string{"hypershift.openshift.io/test-finalizer"}, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "test-cluster", + Platform: hyperv1.NodePoolPlatform{Type: hyperv1.NonePlatform}, + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Management: hyperv1.NodePoolManagement{UpgradeType: hyperv1.UpgradeTypeReplace}, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc, np). + WithStatusSubresource(hc). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + + // First call sets the condition. + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + // Fetch the resource version after first call. + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + rvAfterFirst := updatedHC.ResourceVersion + + // Second call with same state should be idempotent (no status update). + hc.ResourceVersion = updatedHC.ResourceVersion + hc.Status = updatedHC.Status + completed, err = r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + g.Expect(updatedHC.ResourceVersion).To(Equal(rvAfterFirst)) +} + +func TestDeleteNodePoolTeardownPhase(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + + tests := []struct { + name string + existingNodePools []hyperv1.NodePool + expectedConditionReason string + expectedMessage string + }{ + { + name: "When NodePools remain after deletion it should set WaitingForNodePoolDeletion and return incomplete", + existingNodePools: []hyperv1.NodePool{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "np-1", + Namespace: "clusters", + Finalizers: []string{"hypershift.openshift.io/test-finalizer"}, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "test-cluster", + Platform: hyperv1.NodePoolPlatform{Type: hyperv1.NonePlatform}, + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Management: hyperv1.NodePoolManagement{UpgradeType: hyperv1.UpgradeTypeReplace}, + }, + }, + }, + expectedConditionReason: hyperv1.DeletionWaitingForNodePoolDeletionReason, + expectedMessage: "Waiting for 1 NodePool(s) to be deleted: np-1", + }, + { + name: "When multiple NodePools remain it should report the count in the message", + existingNodePools: []hyperv1.NodePool{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "np-1", + Namespace: "clusters", + Finalizers: []string{"hypershift.openshift.io/test-finalizer"}, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "test-cluster", + Platform: hyperv1.NodePoolPlatform{Type: hyperv1.NonePlatform}, + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Management: hyperv1.NodePoolManagement{UpgradeType: hyperv1.UpgradeTypeReplace}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "np-2", + Namespace: "clusters", + Finalizers: []string{"hypershift.openshift.io/test-finalizer"}, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "test-cluster", + Platform: hyperv1.NodePoolPlatform{Type: hyperv1.NonePlatform}, + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Management: hyperv1.NodePoolManagement{UpgradeType: hyperv1.UpgradeTypeReplace}, + }, + }, + }, + expectedConditionReason: hyperv1.DeletionWaitingForNodePoolDeletionReason, + expectedMessage: "Waiting for 2 NodePool(s) to be deleted: np-1, np-2", + }, + { + name: "When no NodePools exist it should proceed past the NodePool phase", + existingNodePools: nil, + expectedConditionReason: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "test-infra-id", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + objs := []crclient.Object{hc} + for i := range tc.existingNodePools { + objs = append(objs, &tc.existingNodePools[i]) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(objs...). + WithStatusSubresource(&hyperv1.HostedCluster{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + + if tc.expectedConditionReason != "" { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(tc.expectedConditionReason)) + g.Expect(cond.Message).To(Equal(tc.expectedMessage)) + g.Expect(cond.ObservedGeneration).To(Equal(int64(1))) + } else { + // No NodePools case: it proceeds past the NodePool phase into later + // deletion stages. We verify it did not set WaitingForNodePoolDeletion. + g.Expect(err).ToNot(HaveOccurred()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + if cond != nil { + g.Expect(cond.Reason).ToNot(Equal(hyperv1.DeletionWaitingForNodePoolDeletionReason)) + } + } + }) + } +} + +func TestDeleteCAPIClusterTeardownPhase(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + controlPlaneNamespace := "clusters-test-cluster" + + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "test-infra-id", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + capiCluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-infra-id", + Namespace: controlPlaneNamespace, + DeletionTimestamp: &now, + Finalizers: []string{"capi.cluster.x-k8s.io/test-finalizer"}, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc, capiCluster). + WithStatusSubresource(&hyperv1.HostedCluster{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(hyperv1.DeletionWaitingForCAPIClusterDeletionReason)) + g.Expect(cond.Message).To(ContainSubstring("Waiting for CAPI cluster")) +} + +func TestDeleteNamespaceTeardownPhase(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + controlPlaneNamespace := "clusters-test-cluster" + + t.Run("When namespace has blocking conditions it should include them in the message", func(t *testing.T) { + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: controlPlaneNamespace, + DeletionTimestamp: &now, + Finalizers: []string{"kubernetes"}, + }, + Status: corev1.NamespaceStatus{ + Phase: corev1.NamespaceTerminating, + Conditions: []corev1.NamespaceCondition{ + { + Type: corev1.NamespaceContentRemaining, + Status: corev1.ConditionTrue, + Message: "Some resources still exist in the namespace", + }, + { + Type: corev1.NamespaceFinalizersRemaining, + Status: corev1.ConditionTrue, + Message: "Some content in the namespace has finalizers", + }, + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc, ns). + WithStatusSubresource(&hyperv1.HostedCluster{}, &corev1.Namespace{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(hyperv1.DeletionWaitingForNamespaceDeletionReason)) + g.Expect(cond.Message).To(ContainSubstring("NamespaceContentRemaining")) + g.Expect(cond.Message).To(ContainSubstring("NamespaceFinalizersRemaining")) + g.Expect(cond.Message).To(ContainSubstring("Terminating")) + }) + + t.Run("When namespace has no blocking conditions it should report waiting with phase only", func(t *testing.T) { + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: controlPlaneNamespace, + DeletionTimestamp: &now, + Finalizers: []string{"kubernetes"}, + }, + Status: corev1.NamespaceStatus{ + Phase: corev1.NamespaceTerminating, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc, ns). + WithStatusSubresource(&hyperv1.HostedCluster{}, &corev1.Namespace{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeFalse()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(hyperv1.DeletionWaitingForNamespaceDeletionReason)) + g.Expect(cond.Message).To(ContainSubstring("Terminating")) + g.Expect(cond.Message).ToNot(ContainSubstring("NamespaceContentRemaining")) + }) +} + +func TestDeleteCompleted(t *testing.T) { + g := NewGomegaWithT(t) + + now := metav1.Now() + + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + DeletionTimestamp: &now, + Finalizers: []string{"hypershift.openshift.io/finalizer"}, + Generation: 1, + Annotations: map[string]string{ + hyperv1.SkipControlPlaneNamespaceDeletionAnnotation: "true", + }, + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.NonePlatform}, + InfraID: "", + Release: hyperv1.Release{Image: "quay.io/test/release:latest"}, + Networking: hyperv1.ClusterNetworking{ + ServiceNetwork: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.31.0.0/16")}}, + ClusterNetwork: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.132.0.0/14")}}, + }, + Services: []hyperv1.ServicePublishingStrategyMapping{}, + PullSecret: corev1.LocalObjectReference{Name: "pull-secret"}, + }, + Status: hyperv1.HostedClusterStatus{}, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hc). + WithStatusSubresource(&hyperv1.HostedCluster{}). + Build() + + r := &HostedClusterReconciler{ + Client: fakeClient, + KubevirtInfraClients: kvinfra.NewKubevirtInfraClientMap(), + ManagementClusterCapabilities: &fakecapabilities.FakeSupportNoCapabilities{}, + } + + ctx := ctrl.LoggerInto(t.Context(), zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout))) + completed, err := r.delete(ctx, hc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(completed).To(BeTrue()) + + updatedHC := &hyperv1.HostedCluster{} + g.Expect(fakeClient.Get(ctx, types.NamespacedName{Namespace: "clusters", Name: "test-cluster"}, updatedHC)).To(Succeed()) + + cond := meta.FindStatusCondition(updatedHC.Status.Conditions, string(hyperv1.HostedClusterDeleting)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(hyperv1.DeletionCompletedReason)) + g.Expect(cond.Message).To(ContainSubstring("namespace deletion skipped")) +} diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_conditions.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_conditions.go index 2a00effd35d0..ceca209597c5 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_conditions.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_conditions.go @@ -266,6 +266,12 @@ const ( // cluster's shared ingress. Status reflects observed state: True means // public endpoints are reachable, False means they are not. PublicEndpointExposed ConditionType = "PublicEndpointExposed" + + // HostedClusterDeleting indicates whether the HostedCluster is being deleted and + // provides first-class visibility into which phase of deletion the cluster is in. + // **False / AsExpected** means the cluster is not being deleted. + // **True** means deletion is in progress; the Reason and Message indicate the current phase. + HostedClusterDeleting ConditionType = "HostedClusterDeleting" ) // Reasons for PublicEndpointExposed condition. @@ -370,6 +376,17 @@ const ( ReEncryptionCompletedReason = "ReEncryptionCompleted" ReEncryptionFailedReason = "ReEncryptionFailed" ReEncryptionWaitingForKASReason = "ReEncryptionWaitingForKASConvergence" + + // HostedClusterDeleting reasons track progress through each phase of deletion. + DeletionWaitingForNodePoolDeletionReason = "WaitingForNodePoolDeletion" + DeletionWaitingForCAPIClusterDeletionReason = "WaitingForCAPIClusterDeletion" + DeletionWaitingForEndpointServiceDeletionReason = "WaitingForEndpointServiceDeletion" + DeletionWaitingForPrivateConnectDeletionReason = "WaitingForPrivateConnectDeletion" + DeletionWaitingForControlPlaneDeletionReason = "WaitingForControlPlaneDeletion" + DeletionWaitingForNamespaceDeletionReason = "WaitingForNamespaceDeletion" + // DeletionCompletedReason indicates all hosted cluster resources have been torn down. + // The HostedCluster object itself may still exist briefly until the finalizer is removed. + DeletionCompletedReason = "DeletionCompleted" ) // Messages.