CNTRLPLANE-2800: Fix APIService race condition and gate HCP availability on aggregated API readiness

api/hypershift/v1beta1/hostedcluster_conditions.go

@@ -243,6 +243,13 @@ const (
 	// **False / AutoNodeProgressing** means AutoNode is being enabled or disabled — the operation is in progress.
 	// **False / AutoNodeNotConfigured** means AutoNode is not configured in the spec and all Karpenter components have been removed.
 	AutoNodeEnabled ConditionType = "AutoNodeEnabled"
+
+	// AggregatedAPIServicesAvailable indicates whether all aggregated APIServices
+	// in the guest cluster are available. This includes OpenShift API Server,
+	// OAuth API Server (when enabled), and OLM PackageServer APIServices.
+	// This condition is an HCP implementation detail set by the HCCO and is not
+	// bubbled up to the HostedCluster level.
+	AggregatedAPIServicesAvailable ConditionType = "AggregatedAPIServicesAvailable"
 )
 
 // Reasons.
@@ -322,6 +329,8 @@ const (
 	AutoNodeNotConfiguredReason    = "AutoNodeNotConfigured"
 	AutoNodeProgressingReason      = "AutoNodeProgressing"
 	AutoNodeEvaluationFailedReason = "AutoNodeEvaluationFailed"
+
+	AggregatedAPIServicesNotAvailableReason = "APIServicesNotAvailable"
 )
 
 // Messages.

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

@@ -725,6 +725,7 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 			healthCheckErr,
 			componentsNotAvailableMsg,
 			componentsErr,
+			alreadyAvailable,
 			hostedControlPlane.Generation,
 		)
 		hostedControlPlane.Status.Ready = ready
@@ -853,18 +854,22 @@ func (r *HostedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 // reconcileAvailabilityStatus determines the HostedControlPlane availability condition
 // and Ready flag based on the current state of all prerequisite conditions.
 // It evaluates conditions in priority order: infrastructure, kubeconfig, etcd, KAS,
-// health check, and control plane components.
+// health check, control plane components, and aggregated API services.
+// The alreadyAvailable parameter implements a latch: once the HCP is Available,
+// transient aggregated API service issues will not regress availability.
 func reconcileAvailabilityStatus(
 	conditions []metav1.Condition,
 	kubeConfigAvailable bool,
 	healthCheckErr error,
 	componentsNotAvailableMsg string,
 	componentsErr error,
+	alreadyAvailable bool,
 	generation int64,
 ) (bool, metav1.Condition) {
 	infrastructureCondition := meta.FindStatusCondition(conditions, string(hyperv1.InfrastructureReady))
 	etcdCondition := meta.FindStatusCondition(conditions, string(hyperv1.EtcdAvailable))
 	kubeAPIServerCondition := meta.FindStatusCondition(conditions, string(hyperv1.KubeAPIServerAvailable))
+	apiServicesCondition := meta.FindStatusCondition(conditions, string(hyperv1.AggregatedAPIServicesAvailable))
 
 	status := metav1.ConditionFalse
 	var reason, message string
@@ -893,6 +898,13 @@ func reconcileAvailabilityStatus(
 	case componentsNotAvailableMsg != "":
 		reason = hyperv1.ControlPlaneComponentsNotAvailable
 		message = componentsNotAvailableMsg
+	case !alreadyAvailable && (apiServicesCondition == nil || apiServicesCondition.Status != metav1.ConditionTrue):
+		reason = hyperv1.AggregatedAPIServicesNotAvailableReason
+		if apiServicesCondition != nil {
+			message = apiServicesCondition.Message
+		} else {
+			message = "Waiting for aggregated API services to be available"
+		}
 	default:
 		reason = hyperv1.AsExpectedReason
 		message = ""

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go

@@ -2202,13 +2202,20 @@ func TestRemoveHCPIngressFromRoutes(t *testing.T) {
 }
 
 func TestReconcileAvailabilityStatus(t *testing.T) {
+	allHealthyConditions := []metav1.Condition{
+		{Type: string(hyperv1.InfrastructureReady), Status: metav1.ConditionTrue, Reason: hyperv1.AsExpectedReason},
+		{Type: string(hyperv1.EtcdAvailable), Status: metav1.ConditionTrue, Reason: hyperv1.EtcdQuorumAvailableReason},
+		{Type: string(hyperv1.KubeAPIServerAvailable), Status: metav1.ConditionTrue, Reason: hyperv1.AsExpectedReason},
+	}
+
 	testCases := []struct {
 		name                      string
 		conditions                []metav1.Condition
 		kubeConfigAvailable       bool
 		healthCheckErr            error
 		componentsNotAvailableMsg string
 		componentsErr             error
+		alreadyAvailable          bool
 		generation                int64
 		expectedReady             bool
 		expectedReason            string
@@ -2414,6 +2421,11 @@ func TestReconcileAvailabilityStatus(t *testing.T) {
 					Status: metav1.ConditionTrue,
 					Reason: hyperv1.AsExpectedReason,
 				},
+				{
+					Type:   string(hyperv1.AggregatedAPIServicesAvailable),
+					Status: metav1.ConditionTrue,
+					Reason: hyperv1.AsExpectedReason,
+				},
 			},
 			kubeConfigAvailable: true,
 			expectedReady:       true,
@@ -2428,6 +2440,11 @@ func TestReconcileAvailabilityStatus(t *testing.T) {
 					Status: metav1.ConditionTrue,
 					Reason: hyperv1.AsExpectedReason,
 				},
+				{
+					Type:   string(hyperv1.AggregatedAPIServicesAvailable),
+					Status: metav1.ConditionTrue,
+					Reason: hyperv1.AsExpectedReason,
+				},
 			},
 			kubeConfigAvailable: true,
 			expectedReady:       true,
@@ -2442,6 +2459,59 @@ func TestReconcileAvailabilityStatus(t *testing.T) {
 			expectedReason:  hyperv1.StatusUnknownReason,
 			expectedMessage: "",
 		},
+		{
+			name: "When AggregatedAPIServicesAvailable condition is False and not already available, it should block availability",
+			conditions: append(append([]metav1.Condition{}, allHealthyConditions...),
+				metav1.Condition{Type: string(hyperv1.AggregatedAPIServicesAvailable), Status: metav1.ConditionFalse, Reason: hyperv1.AggregatedAPIServicesNotAvailableReason, Message: "Unavailable APIServices: v1.apps.openshift.io"},
+			),
+			kubeConfigAvailable: true,
+			alreadyAvailable:    false,
+			expectedReady:       false,
+			expectedReason:      hyperv1.AggregatedAPIServicesNotAvailableReason,
+			expectedMessage:     "Unavailable APIServices: v1.apps.openshift.io",
+		},
+		{
+			name:                "When AggregatedAPIServicesAvailable condition is nil and not already available, it should block availability",
+			conditions:          append([]metav1.Condition{}, allHealthyConditions...),
+			kubeConfigAvailable: true,
+			alreadyAvailable:    false,
+			expectedReady:       false,
+			expectedReason:      hyperv1.AggregatedAPIServicesNotAvailableReason,
+			expectedMessage:     "Waiting for aggregated API services to be available",
+		},
+		{
+			name: "When AggregatedAPIServicesAvailable condition is True, it should allow availability",
+			conditions: append(append([]metav1.Condition{}, allHealthyConditions...),
+				metav1.Condition{Type: string(hyperv1.AggregatedAPIServicesAvailable), Status: metav1.ConditionTrue, Reason: hyperv1.AsExpectedReason},
+			),
+			kubeConfigAvailable: true,
+			alreadyAvailable:    false,
+			expectedReady:       true,
+			expectedReason:      hyperv1.AsExpectedReason,
+			expectedMessage:     "",
+		},
+		{
+			name: "When already available and AggregatedAPIServicesAvailable is False, it should not regress availability",
+			conditions: append(append([]metav1.Condition{}, allHealthyConditions...),
+				metav1.Condition{Type: string(hyperv1.AggregatedAPIServicesAvailable), Status: metav1.ConditionFalse, Reason: hyperv1.AggregatedAPIServicesNotAvailableReason},
+			),
+			kubeConfigAvailable: true,
+			alreadyAvailable:    true,
+			expectedReady:       true,
+			expectedReason:      hyperv1.AsExpectedReason,
+			expectedMessage:     "",
+		},
+		{
+			name: "When AggregatedAPIServicesAvailable is Unknown and not already available, it should block availability",
+			conditions: append(append([]metav1.Condition{}, allHealthyConditions...),
+				metav1.Condition{Type: string(hyperv1.AggregatedAPIServicesAvailable), Status: metav1.ConditionUnknown, Reason: hyperv1.ReconcileErrorReason},
+			),
+			kubeConfigAvailable: true,
+			alreadyAvailable:    false,
+			expectedReady:       false,
+			expectedReason:      hyperv1.AggregatedAPIServicesNotAvailableReason,
+			expectedMessage:     "",
+		},
 	}
 
 	for _, tc := range testCases {
@@ -2454,6 +2524,7 @@ func TestReconcileAvailabilityStatus(t *testing.T) {
 				tc.healthCheckErr,
 				tc.componentsNotAvailableMsg,
 				tc.componentsErr,
+				tc.alreadyAvailable,
 				tc.generation,
 			)
 

control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go

@@ -590,43 +590,58 @@ func (r *reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result
 		errs = append(errs, fmt.Errorf("failed to update ControlPlaneConnectionAvailable condition: %w", err))
 	}
 
-	log.Info("reconciling openshift apiserver apiservices")
-	if err := r.reconcileOpenshiftAPIServerAPIServices(ctx, hcp); err != nil {
-		errs = append(errs, fmt.Errorf("failed to reconcile openshift apiserver service: %w", err))
-	}
-
+	// Reconcile Service and Endpoints before APIServices to avoid a race condition
+	// where the Kubernetes API aggregator picks up an APIService before its backend
+	// Service and Endpoints exist, causing transient 503 errors on aggregated API groups.
+	// APIService reconciliation is gated on successful Service/Endpoints reconciliation
+	// to prevent publishing an APIService that points to a non-ready backend.
+	openshiftAPIServerBackendReady := true
 	log.Info("reconciling openshift apiserver service")
 	openshiftAPIServerService := manifests.OpenShiftAPIServerClusterService()
 	if _, err := r.CreateOrUpdate(ctx, r.client, openshiftAPIServerService, func() error {
 		oapi.ReconcileClusterService(openshiftAPIServerService)
 		return nil
 	}); err != nil {
 		errs = append(errs, fmt.Errorf("failed to reconcile openshift apiserver service: %w", err))
+		openshiftAPIServerBackendReady = false
 	}
 
 	log.Info("reconciling openshift apiserver endpoints")
 	if err := r.reconcileOpenshiftAPIServerEndpoints(ctx, hcp); err != nil {
 		errs = append(errs, fmt.Errorf("failed to reconcile openshift apiserver endpoints: %w", err))
+		openshiftAPIServerBackendReady = false
 	}
 
-	if util.HCPOAuthEnabled(hcp) {
-		log.Info("reconciling openshift oauth apiserver apiservices")
-		if err := r.reconcileOpenshiftOAuthAPIServerAPIServices(ctx, hcp); err != nil {
-			errs = append(errs, fmt.Errorf("failed to reconcile openshift apiserver service: %w", err))
+	if openshiftAPIServerBackendReady {
+		log.Info("reconciling openshift apiserver apiservices")
+		if err := r.reconcileOpenshiftAPIServerAPIServices(ctx, hcp); err != nil {
+			errs = append(errs, fmt.Errorf("failed to reconcile openshift apiserver apiservices: %w", err))
 		}
+	}
 
+	if util.HCPOAuthEnabled(hcp) {
+		openshiftOAuthBackendReady := true
 		log.Info("reconciling openshift oauth apiserver service")
 		openshiftOAuthAPIServerService := manifests.OpenShiftOAuthAPIServerClusterService()
 		if _, err := r.CreateOrUpdate(ctx, r.client, openshiftOAuthAPIServerService, func() error {
 			oapi.ReconcileClusterService(openshiftOAuthAPIServerService)
 			return nil
 		}); err != nil {
 			errs = append(errs, fmt.Errorf("failed to reconcile openshift oauth apiserver service: %w", err))
+			openshiftOAuthBackendReady = false
 		}
 
 		log.Info("reconciling openshift oauth apiserver endpoints")
 		if err := r.reconcileOpenshiftOAuthAPIServerEndpoints(ctx, hcp); err != nil {
-			errs = append(errs, fmt.Errorf("failed to reconcile openshift apiserver endpoints: %w", err))
+			errs = append(errs, fmt.Errorf("failed to reconcile openshift oauth apiserver endpoints: %w", err))
+			openshiftOAuthBackendReady = false
+		}
+
+		if openshiftOAuthBackendReady {
+			log.Info("reconciling openshift oauth apiserver apiservices")
+			if err := r.reconcileOpenshiftOAuthAPIServerAPIServices(ctx, hcp); err != nil {
+				errs = append(errs, fmt.Errorf("failed to reconcile openshift oauth apiserver apiservices: %w", err))
+			}
 		}
 
 		log.Info("reconciling kubeadmin password hash secret")
@@ -763,6 +778,11 @@ func (r *reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result
 	log.Info("reconciling olm resources")
 	errs = append(errs, r.reconcileOLM(ctx, hcp, pullSecret)...)
 
+	log.Info("reconciling aggregated API services available condition")
+	if err := r.reconcileAggregatedAPIServicesAvailableCondition(ctx, hcp); err != nil {
+		errs = append(errs, fmt.Errorf("failed to update AggregatedAPIServicesAvailable condition: %w", err))
+	}
+
 	log.Info("reconciling kubelet configs")
 	if err := r.reconcileKubeletConfig(ctx); err != nil {
 		errs = append(errs, fmt.Errorf("failed to reconcile kubelet config: %w", err))
@@ -1817,6 +1837,67 @@ func (r *reconciler) reconcileControlPlaneConnectionAvailable(ctx context.Contex
 	return r.patchHCPStatusCondition(ctx, hcp, condition)
 }
 
+// reconcileAggregatedAPIServicesAvailableCondition checks all expected aggregated
+// APIServices in the guest cluster and sets the AggregatedAPIServicesAvailable
+// condition on the HostedControlPlane.
+func (r *reconciler) reconcileAggregatedAPIServicesAvailableCondition(ctx context.Context, hcp *hyperv1.HostedControlPlane) error {
+	condition := &metav1.Condition{
+		Type: string(hyperv1.AggregatedAPIServicesAvailable),
+	}
+
+	expected := sets.New[string]()
+	for _, group := range manifests.OpenShiftAPIServerAPIServiceGroups() {
+		expected.Insert(manifests.OpenShiftAPIServerAPIService(group).Name)
+	}
+	if util.HCPOAuthEnabled(hcp) {
+		for _, group := range manifests.OpenShiftOAuthAPIServerAPIServiceGroups() {
+			expected.Insert(manifests.OpenShiftOAuthAPIServerAPIService(group).Name)
+		}
+	}
+	expected.Insert(manifests.OLMPackageServerAPIService().Name)
+
+	apiServiceList := &apiregistrationv1.APIServiceList{}
+	if err := r.client.List(ctx, apiServiceList); err != nil {
+		condition.Status = metav1.ConditionFalse
+		condition.Reason = hyperv1.ReconcileErrorReason
+		condition.Message = fmt.Sprintf("Failed to list APIServices: %v", err)
+		return r.patchHCPStatusCondition(ctx, hcp, condition)
+	}
+
+	found := sets.New[string]()
+	for i := range apiServiceList.Items {
+		apiSvc := &apiServiceList.Items[i]
+		if !expected.Has(apiSvc.Name) {
+			continue
+		}
+		available := false
+		for _, c := range apiSvc.Status.Conditions {
+			if c.Type == apiregistrationv1.Available {
+				if c.Status == apiregistrationv1.ConditionTrue {
+					available = true
+				}
+				break
+			}
+		}
+		if available {
+			found.Insert(apiSvc.Name)
+		}
+	}
+	unavailable := sets.List(expected.Difference(found))
+
+	if len(unavailable) > 0 {
+		condition.Status = metav1.ConditionFalse
+		condition.Reason = hyperv1.AggregatedAPIServicesNotAvailableReason
+		condition.Message = fmt.Sprintf("Unavailable APIServices: %s", strings.Join(unavailable, ", "))
+	} else {
+		condition.Status = metav1.ConditionTrue
+		condition.Reason = hyperv1.AsExpectedReason
+		condition.Message = hyperv1.AllIsWellMessage
+	}
+
+	return r.patchHCPStatusCondition(ctx, hcp, condition)
+}
+
 func (r *reconciler) reconcileOpenshiftAPIServerAPIServices(ctx context.Context, hcp *hyperv1.HostedControlPlane) error {
 	rootCA := cpomanifests.RootCASecret(hcp.Namespace)
 	if err := r.cpClient.Get(ctx, client.ObjectKeyFromObject(rootCA), rootCA); err != nil {
@@ -2322,19 +2403,12 @@ func (r *reconciler) reconcileOLM(ctx context.Context, hcp *hyperv1.HostedContro
 		}
 	}
 
-	rootCA := cpomanifests.RootCASecret(hcp.Namespace)
-	if err := r.cpClient.Get(ctx, client.ObjectKeyFromObject(rootCA), rootCA); err != nil {
-		errs = append(errs, fmt.Errorf("failed to get root ca cert from control plane namespace: %w", err))
-	} else {
-		packageServerAPIService := manifests.OLMPackageServerAPIService()
-		if _, err := r.CreateOrUpdate(ctx, r.client, packageServerAPIService, func() error {
-			olm.ReconcilePackageServerAPIService(packageServerAPIService, rootCA)
-			return nil
-		}); err != nil {
-			errs = append(errs, fmt.Errorf("failed to reconcile OLM packageserver API service: %w", err))
-		}
-	}
-
+	// Reconcile Service and Endpoints before APIService to avoid a race condition
+	// where the Kubernetes API aggregator picks up the APIService before its backend
+	// Service and Endpoints exist, causing transient 503 errors on packages.operators.coreos.com.
+	// APIService reconciliation is gated on successful backend readiness to prevent
+	// publishing an APIService that points to a non-ready backend.
+	packageServerBackendReady := true
 	packageServerService := manifests.OLMPackageServerService()
 	if _, err := r.CreateOrUpdate(ctx, r.client, packageServerService, func() error {
 		olm.ReconcilePackageServerService(packageServerService)
@@ -2346,16 +2420,34 @@ func (r *reconciler) reconcileOLM(ctx context.Context, hcp *hyperv1.HostedContro
 	cpService := manifests.OLMPackageServerControlPlaneService(hcp.Namespace)
 	if err := r.cpClient.Get(ctx, client.ObjectKeyFromObject(cpService), cpService); err != nil {
 		errs = append(errs, fmt.Errorf("failed to get packageserver service from control plane namespace: %w", err))
+		packageServerBackendReady = false
 	} else {
 		if len(cpService.Spec.ClusterIP) == 0 {
 			errs = append(errs, fmt.Errorf("packageserver service does not yet have a cluster IP"))
+			packageServerBackendReady = false
 		} else {
 			packageServerEndpoints := manifests.OLMPackageServerEndpoints()
 			if _, err := r.CreateOrUpdate(ctx, r.client, packageServerEndpoints, func() error {
 				olm.ReconcilePackageServerEndpoints(packageServerEndpoints, cpService.Spec.ClusterIP)
 				return nil
 			}); err != nil {
-				errs = append(errs, fmt.Errorf("failed to reconcile OLM packageserver service: %w", err))
+				errs = append(errs, fmt.Errorf("failed to reconcile OLM packageserver endpoints: %w", err))
+				packageServerBackendReady = false
+			}
+		}
+	}
+
+	if packageServerBackendReady {
+		rootCA := cpomanifests.RootCASecret(hcp.Namespace)
+		if err := r.cpClient.Get(ctx, client.ObjectKeyFromObject(rootCA), rootCA); err != nil {
+			errs = append(errs, fmt.Errorf("failed to get root ca cert from control plane namespace: %w", err))
+		} else {
+			packageServerAPIService := manifests.OLMPackageServerAPIService()
+			if _, err := r.CreateOrUpdate(ctx, r.client, packageServerAPIService, func() error {
+				olm.ReconcilePackageServerAPIService(packageServerAPIService, rootCA)
+				return nil
+			}); err != nil {
+				errs = append(errs, fmt.Errorf("failed to reconcile OLM packageserver API service: %w", err))
 			}
 		}
 	}