From 49a0c849dbced1688ae2b64adca25c16aeecc720 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 14 Jun 2024 15:09:31 -0500 Subject: [PATCH 01/13] Add PVC volume expansion --- controllers/solr_cluster_ops_util.go | 59 ++++++++++++++++++++++- controllers/solrcloud_controller.go | 71 ++++++++++++++++++++++++---- controllers/util/solr_util.go | 24 ++++++++++ 3 files changed, 143 insertions(+), 11 deletions(-) diff --git a/controllers/solr_cluster_ops_util.go b/controllers/solr_cluster_ops_util.go index 916446b3..fa1952a5 100644 --- a/controllers/solr_cluster_ops_util.go +++ b/controllers/solr_cluster_ops_util.go @@ -27,6 +27,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" "net/url" @@ -53,6 +54,7 @@ const ( ScaleUpLock SolrClusterOperationType = "ScalingUp" UpdateLock SolrClusterOperationType = "RollingUpdate" BalanceReplicasLock SolrClusterOperationType = "BalanceReplicas" + PvcExpansionLock SolrClusterOperationType = "PVCExpansion" ) // RollingUpdateMetadata contains metadata for rolling update cluster operations. @@ -150,6 +152,60 @@ func retryNextQueuedClusterOpWithQueue(statefulSet *appsv1.StatefulSet, clusterO return hasOp, err } +func determinePvcExpansionClusterOpLockIfNecessary(instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet) (clusterOp *SolrClusterOp, retryLaterDuration time.Duration, err error) { + if instance.Spec.StorageOptions.PersistentStorage != nil && + instance.Spec.StorageOptions.PersistentStorage.PersistentVolumeClaimTemplate.Spec.Resources.Requests.Storage() != nil && + instance.Spec.StorageOptions.PersistentStorage.PersistentVolumeClaimTemplate.Spec.Resources.Requests.Storage().String() != statefulSet.Annotations[util.StorageMinimumSizeAnnotation] { + // First make sure that the new Storage request is greater than what already is set. + // PVCs cannot be shrunk + newSize := instance.Spec.StorageOptions.PersistentStorage.PersistentVolumeClaimTemplate.Spec.Resources.Requests.Storage() + // If there is no old size to update, the StatefulSet can be just set to use the new PVC size without any issue. + // Only do a cluster operation if we are expanding from an existing size to a new size + if oldSizeStr, hasOldSize := statefulSet.Annotations[util.StorageMinimumSizeAnnotation]; hasOldSize { + if oldSize, e := resource.ParseQuantity(oldSizeStr); e != nil { + err = e + // TODO: add an event + } else { + // Only update to the new size if it is bigger, we cannot shrink PVCs + if newSize.Cmp(oldSize) > 0 { + clusterOp = &SolrClusterOp{ + Operation: PvcExpansionLock, + Metadata: newSize.String(), + } + } + // TODO: add an event saying that we cannot shrink PVCs + } + } + } + return +} + +// handleManagedCloudScaleUp does the logic of a managed and "locked" cloud scale up operation. +// This will likely take many reconcile loops to complete, as it is moving replicas to the pods that have recently been scaled up. +func handlePvcExpansion(ctx context.Context, r *SolrCloudReconciler, instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, clusterOp *SolrClusterOp, logger logr.Logger) (operationComplete bool, retryLaterDuration time.Duration, err error) { + var newSize resource.Quantity + newSize, err = resource.ParseQuantity(clusterOp.Metadata) + if err != nil { + logger.Error(err, "Could not convert PvcExpansion metadata to a resource.Quantity, as it represents the new size of PVCs", "metadata", clusterOp.Metadata) + return + } + operationComplete, err = r.expandPVCs(ctx, instance, statefulSet.Spec.Selector.MatchLabels, newSize, logger) + if err == nil && operationComplete { + originalStatefulSet := statefulSet.DeepCopy() + statefulSet.Annotations[util.StorageMinimumSizeAnnotation] = newSize.String() + statefulSet.Spec.Template.Annotations[util.StorageMinimumSizeAnnotation] = newSize.String() + if err = r.Patch(ctx, statefulSet, client.StrategicMergeFrom(originalStatefulSet)); err != nil { + logger.Error(err, "Error while patching StatefulSet to set the new minimum PVC size after PVCs the completion of PVC resizing", "newSize", newSize) + operationComplete = false + } + // Return and wait for the StatefulSet to be updated which will call the reconcile to start the rolling restart + retryLaterDuration = 0 + } else if err == nil { + retryLaterDuration = time.Second * 5 + } + return +} + func determineScaleClusterOpLockIfNecessary(ctx context.Context, r *SolrCloudReconciler, instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, scaleDownOpIsQueued bool, podList []corev1.Pod, blockReconciliationOfStatefulSet bool, logger logr.Logger) (clusterOp *SolrClusterOp, retryLaterDuration time.Duration, err error) { desiredPods := int(*instance.Spec.Replicas) configuredPods := int(*statefulSet.Spec.Replicas) @@ -291,7 +347,8 @@ func cleanupManagedCloudScaleDown(ctx context.Context, r *SolrCloudReconciler, p // handleManagedCloudScaleUp does the logic of a managed and "locked" cloud scale up operation. // This will likely take many reconcile loops to complete, as it is moving replicas to the pods that have recently been scaled up. func handleManagedCloudScaleUp(ctx context.Context, r *SolrCloudReconciler, instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, clusterOp *SolrClusterOp, podList []corev1.Pod, logger logr.Logger) (operationComplete bool, nextClusterOperation *SolrClusterOp, err error) { - desiredPods, err := strconv.Atoi(clusterOp.Metadata) + desiredPods := 0 + desiredPods, err = strconv.Atoi(clusterOp.Metadata) if err != nil { logger.Error(err, "Could not convert ScaleUp metadata to int, as it represents the number of nodes to scale to", "metadata", clusterOp.Metadata) return diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index 9940ff9e..5e87d166 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -22,6 +22,7 @@ import ( "crypto/md5" "fmt" policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "reflect" "sort" @@ -483,6 +484,8 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( operationComplete, nextClusterOperation, err = handleManagedCloudScaleUp(ctx, r, instance, statefulSet, clusterOp, podList, logger) case BalanceReplicasLock: operationComplete, requestInProgress, retryLaterDuration, err = util.BalanceReplicasForCluster(ctx, instance, statefulSet, clusterOp.Metadata, clusterOp.Metadata, logger) + case PvcExpansionLock: + operationComplete, retryLaterDuration, err = handlePvcExpansion(ctx, r, instance, statefulSet, clusterOp, logger) default: operationFound = false // This shouldn't happen, but we don't want to be stuck if it does. @@ -550,6 +553,12 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( clusterOpQueue[queueIdx] = *clusterOp clusterOp = nil } + clusterOp, retryLaterDuration, err = determinePvcExpansionClusterOpLockIfNecessary(instance, statefulSet) + // If the new clusterOperation is an update to a queued clusterOp, just change the operation that is already queued + if queueIdx, opIsQueued := queuedRetryOps[UpdateLock]; clusterOp != nil && opIsQueued { + clusterOpQueue[queueIdx] = *clusterOp + clusterOp = nil + } // If a non-managed scale needs to take place, this method will update the StatefulSet without starting // a "locked" cluster operation @@ -932,6 +941,46 @@ func (r *SolrCloudReconciler) reconcileZk(ctx context.Context, logger logr.Logge return nil } +func (r *SolrCloudReconciler) expandPVCs(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string, newSize resource.Quantity, logger logr.Logger) (expansionComplete bool, err error) { + var pvcList corev1.PersistentVolumeClaimList + pvcList, err = r.getPVCList(ctx, cloud, pvcLabelSelector) + if err != nil { + return + } + expansionCompleteCount := 0 + for _, pvcItem := range pvcList.Items { + if pvcExpansionComplete, e := r.expandPVC(ctx, &pvcItem, newSize, logger); e != nil { + err = e + } else if pvcExpansionComplete { + expansionCompleteCount += 1 + } + } + // If all PVCs have been expanded, then we are done + expansionComplete = err == nil && expansionCompleteCount == len(pvcList.Items) + return +} + +func (r *SolrCloudReconciler) expandPVC(ctx context.Context, pvc *corev1.PersistentVolumeClaim, newSize resource.Quantity, logger logr.Logger) (expansionComplete bool, err error) { + // If the current capacity is >= the new size, then there is nothing to do, expansion is complete + if pvc.Status.Capacity.Storage().Cmp(newSize) >= 0 { + // TODO: Eventually use the pvc.Status.AllocatedResources and pvc.Status.AllocatedResourceStatuses to determine the status of PVC Expansion and react to failures + expansionComplete = true + } else if !pvc.Spec.Resources.Requests.Storage().Equal(newSize) { + // Update the pvc if the capacity request is different. + // The newSize might be smaller than the current size, but this is supported as the last size might have been too + // big for the storage quota, so it was lowered. + // As long as the PVCs current capacity is lower than the new size, we are still good to update the PVC. + originalPvc := pvc.DeepCopy() + pvc.Spec.Resources.Requests[corev1.ResourceStorage] = newSize + if err = r.Patch(ctx, pvc, client.StrategicMergeFrom(originalPvc)); err != nil { + logger.Error(err, "Error while expanding PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) + } else { + logger.Info("Expanded PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) + } + } + return +} + // Logic derived from: // - https://book.kubebuilder.io/reference/using-finalizers.html // - https://github.com/pravega/zookeeper-operator/blob/v0.2.9/pkg/controller/zookeepercluster/zookeepercluster_controller.go#L629 @@ -978,16 +1027,15 @@ func (r *SolrCloudReconciler) reconcileStorageFinalizer(ctx context.Context, clo return nil } -func (r *SolrCloudReconciler) getPVCCount(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string) (pvcCount int, err error) { +func (r *SolrCloudReconciler) getPVCCount(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string) (int, error) { pvcList, err := r.getPVCList(ctx, cloud, pvcLabelSelector) if err != nil { return -1, err } - pvcCount = len(pvcList.Items) - return pvcCount, nil + return len(pvcList.Items), nil } -func (r *SolrCloudReconciler) cleanupOrphanPVCs(ctx context.Context, cloud *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, pvcLabelSelector map[string]string, logger logr.Logger) (err error) { +func (r *SolrCloudReconciler) cleanupOrphanPVCs(ctx context.Context, cloud *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, pvcLabelSelector map[string]string, logger logr.Logger) error { // this check should make sure we do not delete the PVCs before the STS has scaled down if cloud.Status.ReadyReplicas == cloud.Status.Replicas { pvcList, err := r.getPVCList(ctx, cloud, pvcLabelSelector) @@ -1003,28 +1051,31 @@ func (r *SolrCloudReconciler) cleanupOrphanPVCs(ctx context.Context, cloud *solr // Don't use the Spec replicas here, because we might be rolling down 1-by-1 and the PVCs for // soon-to-be-deleted pods should not be deleted until the pod is deleted. if util.IsPVCOrphan(pvcItem.Name, *statefulSet.Spec.Replicas) { - r.deletePVC(ctx, pvcItem, logger) + if e := r.deletePVC(ctx, pvcItem, logger); e != nil { + err = e + } } } } + return err } return nil } -func (r *SolrCloudReconciler) getPVCList(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string) (pvList corev1.PersistentVolumeClaimList, err error) { +func (r *SolrCloudReconciler) getPVCList(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string) (corev1.PersistentVolumeClaimList, error) { selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ MatchLabels: pvcLabelSelector, }) - pvclistOps := &client.ListOptions{ + pvcListOps := &client.ListOptions{ Namespace: cloud.Namespace, LabelSelector: selector, } pvcList := &corev1.PersistentVolumeClaimList{} - err = r.Client.List(ctx, pvcList, pvclistOps) + err = r.Client.List(ctx, pvcList, pvcListOps) return *pvcList, err } -func (r *SolrCloudReconciler) cleanUpAllPVCs(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string, logger logr.Logger) (err error) { +func (r *SolrCloudReconciler) cleanUpAllPVCs(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string, logger logr.Logger) error { pvcList, err := r.getPVCList(ctx, cloud, pvcLabelSelector) if err != nil { return err @@ -1032,7 +1083,7 @@ func (r *SolrCloudReconciler) cleanUpAllPVCs(ctx context.Context, cloud *solrv1b for _, pvcItem := range pvcList.Items { r.deletePVC(ctx, pvcItem, logger) } - return nil + return err } func (r *SolrCloudReconciler) deletePVC(ctx context.Context, pvcItem corev1.PersistentVolumeClaim, logger logr.Logger) { diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go index de44d7c1..a7111ff3 100644 --- a/controllers/util/solr_util.go +++ b/controllers/util/solr_util.go @@ -57,6 +57,7 @@ const ( // These are to be saved on a statefulSet update ClusterOpsLockAnnotation = "solr.apache.org/clusterOpsLock" ClusterOpsRetryQueueAnnotation = "solr.apache.org/clusterOpsRetryQueue" + StorageMinimumSizeAnnotation = "solr.apache.org/storageMinimumSize" SolrIsNotStoppedReadinessCondition = "solr.apache.org/isNotStopped" SolrReplicasNotEvictedReadinessCondition = "solr.apache.org/replicasNotEvicted" @@ -200,6 +201,13 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl Spec: pvc.Spec, }, } + if pvc.Spec.Resources.Requests.Storage() != nil { + annotations[StorageMinimumSizeAnnotation] = pvc.Spec.Resources.Requests.Storage().String() + if podAnnotations == nil { + podAnnotations = make(map[string]string, 1) + } + podAnnotations[StorageMinimumSizeAnnotation] = pvc.Spec.Resources.Requests.Storage().String() + } } else { ephemeralVolume := corev1.Volume{ Name: solrDataVolumeName, @@ -680,6 +688,22 @@ func MaintainPreservedStatefulSetFields(expected, found *appsv1.StatefulSet) { } expected.Annotations[ClusterOpsRetryQueueAnnotation] = queue } + if storage, hasStorage := found.Annotations[StorageMinimumSizeAnnotation]; hasStorage { + if expected.Annotations == nil { + expected.Annotations = make(map[string]string, 1) + } + expected.Annotations[StorageMinimumSizeAnnotation] = storage + } + } + if found.Spec.Template.Annotations != nil { + // Note: the Pod template storage annotation is used to start a rolling restart, + // it should always match the StatefulSet's storage annotation + if storage, hasStorage := found.Spec.Template.Annotations[StorageMinimumSizeAnnotation]; hasStorage { + if expected.Spec.Template.Annotations == nil { + expected.Spec.Template.Annotations = make(map[string]string, 1) + } + expected.Spec.Template.Annotations[StorageMinimumSizeAnnotation] = storage + } } // Scaling (i.e. changing) the number of replicas in the SolrCloud statefulSet is handled during the clusterOps From 7f8157782a5b0eb43664bfd21f6bce4c88e8cecd Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 14 Jun 2024 15:19:34 -0500 Subject: [PATCH 02/13] Fix error --- controllers/solrcloud_controller.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index 5e87d166..f05e3512 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -1051,9 +1051,7 @@ func (r *SolrCloudReconciler) cleanupOrphanPVCs(ctx context.Context, cloud *solr // Don't use the Spec replicas here, because we might be rolling down 1-by-1 and the PVCs for // soon-to-be-deleted pods should not be deleted until the pod is deleted. if util.IsPVCOrphan(pvcItem.Name, *statefulSet.Spec.Replicas) { - if e := r.deletePVC(ctx, pvcItem, logger); e != nil { - err = e - } + r.deletePVC(ctx, pvcItem, logger) } } } From 83fc30e7419268d777d110fc9a9173db914a9af2 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 20 Jun 2024 14:08:22 -0500 Subject: [PATCH 03/13] Add PVC permissions --- config/rbac/role.yaml | 2 ++ controllers/solrcloud_controller.go | 2 +- helm/solr-operator/templates/role.yaml | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c4f9b41f..45f6d561 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -44,6 +44,8 @@ rules: - delete - get - list + - patch + - update - watch - apiGroups: - "" diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index f05e3512..8e2069b7 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -73,7 +73,7 @@ func UseZkCRD(useCRD bool) { //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=configmaps/status,verbs=get -//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;delete +//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;update;patch;delete //+kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=zookeeper.pravega.io,resources=zookeeperclusters,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=zookeeper.pravega.io,resources=zookeeperclusters/status,verbs=get diff --git a/helm/solr-operator/templates/role.yaml b/helm/solr-operator/templates/role.yaml index da950e4e..f7a28ac0 100644 --- a/helm/solr-operator/templates/role.yaml +++ b/helm/solr-operator/templates/role.yaml @@ -48,6 +48,8 @@ rules: - delete - get - list + - patch + - update - watch - apiGroups: - "" From 069c684bf00dbd3e2c4fe1518577ef62628c6ca1 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 20 Jun 2024 14:26:05 -0500 Subject: [PATCH 04/13] Add changelog and update docs --- docs/solr-cloud/solr-cloud-crd.md | 5 +++-- docs/upgrade-notes.md | 2 +- helm/solr/Chart.yaml | 9 +++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/solr-cloud/solr-cloud-crd.md b/docs/solr-cloud/solr-cloud-crd.md index c1053cd1..79ba9af9 100644 --- a/docs/solr-cloud/solr-cloud-crd.md +++ b/docs/solr-cloud/solr-cloud-crd.md @@ -61,8 +61,9 @@ These options can be found in `SolrCloud.spec.dataStorage` - **`pvcTemplate`** - The template of the PVC to use for the solr data PVCs. By default the name will be "data". Only the `pvcTemplate.spec` field is required, metadata is optional. - Note: This template cannot be changed unless the SolrCloud is deleted and recreated. - This is a [limitation of StatefulSets and PVCs in Kubernetes](https://github.com/kubernetes/enhancements/issues/661). + Note: Currently, [Kubernetes does not support PVC resizing (expanding) in StatefulSets](https://github.com/kubernetes/enhancements/issues/661). + However, The Solr Operator will manage the PVC expansion for users until this is supported by default in Kubernetes. + Therefore the `pvcTemplate.spec` can have an update to `pvcTemplate.spec.resources.requests`, but all other fields should be considered immutable. - **`ephemeral`** There are two types of ephemeral volumes that can be specified. diff --git a/docs/upgrade-notes.md b/docs/upgrade-notes.md index 448dfb24..32fedc58 100644 --- a/docs/upgrade-notes.md +++ b/docs/upgrade-notes.md @@ -124,7 +124,7 @@ _Note that the Helm chart version does not contain a `v` prefix, which the downl ### v0.8.0 - **The minimum supported Solr version is now 8.11** If you are unable to use a newer version of Solr, please install the `v0.7.1` version of the Solr Operator. - However, it is strongly suggested to upgrade to newer versions of Solr that are actively supported.q + However, it is strongly suggested to upgrade to newer versions of Solr that are actively supported. See the [version compatibility matrix](#solr-versions) for more information. - **Kubernetes support is now limited to 1.22+.** diff --git a/helm/solr/Chart.yaml b/helm/solr/Chart.yaml index f93a1182..03e3bf00 100644 --- a/helm/solr/Chart.yaml +++ b/helm/solr/Chart.yaml @@ -42,15 +42,12 @@ annotations: # Allowed syntax is described at: https://artifacthub.io/docs/topics/annotations/helm/#example artifacthub.io/changes: | - kind: added - description: Addition 1 + description: Allow resizing (expanding) of persistent data PVCs links: - name: Github Issue - url: https://github.com/issue-url - - kind: changed - description: Change 2 - links: + url: https://github.com/apache/solr-operator/issues/709 - name: Github PR - url: https://github.com/pr-url + url: https://github.com/apache/solr-operator/pull/712 artifacthub.io/containsSecurityUpdates: "false" artifacthub.io/recommendations: | - url: https://artifacthub.io/packages/helm/apache-solr/solr-operator From 441931fc6315c12d59fa8c94a52b72984d548301 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Thu, 5 Sep 2024 17:51:55 -0500 Subject: [PATCH 05/13] Add integration test. Cannot test yet, since volume expansion is not supported --- tests/e2e/solrcloud_storage_test.go | 151 ++++++++++++++++++++++++++++ tests/e2e/suite_test.go | 43 +++++++- tests/scripts/manage_e2e_tests.sh | 3 + 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/solrcloud_storage_test.go diff --git a/tests/e2e/solrcloud_storage_test.go b/tests/e2e/solrcloud_storage_test.go new file mode 100644 index 00000000..94289554 --- /dev/null +++ b/tests/e2e/solrcloud_storage_test.go @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "context" + solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" + "github.com/apache/solr-operator/controllers" + "github.com/apache/solr-operator/controllers/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" +) + +var _ = FDescribe("E2E - SolrCloud - Storage", func() { + var ( + solrCloud *solrv1beta1.SolrCloud + + solrCollection1 = "e2e-1" + + solrCollection2 = "e2e-2" + ) + + BeforeEach(func() { + solrCloud = generateBaseSolrCloud(2) + }) + + JustBeforeEach(func(ctx context.Context) { + By("creating the SolrCloud") + Expect(k8sClient.Create(ctx, solrCloud)).To(Succeed()) + + DeferCleanup(func(ctx context.Context) { + cleanupTest(ctx, solrCloud) + }) + + By("Waiting for the SolrCloud to come up healthy") + solrCloud = expectSolrCloudToBeReady(ctx, solrCloud) + + By("creating a first Solr Collection") + createAndQueryCollection(ctx, solrCloud, solrCollection1, 1, 2) + + By("creating a second Solr Collection") + createAndQueryCollection(ctx, solrCloud, solrCollection2, 2, 1) + }) + + FContext("Persistent Data - Expansion", func() { + BeforeEach(func() { + solrCloud.Spec.StorageOptions = solrv1beta1.SolrDataStorageOptions{ + PersistentStorage: &solrv1beta1.SolrPersistentDataStorageOptions{ + PersistentVolumeClaimTemplate: solrv1beta1.PersistentVolumeClaimTemplate{ + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1G"), + }, + }, + }, + }, + }, + } + }) + + FIt("Fully Expands", func(ctx context.Context) { + newStorageSize := resource.MustParse("1500M") + patchedSolrCloud := solrCloud.DeepCopy() + patchedSolrCloud.Spec.StorageOptions.PersistentStorage.PersistentVolumeClaimTemplate.Spec.Resources.Requests[corev1.ResourceStorage] = newStorageSize + By("triggering a rolling restart via pod annotations") + Expect(k8sClient.Patch(ctx, patchedSolrCloud, client.MergeFrom(solrCloud))).To(Succeed(), "Could not add annotation to SolrCloud pod to initiate rolling restart") + + // Wait for new pods to come up, and when they do we should be doing a balanceReplicas clusterOp + expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*5, time.Millisecond*50, func(g Gomega, found *appsv1.StatefulSet) { + clusterOp, err := controllers.GetCurrentClusterOp(found) + g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") + g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a PvcExpansion lock.") + g.Expect(clusterOp.Operation).To(Equal(controllers.PvcExpansionLock), "StatefulSet does not have a PvcExpansion lock after starting managed update.") + }) + + By("waiting for the expansion's rolling restart to begin") + solrCloud = expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*30, time.Millisecond*100, func(g Gomega, found *solrv1beta1.SolrCloud) { + g.Expect(found.Status.UpToDateNodes).To(BeZero(), "Cloud did not get to a state with zero up-to-date replicas when rolling restart began.") + for _, nodeStatus := range found.Status.SolrNodes { + g.Expect(nodeStatus.SpecUpToDate).To(BeFalse(), "Node not starting as out-of-date when rolling restart begins: %s", nodeStatus.Name) + } + }) + + By("checking that all PVCs have been expanded when the restart begins") + internalLabels := map[string]string{ + util.SolrPVCTechnologyLabel: util.SolrCloudPVCTechnology, + util.SolrPVCStorageLabel: util.SolrCloudPVCDataStorage, + util.SolrPVCInstanceLabel: solrCloud.Name, + } + pvcListOps := &client.ListOptions{ + Namespace: solrCloud.Namespace, + LabelSelector: labels.SelectorFromSet(internalLabels), + } + + foundPVCs := &corev1.PersistentVolumeClaimList{} + Expect(k8sClient.List(ctx, foundPVCs, pvcListOps)).To(Succeed(), "Could not fetch PVC list") + Expect(foundPVCs.Items).To(HaveLen(int(*solrCloud.Spec.Replicas)), "Did not find the same number of PVCs as Solr Pods") + for _, pvc := range foundPVCs.Items { + Expect(pvc.Spec.Resources).To(HaveKeyWithValue(corev1.ResourceStorage, newStorageSize), "The PVC %q does not have the new storage size in its resource requests", pvc.Name) + Expect(pvc.Status.Capacity).To(HaveKeyWithValue(corev1.ResourceStorage, newStorageSize), "The PVC %q does not have the new storage size in its status.capacity", pvc.Name) + } + + statefulSet := expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), 1, time.Millisecond, func(g Gomega, found *appsv1.StatefulSet) { + clusterOp, err := controllers.GetCurrentClusterOp(found) + g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") + g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a RollingUpdate lock.") + g.Expect(clusterOp.Operation).To(Equal(controllers.UpdateLock), "StatefulSet does not have a RollingUpdate lock after starting managed update to increase the storage size.") + g.Expect(clusterOp.Metadata).To(Equal(controllers.RollingUpdateMetadata{RequiresReplicaMigration: false}), "StatefulSet should not require replica migration, since PVCs are being used.") + }) + + By("waiting for the rolling restart to complete") + expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*90, time.Millisecond*5, func(g Gomega, cloud *solrv1beta1.SolrCloud) { + g.Expect(cloud.Status.UpToDateNodes).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas up to date") + g.Expect(cloud.Status.ReadyReplicas).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas ready") + }) + + By("waiting for the rolling restart to complete") + expectStatefulSetWithConsistentChecksAndDuration(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*2, func(g Gomega, found *appsv1.StatefulSet) { + clusterOp, err := controllers.GetCurrentClusterOp(found) + g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") + g.Expect(clusterOp).To(BeNil(), "StatefulSet should not have any cluster lock after finishing its rolling update.") + }) + + By("checking that the collections can be queried after the restart") + queryCollection(ctx, solrCloud, solrCollection1, 0) + queryCollection(ctx, solrCloud, solrCollection2, 0) + }) + }) +}) diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index 1c2aec54..d15dde57 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -312,11 +312,26 @@ func writeAllSolrInfoToFiles(ctx context.Context, directory string, namespace st for _, pod := range foundPods.Items { writeAllPodInfoToFiles( ctx, - directory+pod.Name, + directory+pod.Name+".pod", &pod, ) } + listOps = &client.ListOptions{ + Namespace: namespace, + LabelSelector: labelSelector, + } + + foundPVCs := &corev1.PersistentVolumeClaimList{} + Expect(k8sClient.List(ctx, foundPVCs, listOps)).To(Succeed(), "Could not fetch Solr PVCs") + Expect(foundPVCs).ToNot(BeNil(), "No Solr PVCs could be found") + for _, pvc := range foundPVCs.Items { + writeAllPvcInfoToFiles( + directory+pvc.Name+".pvc", + &pvc, + ) + } + foundStatefulSets := &appsv1.StatefulSetList{} Expect(k8sClient.List(ctx, foundStatefulSets, listOps)).To(Succeed(), "Could not fetch Solr statefulSets") Expect(foundStatefulSets).ToNot(BeNil(), "No Solr statefulSet could be found") @@ -388,6 +403,32 @@ func writeAllStatefulSetInfoToFiles(baseFilename string, statefulSet *appsv1.Sta Expect(writeErr).ToNot(HaveOccurred(), "Could not write statefulSet events json to file") } +// writeAllPvcInfoToFiles writes the following each to a separate file with the given base name & directory. +// - PVC Spec/Status +// - PVC Events +func writeAllPvcInfoToFiles(baseFilename string, pvc *corev1.PersistentVolumeClaim) { + // Write PVC to a file + statusFile, err := os.Create(baseFilename + ".status.json") + defer statusFile.Close() + Expect(err).ToNot(HaveOccurred(), "Could not open file to save PVC status: %s", baseFilename+".status.json") + jsonBytes, marshErr := json.MarshalIndent(pvc, "", "\t") + Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize PVC json") + _, writeErr := statusFile.Write(jsonBytes) + Expect(writeErr).ToNot(HaveOccurred(), "Could not write PVC json to file") + + // Write events for PVC to a file + eventsFile, err := os.Create(baseFilename + ".events.json") + defer eventsFile.Close() + Expect(err).ToNot(HaveOccurred(), "Could not open file to save PVC events: %s", baseFilename+".events.yaml") + + eventList, err := rawK8sClient.CoreV1().Events(pvc.Namespace).Search(scheme.Scheme, pvc) + Expect(err).ToNot(HaveOccurred(), "Could not find events for PVC: %s", pvc.Name) + jsonBytes, marshErr = json.MarshalIndent(eventList, "", "\t") + Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize PVC events json") + _, writeErr = eventsFile.Write(jsonBytes) + Expect(writeErr).ToNot(HaveOccurred(), "Could not write PVC events json to file") +} + // writeAllServiceInfoToFiles writes the following each to a separate file with the given base name & directory. // - Service func writeAllServiceInfoToFiles(baseFilename string, service *corev1.Service) { diff --git a/tests/scripts/manage_e2e_tests.sh b/tests/scripts/manage_e2e_tests.sh index c42b92fb..61621324 100755 --- a/tests/scripts/manage_e2e_tests.sh +++ b/tests/scripts/manage_e2e_tests.sh @@ -167,6 +167,9 @@ function start_cluster() { echo "Create test Kubernetes ${KUBERNETES_VERSION} cluster in KinD. This will allow us to test the CRDs, Helm chart and the Docker image." kind create cluster --name "${CLUSTER_NAME}" --image "kindest/node:${KUBERNETES_VERSION}" --config "${SCRIPT_DIR}/e2e-kind-config.yaml" + # TODO: Remove when the following issue is resolved: https://github.com/kubernetes-sigs/kind/issues/3734 + kubectl patch storageclass standard -p '{"allowVolumeExpansion":true}' + setup_cluster } From 75b9c73b1b826887f5dd8dc87dcb392c1b9a2c2c Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 27 Mar 2026 17:17:59 -0700 Subject: [PATCH 06/13] Limit testing until volume expansion is supported locally --- tests/e2e/solrcloud_storage_test.go | 79 ++++++++++++++++------------- tests/e2e/suite_test.go | 30 +++++------ 2 files changed, 57 insertions(+), 52 deletions(-) diff --git a/tests/e2e/solrcloud_storage_test.go b/tests/e2e/solrcloud_storage_test.go index 94289554..bff2dfb8 100644 --- a/tests/e2e/solrcloud_storage_test.go +++ b/tests/e2e/solrcloud_storage_test.go @@ -19,6 +19,8 @@ package e2e import ( "context" + "time" + solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" "github.com/apache/solr-operator/controllers" "github.com/apache/solr-operator/controllers/util" @@ -29,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" - "time" ) var _ = FDescribe("E2E - SolrCloud - Storage", func() { @@ -69,7 +70,7 @@ var _ = FDescribe("E2E - SolrCloud - Storage", func() { PersistentStorage: &solrv1beta1.SolrPersistentDataStorageOptions{ PersistentVolumeClaimTemplate: solrv1beta1.PersistentVolumeClaimTemplate{ Spec: corev1.PersistentVolumeClaimSpec{ - Resources: corev1.ResourceRequirements{ + Resources: corev1.VolumeResourceRequirements{ Requests: map[corev1.ResourceName]resource.Quantity{ corev1.ResourceStorage: resource.MustParse("1G"), }, @@ -95,13 +96,20 @@ var _ = FDescribe("E2E - SolrCloud - Storage", func() { g.Expect(clusterOp.Operation).To(Equal(controllers.PvcExpansionLock), "StatefulSet does not have a PvcExpansion lock after starting managed update.") }) - By("waiting for the expansion's rolling restart to begin") - solrCloud = expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*30, time.Millisecond*100, func(g Gomega, found *solrv1beta1.SolrCloud) { - g.Expect(found.Status.UpToDateNodes).To(BeZero(), "Cloud did not get to a state with zero up-to-date replicas when rolling restart began.") - for _, nodeStatus := range found.Status.SolrNodes { - g.Expect(nodeStatus.SpecUpToDate).To(BeFalse(), "Node not starting as out-of-date when rolling restart begins: %s", nodeStatus.Name) - } - }) + // The rest of the test will not work until Kind supports local volume expansion either via the default local-volume-provisioner or a separate volume provisioner that we can use. For now, just check that the PVCs look good + // - https://github.com/kubernetes-sigs/kind/issues/3734 + // - https://github.com/rancher/local-path-provisioner/issues/190 + + //By("waiting for the expansion's rolling restart to begin") + //solrCloud = expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*30, time.Millisecond*100, func(g Gomega, found *solrv1beta1.SolrCloud) { + // g.Expect(found.Status.UpToDateNodes).To(BeZero(), "Cloud did not get to a state with zero up-to-date replicas when rolling restart began.") + // for _, nodeStatus := range found.Status.SolrNodes { + // g.Expect(nodeStatus.SpecUpToDate).To(BeFalse(), "Node not starting as out-of-date when rolling restart begins: %s", nodeStatus.Name) + // } + //}) + + // TODO: The sleep can be removed when the rest of the test is enabled + time.Sleep(time.Second) By("checking that all PVCs have been expanded when the restart begins") internalLabels := map[string]string{ @@ -118,34 +126,35 @@ var _ = FDescribe("E2E - SolrCloud - Storage", func() { Expect(k8sClient.List(ctx, foundPVCs, pvcListOps)).To(Succeed(), "Could not fetch PVC list") Expect(foundPVCs.Items).To(HaveLen(int(*solrCloud.Spec.Replicas)), "Did not find the same number of PVCs as Solr Pods") for _, pvc := range foundPVCs.Items { - Expect(pvc.Spec.Resources).To(HaveKeyWithValue(corev1.ResourceStorage, newStorageSize), "The PVC %q does not have the new storage size in its resource requests", pvc.Name) - Expect(pvc.Status.Capacity).To(HaveKeyWithValue(corev1.ResourceStorage, newStorageSize), "The PVC %q does not have the new storage size in its status.capacity", pvc.Name) + Expect(pvc.Spec.Resources.Requests).To(HaveKeyWithValue(corev1.ResourceStorage, newStorageSize), "The PVC %q does not have the new storage size in its resource requests", pvc.Name) + // TODO: Re-enable with rest of test + //Expect(pvc.Status.Capacity).To(HaveKeyWithValue(corev1.ResourceStorage, newStorageSize), "The PVC %q does not have the new storage size in its status.capacity", pvc.Name) } - statefulSet := expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), 1, time.Millisecond, func(g Gomega, found *appsv1.StatefulSet) { - clusterOp, err := controllers.GetCurrentClusterOp(found) - g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") - g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a RollingUpdate lock.") - g.Expect(clusterOp.Operation).To(Equal(controllers.UpdateLock), "StatefulSet does not have a RollingUpdate lock after starting managed update to increase the storage size.") - g.Expect(clusterOp.Metadata).To(Equal(controllers.RollingUpdateMetadata{RequiresReplicaMigration: false}), "StatefulSet should not require replica migration, since PVCs are being used.") - }) - - By("waiting for the rolling restart to complete") - expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*90, time.Millisecond*5, func(g Gomega, cloud *solrv1beta1.SolrCloud) { - g.Expect(cloud.Status.UpToDateNodes).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas up to date") - g.Expect(cloud.Status.ReadyReplicas).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas ready") - }) - - By("waiting for the rolling restart to complete") - expectStatefulSetWithConsistentChecksAndDuration(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*2, func(g Gomega, found *appsv1.StatefulSet) { - clusterOp, err := controllers.GetCurrentClusterOp(found) - g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") - g.Expect(clusterOp).To(BeNil(), "StatefulSet should not have any cluster lock after finishing its rolling update.") - }) - - By("checking that the collections can be queried after the restart") - queryCollection(ctx, solrCloud, solrCollection1, 0) - queryCollection(ctx, solrCloud, solrCollection2, 0) + //statefulSet := expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), 1, time.Millisecond, func(g Gomega, found *appsv1.StatefulSet) { + // clusterOp, err := controllers.GetCurrentClusterOp(found) + // g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") + // g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a RollingUpdate lock.") + // g.Expect(clusterOp.Operation).To(Equal(controllers.UpdateLock), "StatefulSet does not have a RollingUpdate lock after starting managed update to increase the storage size.") + // g.Expect(clusterOp.Metadata).To(Equal(controllers.RollingUpdateMetadata{RequiresReplicaMigration: false}), "StatefulSet should not require replica migration, since PVCs are being used.") + //}) + // + //By("waiting for the rolling restart to complete") + //expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*90, time.Millisecond*5, func(g Gomega, cloud *solrv1beta1.SolrCloud) { + // g.Expect(cloud.Status.UpToDateNodes).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas up to date") + // g.Expect(cloud.Status.ReadyReplicas).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas ready") + //}) + // + //By("waiting for the rolling restart to complete") + //expectStatefulSetWithConsistentChecksAndDuration(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*2, func(g Gomega, found *appsv1.StatefulSet) { + // clusterOp, err := controllers.GetCurrentClusterOp(found) + // g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") + // g.Expect(clusterOp).To(BeNil(), "StatefulSet should not have any cluster lock after finishing its rolling update.") + //}) + // + //By("checking that the collections can be queried after the restart") + //queryCollection(ctx, solrCloud, solrCollection1, 0) + //queryCollection(ctx, solrCloud, solrCollection2, 0) }) }) }) diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index 8facb586..1e9a6d23 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -19,10 +19,17 @@ package e2e import ( "bufio" - "bytes" "context" "encoding/json" "fmt" + "io" + "math/rand" + "os" + "path/filepath" + "strings" + "testing" + "time" + solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" "github.com/apache/solr-operator/version" certManagerApi "github.com/cert-manager/cert-manager/pkg/api" @@ -31,7 +38,6 @@ import ( zkApi "github.com/pravega/zookeeper-operator/api/v1beta1" "golang.org/x/text/cases" "golang.org/x/text/language" - "io" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,16 +46,10 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - "math/rand" - "os" - "path/filepath" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "strings" - "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -229,7 +229,7 @@ var _ = JustAfterEach(func(ctx context.Context) { getSolrOperatorPodName(ctx, solrOperatorReleaseNamespace), solrOperatorReleaseNamespace, &startTime, - fmt.Sprintf("%q: %q", "namespace", testNamespace()), + fmt.Sprintf("%q:%q", "namespace", testNamespace()), ) // Always save the logs of the Solr Operator for the test writeAllSolrInfoToFiles( @@ -530,22 +530,18 @@ func writePodLogsToFile(ctx context.Context, filename string, podName string, po Expect(logsErr).ToNot(HaveOccurred(), "Could not open stream to fetch pod logs. namespace: %s, pod: %s", podNamespace, podName) defer podLogs.Close() - var logReader io.Reader - logReader = podLogs - if filterLinesWithString != "" { - filteredWriter := bytes.NewBufferString("") scanner := bufio.NewScanner(podLogs) for scanner.Scan() { line := scanner.Text() if strings.Contains(line, filterLinesWithString) { - io.WriteString(filteredWriter, line) - io.WriteString(filteredWriter, "\n") + io.WriteString(logFile, line) + io.WriteString(logFile, "\n") } } - logReader = filteredWriter + } else { + _, err = io.Copy(logFile, podLogs) } - _, err = io.Copy(logFile, logReader) Expect(err).ToNot(HaveOccurred(), "Could not write podLogs to file: %s", filename) } From 01212471c50d3b6e9a049cf716c573bc99e9ffd6 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 27 Mar 2026 17:27:12 -0700 Subject: [PATCH 07/13] Update controllers/solr_cluster_ops_util.go Co-authored-by: Steffen Moldenhauer <54577793+smoldenhauer-ish@users.noreply.github.com> --- controllers/solr_cluster_ops_util.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/controllers/solr_cluster_ops_util.go b/controllers/solr_cluster_ops_util.go index fa1952a5..2af00f09 100644 --- a/controllers/solr_cluster_ops_util.go +++ b/controllers/solr_cluster_ops_util.go @@ -180,8 +180,7 @@ func determinePvcExpansionClusterOpLockIfNecessary(instance *solrv1beta1.SolrClo return } -// handleManagedCloudScaleUp does the logic of a managed and "locked" cloud scale up operation. -// This will likely take many reconcile loops to complete, as it is moving replicas to the pods that have recently been scaled up. +// handlePvcExpansion handles the logic of a persistent volume claim expansion operation. func handlePvcExpansion(ctx context.Context, r *SolrCloudReconciler, instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, clusterOp *SolrClusterOp, logger logr.Logger) (operationComplete bool, retryLaterDuration time.Duration, err error) { var newSize resource.Quantity newSize, err = resource.ParseQuantity(clusterOp.Metadata) From deb86859dcdf0f83db89eabf04e4344dcdc0430d Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 30 Mar 2026 12:16:09 -0700 Subject: [PATCH 08/13] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- controllers/solrcloud_controller.go | 43 +++++++++++++++++++---------- docs/solr-cloud/solr-cloud-crd.md | 2 +- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index 093f3a90..4945356a 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -564,8 +564,8 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( clusterOp = nil } clusterOp, retryLaterDuration, err = determinePvcExpansionClusterOpLockIfNecessary(instance, statefulSet) - // If the new clusterOperation is an update to a queued clusterOp, just change the operation that is already queued - if queueIdx, opIsQueued := queuedRetryOps[UpdateLock]; clusterOp != nil && opIsQueued { + // If the new clusterOperation is an update to a queued PVC expansion clusterOp, just change the operation that is already queued + if queueIdx, opIsQueued := queuedRetryOps[PvcExpansionLock]; clusterOp != nil && opIsQueued { clusterOpQueue[queueIdx] = *clusterOp clusterOp = nil } @@ -1047,21 +1047,34 @@ func (r *SolrCloudReconciler) expandPVCs(ctx context.Context, cloud *solrv1beta1 } func (r *SolrCloudReconciler) expandPVC(ctx context.Context, pvc *corev1.PersistentVolumeClaim, newSize resource.Quantity, logger logr.Logger) (expansionComplete bool, err error) { - // If the current capacity is >= the new size, then there is nothing to do, expansion is complete - if pvc.Status.Capacity.Storage().Cmp(newSize) >= 0 { + // If the current capacity is >= the new size, then there is nothing to do, expansion is complete. + // Treat missing capacity as zero. + capacityQty, hasCapacity := pvc.Status.Capacity[corev1.ResourceStorage] + if !hasCapacity { + capacityQty = resource.Quantity{} + } + if capacityQty.Cmp(newSize) >= 0 { // TODO: Eventually use the pvc.Status.AllocatedResources and pvc.Status.AllocatedResourceStatuses to determine the status of PVC Expansion and react to failures expansionComplete = true - } else if !pvc.Spec.Resources.Requests.Storage().Equal(newSize) { - // Update the pvc if the capacity request is different. - // The newSize might be smaller than the current size, but this is supported as the last size might have been too - // big for the storage quota, so it was lowered. - // As long as the PVCs current capacity is lower than the new size, we are still good to update the PVC. - originalPvc := pvc.DeepCopy() - pvc.Spec.Resources.Requests[corev1.ResourceStorage] = newSize - if err = r.Patch(ctx, pvc, client.StrategicMergeFrom(originalPvc)); err != nil { - logger.Error(err, "Error while expanding PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) - } else { - logger.Info("Expanded PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) + } else { + // Determine if the current request already matches the desired size. + requestQty, hasRequest := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + sameRequest := hasRequest && requestQty.Equal(newSize) + if !sameRequest { + // Update the pvc if the capacity request is different. + // The newSize might be smaller than the current size, but this is supported as the last size might have been too + // big for the storage quota, so it was lowered. + // As long as the PVCs current capacity is lower than the new size, we are still good to update the PVC. + originalPvc := pvc.DeepCopy() + if pvc.Spec.Resources.Requests == nil { + pvc.Spec.Resources.Requests = corev1.ResourceList{} + } + pvc.Spec.Resources.Requests[corev1.ResourceStorage] = newSize + if err = r.Patch(ctx, pvc, client.StrategicMergeFrom(originalPvc)); err != nil { + logger.Error(err, "Error while expanding PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) + } else { + logger.Info("Expanded PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) + } } } return diff --git a/docs/solr-cloud/solr-cloud-crd.md b/docs/solr-cloud/solr-cloud-crd.md index a1da877f..e2671eeb 100644 --- a/docs/solr-cloud/solr-cloud-crd.md +++ b/docs/solr-cloud/solr-cloud-crd.md @@ -62,7 +62,7 @@ These options can be found in `SolrCloud.spec.dataStorage` Only the `pvcTemplate.spec` field is required, metadata is optional. Note: Currently, [Kubernetes does not support PVC resizing (expanding) in StatefulSets](https://github.com/kubernetes/enhancements/issues/661). - However, The Solr Operator will manage the PVC expansion for users until this is supported by default in Kubernetes. + However, the Solr Operator will manage the PVC expansion for users until this is supported by default in Kubernetes. Therefore the `pvcTemplate.spec` can have an update to `pvcTemplate.spec.resources.requests`, but all other fields should be considered immutable. - **`ephemeral`** From b100ae876b91375717661ee2dfaf58b8b7b62f2c Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 30 Mar 2026 12:20:23 -0700 Subject: [PATCH 09/13] Fix closing of test files on error --- tests/e2e/suite_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index 1e9a6d23..b63d2275 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -403,8 +403,8 @@ func writeSolrClusterStatusInfoToFile(ctx context.Context, baseFilename string, func writeAllStatefulSetInfoToFiles(baseFilename string, statefulSet *appsv1.StatefulSet) { // Write statefulSet to a file statusFile, err := os.Create(baseFilename + ".status.json") - defer statusFile.Close() Expect(err).ToNot(HaveOccurred(), "Could not open file to save statefulSet status: %s", baseFilename+".status.json") + defer statusFile.Close() jsonBytes, marshErr := json.MarshalIndent(statefulSet, "", "\t") Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize statefulSet json") _, writeErr := statusFile.Write(jsonBytes) @@ -412,8 +412,8 @@ func writeAllStatefulSetInfoToFiles(baseFilename string, statefulSet *appsv1.Sta // Write events for statefulSet to a file eventsFile, err := os.Create(baseFilename + ".events.json") - defer eventsFile.Close() Expect(err).ToNot(HaveOccurred(), "Could not open file to save statefulSet events: %s", baseFilename+".events.yaml") + defer eventsFile.Close() eventList, err := rawK8sClient.CoreV1().Events(statefulSet.Namespace).Search(scheme.Scheme, statefulSet) Expect(err).ToNot(HaveOccurred(), "Could not find events for statefulSet: %s", statefulSet.Name) @@ -429,8 +429,8 @@ func writeAllStatefulSetInfoToFiles(baseFilename string, statefulSet *appsv1.Sta func writeAllPvcInfoToFiles(baseFilename string, pvc *corev1.PersistentVolumeClaim) { // Write PVC to a file statusFile, err := os.Create(baseFilename + ".status.json") - defer statusFile.Close() Expect(err).ToNot(HaveOccurred(), "Could not open file to save PVC status: %s", baseFilename+".status.json") + defer statusFile.Close() jsonBytes, marshErr := json.MarshalIndent(pvc, "", "\t") Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize PVC json") _, writeErr := statusFile.Write(jsonBytes) @@ -438,8 +438,8 @@ func writeAllPvcInfoToFiles(baseFilename string, pvc *corev1.PersistentVolumeCla // Write events for PVC to a file eventsFile, err := os.Create(baseFilename + ".events.json") - defer eventsFile.Close() Expect(err).ToNot(HaveOccurred(), "Could not open file to save PVC events: %s", baseFilename+".events.yaml") + defer eventsFile.Close() eventList, err := rawK8sClient.CoreV1().Events(pvc.Namespace).Search(scheme.Scheme, pvc) Expect(err).ToNot(HaveOccurred(), "Could not find events for PVC: %s", pvc.Name) @@ -454,8 +454,8 @@ func writeAllPvcInfoToFiles(baseFilename string, pvc *corev1.PersistentVolumeCla func writeAllServiceInfoToFiles(baseFilename string, service *corev1.Service) { // Write service to a file statusFile, err := os.Create(baseFilename + ".json") - defer statusFile.Close() Expect(err).ToNot(HaveOccurred(), "Could not open file to save service status: %s", baseFilename+".json") + defer statusFile.Close() jsonBytes, marshErr := json.MarshalIndent(service, "", "\t") Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize service json") _, writeErr := statusFile.Write(jsonBytes) @@ -467,8 +467,8 @@ func writeAllServiceInfoToFiles(baseFilename string, service *corev1.Service) { func writeAllSecretInfoToFiles(baseFilename string, secret *corev1.Secret) { // Write service to a file statusFile, err := os.Create(baseFilename + ".json") - defer statusFile.Close() Expect(err).ToNot(HaveOccurred(), "Could not open file to save secret status: %s", baseFilename+".json") + defer statusFile.Close() jsonBytes, marshErr := json.MarshalIndent(secret, "", "\t") Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize secret json") _, writeErr := statusFile.Write(jsonBytes) @@ -482,8 +482,8 @@ func writeAllSecretInfoToFiles(baseFilename string, secret *corev1.Secret) { func writeAllPodInfoToFiles(ctx context.Context, baseFilename string, pod *corev1.Pod) { // Write pod to a file statusFile, err := os.Create(baseFilename + ".status.json") - defer statusFile.Close() Expect(err).ToNot(HaveOccurred(), "Could not open file to save pod status: %s", baseFilename+".status.json") + defer statusFile.Close() jsonBytes, marshErr := json.MarshalIndent(pod, "", "\t") Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize pod json") _, writeErr := statusFile.Write(jsonBytes) @@ -491,8 +491,8 @@ func writeAllPodInfoToFiles(ctx context.Context, baseFilename string, pod *corev // Write events for pod to a file eventsFile, err := os.Create(baseFilename + ".events.json") - defer eventsFile.Close() Expect(err).ToNot(HaveOccurred(), "Could not open file to save pod events: %s", baseFilename+".events.yaml") + defer eventsFile.Close() eventList, err := rawK8sClient.CoreV1().Events(pod.Namespace).Search(scheme.Scheme, pod) Expect(err).ToNot(HaveOccurred(), "Could not find events for pod: %s", pod.Name) @@ -535,8 +535,8 @@ func writePodLogsToFile(ctx context.Context, filename string, podName string, po for scanner.Scan() { line := scanner.Text() if strings.Contains(line, filterLinesWithString) { - io.WriteString(logFile, line) - io.WriteString(logFile, "\n") + _, err = io.WriteString(logFile, line) + _, err = io.WriteString(logFile, "\n") } } } else { From 89a38cb5c8ff334cb0eb8377f636e94844cab9d0 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 30 Mar 2026 17:04:43 -0700 Subject: [PATCH 10/13] Fix cluster op logic --- controllers/solrcloud_controller.go | 20 ++++++++++++-------- tests/scripts/manage_e2e_tests.sh | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index 4945356a..3f054281 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -21,14 +21,15 @@ import ( "context" "crypto/md5" "fmt" - policyv1 "k8s.io/api/policy/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/runtime" "reflect" "sort" "strings" "time" + policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" "github.com/apache/solr-operator/controllers/util" "github.com/go-logr/logr" @@ -563,11 +564,14 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( clusterOpQueue[queueIdx] = *clusterOp clusterOp = nil } - clusterOp, retryLaterDuration, err = determinePvcExpansionClusterOpLockIfNecessary(instance, statefulSet) - // If the new clusterOperation is an update to a queued PVC expansion clusterOp, just change the operation that is already queued - if queueIdx, opIsQueued := queuedRetryOps[PvcExpansionLock]; clusterOp != nil && opIsQueued { - clusterOpQueue[queueIdx] = *clusterOp - clusterOp = nil + + if clusterOp == nil { + clusterOp, retryLaterDuration, err = determinePvcExpansionClusterOpLockIfNecessary(instance, statefulSet) + // If the new clusterOperation is an update to a queued PVC expansion clusterOp, just change the operation that is already queued + if queueIdx, opIsQueued := queuedRetryOps[PvcExpansionLock]; clusterOp != nil && opIsQueued { + clusterOpQueue[queueIdx] = *clusterOp + clusterOp = nil + } } // If a non-managed scale needs to take place, this method will update the StatefulSet without starting diff --git a/tests/scripts/manage_e2e_tests.sh b/tests/scripts/manage_e2e_tests.sh index 2649867c..f1e55af9 100755 --- a/tests/scripts/manage_e2e_tests.sh +++ b/tests/scripts/manage_e2e_tests.sh @@ -96,7 +96,7 @@ export RAW_GINKGO export REUSE_KIND_CLUSTER_IF_EXISTS="${REUSE_KIND_CLUSTER_IF_EXISTS:-true}" # This is used for all start_cluster calls export LEAVE_KIND_CLUSTER_ON_SUCCESS="${LEAVE_KIND_CLUSTER_ON_SUCCESS:-false}" # This is only used when using run_tests or run_with_cluster -export CERT_MANAGER_VERSION=1.12.3 +export CERT_MANAGER_VERSION=1.17.4 export CERT_MANAGER_CSI_DRIVER_VERSION=0.5.0 function add_image_to_kind_repo_if_local() { From 55a9b1b940c67052a594697615535658863925a9 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 5 Jun 2026 15:53:31 -0700 Subject: [PATCH 11/13] Handle PVC expansion across online, offline and unsupported provisioners Complete the PvcExpansion cluster op once the controller-side resize is done (capacity reached, or FileSystemResizePending / NodeResizePending), then hand off to the rolling restart to apply any node-side filesystem resize on remount. This avoids the deadlock where offline provisioners could never update status.capacity without a restart that was itself gated on capacity. Add a StorageClass allowVolumeExpansion pre-flight and emit warning events for unsupported classes, shrink requests, and backend-reported infeasible resizes (best-effort via allocatedResourceStatuses). Wire an EventRecorder into the reconciler and add the storageclasses RBAC permission. Co-Authored-By: Claude Opus 4.8 (1M context) --- config/rbac/role.yaml | 8 ++ controllers/solr_cluster_ops_util.go | 99 +++++++++++----- controllers/solr_pvc_expansion_test.go | 94 ++++++++++++++++ controllers/solrcloud_controller.go | 150 ++++++++++++++++++++----- controllers/suite_test.go | 5 +- helm/solr-operator/templates/role.yaml | 8 ++ main.go | 5 +- 7 files changed, 308 insertions(+), 61 deletions(-) create mode 100644 controllers/solr_pvc_expansion_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8341acb3..53b8d47a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -154,6 +154,14 @@ rules: - get - patch - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch - apiGroups: - zookeeper.pravega.io resources: diff --git a/controllers/solr_cluster_ops_util.go b/controllers/solr_cluster_ops_util.go index 2af00f09..deecd216 100644 --- a/controllers/solr_cluster_ops_util.go +++ b/controllers/solr_cluster_ops_util.go @@ -21,6 +21,10 @@ import ( "context" "encoding/json" "errors" + "net/url" + "strconv" + "time" + solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" "github.com/apache/solr-operator/controllers/util" "github.com/apache/solr-operator/controllers/util/solr_api" @@ -30,10 +34,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" - "net/url" "sigs.k8s.io/controller-runtime/pkg/client" - "strconv" - "time" ) // SolrClusterOp contains metadata for cluster operations performed on SolrClouds. @@ -152,30 +153,54 @@ func retryNextQueuedClusterOpWithQueue(statefulSet *appsv1.StatefulSet, clusterO return hasOp, err } -func determinePvcExpansionClusterOpLockIfNecessary(instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet) (clusterOp *SolrClusterOp, retryLaterDuration time.Duration, err error) { - if instance.Spec.StorageOptions.PersistentStorage != nil && - instance.Spec.StorageOptions.PersistentStorage.PersistentVolumeClaimTemplate.Spec.Resources.Requests.Storage() != nil && - instance.Spec.StorageOptions.PersistentStorage.PersistentVolumeClaimTemplate.Spec.Resources.Requests.Storage().String() != statefulSet.Annotations[util.StorageMinimumSizeAnnotation] { - // First make sure that the new Storage request is greater than what already is set. - // PVCs cannot be shrunk - newSize := instance.Spec.StorageOptions.PersistentStorage.PersistentVolumeClaimTemplate.Spec.Resources.Requests.Storage() - // If there is no old size to update, the StatefulSet can be just set to use the new PVC size without any issue. - // Only do a cluster operation if we are expanding from an existing size to a new size - if oldSizeStr, hasOldSize := statefulSet.Annotations[util.StorageMinimumSizeAnnotation]; hasOldSize { - if oldSize, e := resource.ParseQuantity(oldSizeStr); e != nil { - err = e - // TODO: add an event - } else { - // Only update to the new size if it is bigger, we cannot shrink PVCs - if newSize.Cmp(oldSize) > 0 { - clusterOp = &SolrClusterOp{ - Operation: PvcExpansionLock, - Metadata: newSize.String(), - } - } - // TODO: add an event saying that we cannot shrink PVCs - } +func determinePvcExpansionClusterOpLockIfNecessary(ctx context.Context, r *SolrCloudReconciler, instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, logger logr.Logger) (clusterOp *SolrClusterOp, retryLaterDuration time.Duration, err error) { + if instance.Spec.StorageOptions.PersistentStorage == nil || + instance.Spec.StorageOptions.PersistentStorage.PersistentVolumeClaimTemplate.Spec.Resources.Requests.Storage() == nil { + return + } + newSize := instance.Spec.StorageOptions.PersistentStorage.PersistentVolumeClaimTemplate.Spec.Resources.Requests.Storage() + // If there is no old size to update, the StatefulSet can just be set to use the new PVC size without any issue. + // Only do a cluster operation if we are expanding from an existing size to a new size. + oldSizeStr, hasOldSize := statefulSet.Annotations[util.StorageMinimumSizeAnnotation] + if !hasOldSize || newSize.String() == oldSizeStr { + return + } + oldSize, e := resource.ParseQuantity(oldSizeStr) + if e != nil { + err = e + logger.Error(err, "Could not parse the existing minimum PVC size from the StatefulSet annotation", "annotation", util.StorageMinimumSizeAnnotation, "value", oldSizeStr) + if r.Recorder != nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "PVCExpansionError", + "Could not parse the existing minimum data PVC size %q recorded on the StatefulSet: %v", oldSizeStr, e) } + return + } + // PVCs cannot be shrunk, so only proceed if the new size is strictly bigger than the recorded size. + if newSize.Cmp(oldSize) <= 0 { + logger.Info("Cannot shrink existing data PVCs; ignoring the decreased storage request", "currentSize", oldSize.String(), "requestedSize", newSize.String()) + if r.Recorder != nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "PVCExpansionForbidden", + "Cannot shrink data PersistentVolumeClaims from %s to %s; PersistentVolumeClaims can only be expanded.", oldSize.String(), newSize.String()) + } + return + } + // Pre-flight: make sure the storage class backing the data PVCs allows volume expansion. If it + // explicitly does not, there is no point acquiring a cluster operation lock that can never + // complete; surface it as an event instead. + if allowed, className, scErr := r.storageClassAllowsExpansion(ctx, instance, statefulSet.Spec.Selector.MatchLabels); scErr != nil { + // Could not determine; proceed best-effort and let the PVC patch surface any hard rejection. + logger.Error(scErr, "Could not verify whether the storage class allows volume expansion; proceeding with the expansion attempt") + } else if !allowed { + logger.Info("Storage class does not allow volume expansion; ignoring the increased storage request", "storageClass", className, "currentSize", oldSize.String(), "requestedSize", newSize.String()) + if r.Recorder != nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "PVCExpansionForbidden", + "Storage class %q does not allow volume expansion (allowVolumeExpansion); cannot expand data PersistentVolumeClaims from %s to %s.", className, oldSize.String(), newSize.String()) + } + return + } + clusterOp = &SolrClusterOp{ + Operation: PvcExpansionLock, + Metadata: newSize.String(), } return } @@ -188,19 +213,37 @@ func handlePvcExpansion(ctx context.Context, r *SolrCloudReconciler, instance *s logger.Error(err, "Could not convert PvcExpansion metadata to a resource.Quantity, as it represents the new size of PVCs", "metadata", clusterOp.Metadata) return } - operationComplete, err = r.expandPVCs(ctx, instance, statefulSet.Spec.Selector.MatchLabels, newSize, logger) + var resizeInfeasible bool + operationComplete, resizeInfeasible, err = r.expandPVCs(ctx, instance, statefulSet.Spec.Selector.MatchLabels, newSize, logger) if err == nil && operationComplete { originalStatefulSet := statefulSet.DeepCopy() statefulSet.Annotations[util.StorageMinimumSizeAnnotation] = newSize.String() + if statefulSet.Spec.Template.Annotations == nil { + statefulSet.Spec.Template.Annotations = make(map[string]string, 1) + } statefulSet.Spec.Template.Annotations[util.StorageMinimumSizeAnnotation] = newSize.String() if err = r.Patch(ctx, statefulSet, client.StrategicMergeFrom(originalStatefulSet)); err != nil { logger.Error(err, "Error while patching StatefulSet to set the new minimum PVC size after PVCs the completion of PVC resizing", "newSize", newSize) operationComplete = false + } else { + logger.Info("All PersistentVolumeClaims have been expanded, now issuing a rolling restart", "statefulSet", statefulSet.Name) } // Return and wait for the StatefulSet to be updated which will call the reconcile to start the rolling restart retryLaterDuration = 0 } else if err == nil { - retryLaterDuration = time.Second * 5 + if resizeInfeasible { + // The storage backend has declared the requested size infeasible. There is nothing the + // operator can do until the user lowers the requested size, so surface it as an event and + // back off significantly instead of retrying tightly. + if r.Recorder != nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "PVCExpansionInfeasible", + "The storage backend reported that expanding the data PersistentVolumeClaims to %s is infeasible (e.g. it exceeds backend or quota limits). Reduce the requested storage size to a feasible value to recover.", + newSize.String()) + } + retryLaterDuration = time.Minute + } else { + retryLaterDuration = time.Second * 5 + } } return } diff --git a/controllers/solr_pvc_expansion_test.go b/controllers/solr_pvc_expansion_test.go new file mode 100644 index 00000000..9ac4ca2c --- /dev/null +++ b/controllers/solr_pvc_expansion_test.go @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" +) + +// pvcWithCondition builds a PVC carrying a single resize condition. +func pvcWithCondition(condType corev1.PersistentVolumeClaimConditionType, status corev1.ConditionStatus) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + Status: corev1.PersistentVolumeClaimStatus{ + Conditions: []corev1.PersistentVolumeClaimCondition{{Type: condType, Status: status}}, + }, + } +} + +// pvcWithAllocatedStatus builds a PVC carrying a storage allocatedResourceStatus. +func pvcWithAllocatedStatus(status corev1.ClaimResourceStatus) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + Status: corev1.PersistentVolumeClaimStatus{ + AllocatedResourceStatuses: map[corev1.ResourceName]corev1.ClaimResourceStatus{ + corev1.ResourceStorage: status, + }, + }, + } +} + +// TestPvcControllerExpansionComplete verifies that the controller-side expansion is reported as +// complete for the "offline" provisioner signals (FileSystemResizePending condition or a pending/ +// in-progress node resize status), so that the rolling restart is not gated on status.capacity. +func TestPvcControllerExpansionComplete(t *testing.T) { + cases := []struct { + name string + pvc *corev1.PersistentVolumeClaim + want bool + }{ + {"empty pvc", &corev1.PersistentVolumeClaim{}, false}, + {"filesystem resize pending (offline ready-to-restart)", pvcWithCondition(corev1.PersistentVolumeClaimFileSystemResizePending, corev1.ConditionTrue), true}, + {"filesystem resize pending but condition false", pvcWithCondition(corev1.PersistentVolumeClaimFileSystemResizePending, corev1.ConditionFalse), false}, + {"unrelated resizing condition", pvcWithCondition(corev1.PersistentVolumeClaimResizing, corev1.ConditionTrue), false}, + {"node resize pending status", pvcWithAllocatedStatus(corev1.PersistentVolumeClaimNodeResizePending), true}, + {"node resize in progress status", pvcWithAllocatedStatus(corev1.PersistentVolumeClaimNodeResizeInProgress), true}, + {"controller resize in progress status", pvcWithAllocatedStatus(corev1.PersistentVolumeClaimControllerResizeInProgress), false}, + {"controller resize infeasible status", pvcWithAllocatedStatus(corev1.PersistentVolumeClaimControllerResizeInfeasible), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := pvcControllerExpansionComplete(tc.pvc); got != tc.want { + t.Errorf("pvcControllerExpansionComplete() = %v, want %v", got, tc.want) + } + }) + } +} + +// TestPvcResizeInfeasible verifies that a backend-declared infeasible expansion is detected from the +// allocatedResourceStatuses (best-effort; populated on Kubernetes >= 1.34). +func TestPvcResizeInfeasible(t *testing.T) { + cases := []struct { + name string + pvc *corev1.PersistentVolumeClaim + want bool + }{ + {"empty pvc", &corev1.PersistentVolumeClaim{}, false}, + {"controller resize infeasible", pvcWithAllocatedStatus(corev1.PersistentVolumeClaimControllerResizeInfeasible), true}, + {"node resize infeasible", pvcWithAllocatedStatus(corev1.PersistentVolumeClaimNodeResizeInfeasible), true}, + {"node resize pending is not infeasible", pvcWithAllocatedStatus(corev1.PersistentVolumeClaimNodeResizePending), false}, + {"controller resize in progress is not infeasible", pvcWithAllocatedStatus(corev1.PersistentVolumeClaimControllerResizeInProgress), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := pvcResizeInfeasible(tc.pvc); got != tc.want { + t.Errorf("pvcResizeInfeasible() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go index 3f054281..e94ef7bb 100644 --- a/controllers/solrcloud_controller.go +++ b/controllers/solrcloud_controller.go @@ -37,11 +37,13 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -55,7 +57,8 @@ import ( // SolrCloudReconciler reconciles a SolrCloud object type SolrCloudReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + Recorder record.EventRecorder } var useZkCRD bool @@ -75,6 +78,7 @@ func UseZkCRD(useCRD bool) { //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=configmaps/status,verbs=get //+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;update;patch;delete +//+kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch //+kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=zookeeper.pravega.io,resources=zookeeperclusters,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=zookeeper.pravega.io,resources=zookeeperclusters/status,verbs=get @@ -497,6 +501,9 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( operationComplete, requestInProgress, retryLaterDuration, err = util.BalanceReplicasForCluster(ctx, instance, statefulSet, clusterOp.Metadata, clusterOp.Metadata, logger) case PvcExpansionLock: operationComplete, retryLaterDuration, err = handlePvcExpansion(ctx, r, instance, statefulSet, clusterOp, logger) + // PVC expansion (the controller-side volume resize) can take a long time on some provisioners, + // so it should use the long requeue timeout rather than being preempted after a minute. + shortTimeoutForRequeue = false default: operationFound = false // This shouldn't happen, but we don't want to be stuck if it does. @@ -566,7 +573,7 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } if clusterOp == nil { - clusterOp, retryLaterDuration, err = determinePvcExpansionClusterOpLockIfNecessary(instance, statefulSet) + clusterOp, retryLaterDuration, err = determinePvcExpansionClusterOpLockIfNecessary(ctx, r, instance, statefulSet, logger) // If the new clusterOperation is an update to a queued PVC expansion clusterOp, just change the operation that is already queued if queueIdx, opIsQueued := queuedRetryOps[PvcExpansionLock]; clusterOp != nil && opIsQueued { clusterOpQueue[queueIdx] = *clusterOp @@ -1031,7 +1038,7 @@ func (r *SolrCloudReconciler) reconcileZk(ctx context.Context, logger logr.Logge return nil } -func (r *SolrCloudReconciler) expandPVCs(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string, newSize resource.Quantity, logger logr.Logger) (expansionComplete bool, err error) { +func (r *SolrCloudReconciler) expandPVCs(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string, newSize resource.Quantity, logger logr.Logger) (expansionComplete bool, resizeInfeasible bool, err error) { var pvcList corev1.PersistentVolumeClaimList pvcList, err = r.getPVCList(ctx, cloud, pvcLabelSelector) if err != nil { @@ -1039,51 +1046,136 @@ func (r *SolrCloudReconciler) expandPVCs(ctx context.Context, cloud *solrv1beta1 } expansionCompleteCount := 0 for _, pvcItem := range pvcList.Items { - if pvcExpansionComplete, e := r.expandPVC(ctx, &pvcItem, newSize, logger); e != nil { + if pvcExpansionComplete, pvcInfeasible, e := r.expandPVC(ctx, &pvcItem, newSize, logger); e != nil { err = e - } else if pvcExpansionComplete { - expansionCompleteCount += 1 + } else { + if pvcExpansionComplete { + expansionCompleteCount += 1 + } + if pvcInfeasible { + resizeInfeasible = true + } } } - // If all PVCs have been expanded, then we are done + // If all PVCs have completed their controller-side expansion, then we are done expansionComplete = err == nil && expansionCompleteCount == len(pvcList.Items) return } -func (r *SolrCloudReconciler) expandPVC(ctx context.Context, pvc *corev1.PersistentVolumeClaim, newSize resource.Quantity, logger logr.Logger) (expansionComplete bool, err error) { +// expandPVC requests (and detects the completion of) the controller-side expansion of a single PVC. +// +// "Complete" here means the controller-side volume expansion has finished, so the cluster operation +// can hand off to a rolling restart that will carry out any remaining node-side filesystem resize. +// This intentionally does NOT wait for the filesystem resize itself, because some provisioners only +// resize the filesystem "offline" (when the volume is remounted during the restart). Waiting for +// status.capacity in that case would deadlock: capacity can't update until the pod restarts, but the +// operator wouldn't restart until capacity updated. +func (r *SolrCloudReconciler) expandPVC(ctx context.Context, pvc *corev1.PersistentVolumeClaim, newSize resource.Quantity, logger logr.Logger) (expansionComplete bool, resizeInfeasible bool, err error) { // If the current capacity is >= the new size, then there is nothing to do, expansion is complete. // Treat missing capacity as zero. capacityQty, hasCapacity := pvc.Status.Capacity[corev1.ResourceStorage] if !hasCapacity { capacityQty = resource.Quantity{} } - if capacityQty.Cmp(newSize) >= 0 { - // TODO: Eventually use the pvc.Status.AllocatedResources and pvc.Status.AllocatedResourceStatuses to determine the status of PVC Expansion and react to failures + if capacityQty.Cmp(newSize) >= 0 || pvcControllerExpansionComplete(pvc) { + // Either the volume has already been fully expanded (online resize), or the controller-side + // expansion is done and only a node/filesystem resize remains (offline resize), which the + // subsequent rolling restart will complete on remount. expansionComplete = true - } else { - // Determine if the current request already matches the desired size. - requestQty, hasRequest := pvc.Spec.Resources.Requests[corev1.ResourceStorage] - sameRequest := hasRequest && requestQty.Equal(newSize) - if !sameRequest { - // Update the pvc if the capacity request is different. - // The newSize might be smaller than the current size, but this is supported as the last size might have been too - // big for the storage quota, so it was lowered. - // As long as the PVCs current capacity is lower than the new size, we are still good to update the PVC. - originalPvc := pvc.DeepCopy() - if pvc.Spec.Resources.Requests == nil { - pvc.Spec.Resources.Requests = corev1.ResourceList{} - } - pvc.Spec.Resources.Requests[corev1.ResourceStorage] = newSize - if err = r.Patch(ctx, pvc, client.StrategicMergeFrom(originalPvc)); err != nil { - logger.Error(err, "Error while expanding PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) - } else { - logger.Info("Expanded PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) - } + return + } + // Surface (best-effort) a backend that has declared the requested size infeasible, so it can be + // reported instead of being silently retried forever. allocatedResourceStatuses is populated on + // Kubernetes clusters with the RecoverVolumeExpansionFailure feature (GA in 1.34); on older + // clusters this is simply never true and behavior is unchanged. + resizeInfeasible = pvcResizeInfeasible(pvc) + + // Determine if the current request already matches the desired size. + requestQty, hasRequest := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + sameRequest := hasRequest && requestQty.Equal(newSize) + if !sameRequest { + // Update the pvc if the capacity request is different. + // The newSize might be smaller than the current size, but this is supported as the last size might have been too + // big for the storage quota, so it was lowered. + // As long as the PVCs current capacity is lower than the new size, we are still good to update the PVC. + originalPvc := pvc.DeepCopy() + if pvc.Spec.Resources.Requests == nil { + pvc.Spec.Resources.Requests = corev1.ResourceList{} + } + pvc.Spec.Resources.Requests[corev1.ResourceStorage] = newSize + if err = r.Patch(ctx, pvc, client.StrategicMergeFrom(originalPvc)); err != nil { + logger.Error(err, "Error while expanding PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) + } else { + logger.Info("Expanded PersistentVolumeClaim size", "persistentVolumeClaim", pvc.Name, "size", newSize) } } return } +// pvcControllerExpansionComplete reports whether the controller-side expansion of the PVC has +// finished and only a node-side filesystem resize remains. This is the signal that it is safe (and, +// for offline provisioners, necessary) to proceed to a rolling restart to apply the resize. +// +// It checks the FileSystemResizePending condition (available on all supported Kubernetes versions) +// as the primary signal, and falls back to allocatedResourceStatuses (best-effort, populated on +// clusters with RecoverVolumeExpansionFailure / Kubernetes >= 1.34). +func pvcControllerExpansionComplete(pvc *corev1.PersistentVolumeClaim) bool { + for _, cond := range pvc.Status.Conditions { + if cond.Type == corev1.PersistentVolumeClaimFileSystemResizePending && cond.Status == corev1.ConditionTrue { + return true + } + } + if status, hasStatus := pvc.Status.AllocatedResourceStatuses[corev1.ResourceStorage]; hasStatus { + if status == corev1.PersistentVolumeClaimNodeResizePending || status == corev1.PersistentVolumeClaimNodeResizeInProgress { + return true + } + } + return false +} + +// pvcResizeInfeasible reports (best-effort) whether the storage backend has declared the requested +// expansion infeasible (e.g. the size exceeds backend/quota limits). This relies on +// allocatedResourceStatuses, which is populated on Kubernetes clusters with the +// RecoverVolumeExpansionFailure feature (GA in 1.34); on older clusters it is never true. +func pvcResizeInfeasible(pvc *corev1.PersistentVolumeClaim) bool { + if status, hasStatus := pvc.Status.AllocatedResourceStatuses[corev1.ResourceStorage]; hasStatus { + return status == corev1.PersistentVolumeClaimControllerResizeInfeasible || status == corev1.PersistentVolumeClaimNodeResizeInfeasible + } + return false +} + +// storageClassAllowsExpansion reports whether the storage class backing the SolrCloud's data PVCs +// allows volume expansion. The storage class name is resolved from the actual provisioned PVCs +// (whose StorageClassName is always populated, even when the SolrCloud relies on the cluster +// default). When the class cannot be determined, this returns allowed=true so the expansion is still +// attempted (the PVC patch itself will surface a hard rejection). +func (r *SolrCloudReconciler) storageClassAllowsExpansion(ctx context.Context, cloud *solrv1beta1.SolrCloud, pvcLabelSelector map[string]string) (allowed bool, className string, err error) { + pvcList, err := r.getPVCList(ctx, cloud, pvcLabelSelector) + if err != nil { + return false, "", err + } + for i := range pvcList.Items { + if scn := pvcList.Items[i].Spec.StorageClassName; scn != nil && *scn != "" { + className = *scn + break + } + } + if className == "" { + // Could not determine the storage class; allow the attempt. + return true, "", nil + } + storageClass := &storagev1.StorageClass{} + if err = r.Get(ctx, types.NamespacedName{Name: className}, storageClass); err != nil { + if errors.IsNotFound(err) { + // Could not find the storage class; allow the attempt and let the PVC patch surface any error. + return true, className, nil + } + return false, className, err + } + allowed = storageClass.AllowVolumeExpansion != nil && *storageClass.AllowVolumeExpansion + return allowed, className, nil +} + // Logic derived from: // - https://book.kubebuilder.io/reference/using-finalizers.html // - https://github.com/pravega/zookeeper-operator/blob/v0.2.9/pkg/controller/zookeepercluster/zookeepercluster_controller.go#L629 diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 4d49ef58..7b89ee81 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -106,8 +106,9 @@ var _ = BeforeSuite(func(ctx context.Context) { // Start up Reconcilers By("starting the reconcilers") Expect((&SolrCloudReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + Recorder: k8sManager.GetEventRecorderFor("solrcloud-controller"), }).SetupWithManager(k8sManager)).To(Succeed()) Expect((&SolrPrometheusExporterReconciler{ diff --git a/helm/solr-operator/templates/role.yaml b/helm/solr-operator/templates/role.yaml index aa966cbf..6a267a08 100644 --- a/helm/solr-operator/templates/role.yaml +++ b/helm/solr-operator/templates/role.yaml @@ -158,6 +158,14 @@ rules: - get - patch - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch - apiGroups: - zookeeper.pravega.io resources: diff --git a/main.go b/main.go index c4aee805..d504995b 100644 --- a/main.go +++ b/main.go @@ -199,8 +199,9 @@ func main() { } if err = (&controllers.SolrCloudReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("solrcloud-controller"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SolrCloud") os.Exit(1) From 17f29a651243264d590b6daae51965d30a627f41 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 5 Jun 2026 15:55:00 -0700 Subject: [PATCH 12/13] Run the e2e PVC expansion test against a CSI provisioner Install the rawfile-localpv CSI provisioner (which supports volume expansion) for the e2e cluster, bump the test Kubernetes version to v1.33.7, cert-manager to 1.17.4, and kind to v0.30.0 so the cluster can be created. Enable the full storage expansion assertions: check spec.resources.requests when the restart begins and status.capacity after the rolling restart completes, so the test is valid for both online- and offline-resizing provisioners. Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 2 +- tests/e2e/solrcloud_storage_test.go | 93 ++++++++++++++++------------- tests/scripts/manage_e2e_tests.sh | 11 ++-- 3 files changed, 60 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index c6372250..c9a1044f 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ KUSTOMIZE_VERSION=v4.5.2 CONTROLLER_GEN_VERSION=v0.16.4 GO_LICENSES_VERSION=v1.6.0 GINKGO_VERSION = $(shell cat go.mod | grep 'github.com/onsi/ginkgo' | sed 's/.*\(v.*\)$$/\1/g') -KIND_VERSION=v0.23.0 +KIND_VERSION=v0.30.0 YQ_VERSION=v4.33.3 CONTROLLER_RUNTIME_VERSION = $(shell cat go.mod | grep 'sigs.k8s.io/controller-runtime' | sed 's/.*\(v\(.*\)\.[^.]*\)$$/\2/g') # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. diff --git a/tests/e2e/solrcloud_storage_test.go b/tests/e2e/solrcloud_storage_test.go index bff2dfb8..9c96d054 100644 --- a/tests/e2e/solrcloud_storage_test.go +++ b/tests/e2e/solrcloud_storage_test.go @@ -70,6 +70,7 @@ var _ = FDescribe("E2E - SolrCloud - Storage", func() { PersistentStorage: &solrv1beta1.SolrPersistentDataStorageOptions{ PersistentVolumeClaimTemplate: solrv1beta1.PersistentVolumeClaimTemplate{ Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: new("rawfile-localpv"), Resources: corev1.VolumeResourceRequirements{ Requests: map[corev1.ResourceName]resource.Quantity{ corev1.ResourceStorage: resource.MustParse("1G"), @@ -96,22 +97,15 @@ var _ = FDescribe("E2E - SolrCloud - Storage", func() { g.Expect(clusterOp.Operation).To(Equal(controllers.PvcExpansionLock), "StatefulSet does not have a PvcExpansion lock after starting managed update.") }) - // The rest of the test will not work until Kind supports local volume expansion either via the default local-volume-provisioner or a separate volume provisioner that we can use. For now, just check that the PVCs look good - // - https://github.com/kubernetes-sigs/kind/issues/3734 - // - https://github.com/rancher/local-path-provisioner/issues/190 - - //By("waiting for the expansion's rolling restart to begin") - //solrCloud = expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*30, time.Millisecond*100, func(g Gomega, found *solrv1beta1.SolrCloud) { - // g.Expect(found.Status.UpToDateNodes).To(BeZero(), "Cloud did not get to a state with zero up-to-date replicas when rolling restart began.") - // for _, nodeStatus := range found.Status.SolrNodes { - // g.Expect(nodeStatus.SpecUpToDate).To(BeFalse(), "Node not starting as out-of-date when rolling restart begins: %s", nodeStatus.Name) - // } - //}) - - // TODO: The sleep can be removed when the rest of the test is enabled - time.Sleep(time.Second) + By("waiting for the expansion's rolling restart to begin") + solrCloud = expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*30, time.Millisecond*100, func(g Gomega, found *solrv1beta1.SolrCloud) { + g.Expect(found.Status.UpToDateNodes).To(BeZero(), "Cloud did not get to a state with zero up-to-date replicas when rolling restart began.") + for _, nodeStatus := range found.Status.SolrNodes { + g.Expect(nodeStatus.SpecUpToDate).To(BeFalse(), "Node not starting as out-of-date when rolling restart begins: %s", nodeStatus.Name) + } + }) - By("checking that all PVCs have been expanded when the restart begins") + By("checking that the resize has been requested on all PVCs when the restart begins") internalLabels := map[string]string{ util.SolrPVCTechnologyLabel: util.SolrCloudPVCTechnology, util.SolrPVCStorageLabel: util.SolrCloudPVCDataStorage, @@ -126,35 +120,52 @@ var _ = FDescribe("E2E - SolrCloud - Storage", func() { Expect(k8sClient.List(ctx, foundPVCs, pvcListOps)).To(Succeed(), "Could not fetch PVC list") Expect(foundPVCs.Items).To(HaveLen(int(*solrCloud.Spec.Replicas)), "Did not find the same number of PVCs as Solr Pods") for _, pvc := range foundPVCs.Items { + // The resize request (spec) is always set when the operator hands off to the rolling restart. + // The node-side filesystem resize (status.capacity) may still be pending here, since some + // provisioners only complete it when the volume is remounted during the restart below. Expect(pvc.Spec.Resources.Requests).To(HaveKeyWithValue(corev1.ResourceStorage, newStorageSize), "The PVC %q does not have the new storage size in its resource requests", pvc.Name) - // TODO: Re-enable with rest of test - //Expect(pvc.Status.Capacity).To(HaveKeyWithValue(corev1.ResourceStorage, newStorageSize), "The PVC %q does not have the new storage size in its status.capacity", pvc.Name) } - //statefulSet := expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), 1, time.Millisecond, func(g Gomega, found *appsv1.StatefulSet) { - // clusterOp, err := controllers.GetCurrentClusterOp(found) - // g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") - // g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a RollingUpdate lock.") - // g.Expect(clusterOp.Operation).To(Equal(controllers.UpdateLock), "StatefulSet does not have a RollingUpdate lock after starting managed update to increase the storage size.") - // g.Expect(clusterOp.Metadata).To(Equal(controllers.RollingUpdateMetadata{RequiresReplicaMigration: false}), "StatefulSet should not require replica migration, since PVCs are being used.") - //}) - // - //By("waiting for the rolling restart to complete") - //expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*90, time.Millisecond*5, func(g Gomega, cloud *solrv1beta1.SolrCloud) { - // g.Expect(cloud.Status.UpToDateNodes).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas up to date") - // g.Expect(cloud.Status.ReadyReplicas).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas ready") - //}) - // - //By("waiting for the rolling restart to complete") - //expectStatefulSetWithConsistentChecksAndDuration(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*2, func(g Gomega, found *appsv1.StatefulSet) { - // clusterOp, err := controllers.GetCurrentClusterOp(found) - // g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") - // g.Expect(clusterOp).To(BeNil(), "StatefulSet should not have any cluster lock after finishing its rolling update.") - //}) - // - //By("checking that the collections can be queried after the restart") - //queryCollection(ctx, solrCloud, solrCollection1, 0) - //queryCollection(ctx, solrCloud, solrCollection2, 0) + statefulSet := expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), 1, time.Millisecond, func(g Gomega, found *appsv1.StatefulSet) { + clusterOp, err := controllers.GetCurrentClusterOp(found) + g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") + g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a RollingUpdate lock.") + g.Expect(clusterOp.Operation).To(Equal(controllers.UpdateLock), "StatefulSet does not have a RollingUpdate lock after starting managed update to increase the storage size.") + // The lock metadata is the JSON-encoded RollingUpdateMetadata. PVC-backed clouds do not require replica migration. + g.Expect(clusterOp.Metadata).To(Equal(`{"requiresReplicaMigration":false}`), "StatefulSet should not require replica migration, since PVCs are being used.") + }) + + By("waiting for the rolling restart to complete") + // Use the default (longer) timeout, since a managed rolling restart of multiple pods waits for + // Solr replicas to recover between pod restarts and can take a while on a busy cluster. + expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, cloud *solrv1beta1.SolrCloud) { + g.Expect(cloud.Status.UpToDateNodes).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas up to date") + g.Expect(cloud.Status.ReadyReplicas).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas ready") + }) + + By("waiting for the cluster operation lock to be cleared") + expectStatefulSetWithConsistentChecksAndDuration(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*2, func(g Gomega, found *appsv1.StatefulSet) { + clusterOp, err := controllers.GetCurrentClusterOp(found) + g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud") + g.Expect(clusterOp).To(BeNil(), "StatefulSet should not have any cluster lock after finishing its rolling update.") + }) + + By("checking that all PVCs have been fully expanded (status.capacity) after the restart") + // The node-side filesystem resize completes as the volumes are remounted during the rolling + // restart, so the reported capacity is only guaranteed to reflect the new size once the + // restart has finished. This holds for both online- and offline-resizing provisioners. + Eventually(func(g Gomega) { + updatedPVCs := &corev1.PersistentVolumeClaimList{} + g.Expect(k8sClient.List(ctx, updatedPVCs, pvcListOps)).To(Succeed(), "Could not fetch PVC list") + g.Expect(updatedPVCs.Items).To(HaveLen(int(*solrCloud.Spec.Replicas)), "Did not find the same number of PVCs as Solr Pods") + for _, pvc := range updatedPVCs.Items { + g.Expect(pvc.Status.Capacity).To(HaveKeyWithValue(corev1.ResourceStorage, newStorageSize), "The PVC %q does not have the new storage size in its status.capacity", pvc.Name) + } + }).WithContext(ctx).WithTimeout(time.Second * 90).WithPolling(time.Second).Should(Succeed()) + + By("checking that the collections can be queried after the restart") + queryCollection(ctx, solrCloud, solrCollection1, 0) + queryCollection(ctx, solrCloud, solrCollection2, 0) }) }) }) diff --git a/tests/scripts/manage_e2e_tests.sh b/tests/scripts/manage_e2e_tests.sh index f1e55af9..c9cfd7ff 100755 --- a/tests/scripts/manage_e2e_tests.sh +++ b/tests/scripts/manage_e2e_tests.sh @@ -73,7 +73,7 @@ if [[ -z "${OPERATOR_IMAGE:-}" ]]; then echo "Specify a Docker image for the Solr Operator through -i, or through the OPERATOR_IMAGE env var" >&2 && exit 1 fi if [[ -z "${KUBERNETES_VERSION:-}" ]]; then - KUBERNETES_VERSION="v1.26.6" + KUBERNETES_VERSION="v1.33.7" fi if [[ -z "${SOLR_IMAGE:-}" ]]; then SOLR_IMAGE="${SOLR_VERSION:-9.10.0}" @@ -96,6 +96,7 @@ export RAW_GINKGO export REUSE_KIND_CLUSTER_IF_EXISTS="${REUSE_KIND_CLUSTER_IF_EXISTS:-true}" # This is used for all start_cluster calls export LEAVE_KIND_CLUSTER_ON_SUCCESS="${LEAVE_KIND_CLUSTER_ON_SUCCESS:-false}" # This is only used when using run_tests or run_with_cluster +export RAWFILE_LOCAL_PV_VERSION=0.13.1 export CERT_MANAGER_VERSION=1.17.4 export CERT_MANAGER_CSI_DRIVER_VERSION=0.5.0 @@ -169,9 +170,6 @@ function start_cluster() { echo "Create test Kubernetes ${KUBERNETES_VERSION} cluster in KinD. This will allow us to test the CRDs, Helm chart and the Docker image." kind create cluster --name "${CLUSTER_NAME}" --image "kindest/node:${KUBERNETES_VERSION}" --config "${SCRIPT_DIR}/e2e-kind-config.yaml" - # TODO: Remove when the following issue is resolved: https://github.com/kubernetes-sigs/kind/issues/3734 - kubectl patch storageclass standard -p '{"allowVolumeExpansion":true}' - setup_cluster } @@ -193,6 +191,11 @@ function setup_cluster() { kubectl get configmap coredns -n kube-system -o yaml | sed 's/\(.*\)ttl 30\(.*\)/\1ttl 5\2/' | kubectl replace -n kube-system -f - echo "" + printf "Installing Rawfile LocalPV Provisioner\n" + helm repo add rawfile-localpv https://openebs.github.io/rawfile-localpv --force-update + helm upgrade -i -n openebs --create-namespace rawfile-localpv rawfile-localpv/rawfile-localpv --version "${RAWFILE_LOCAL_PV_VERSION}" --set analytics.enabled=false + echo "" + printf "Installing Cert Manager\n" helm repo add cert-manager https://charts.jetstack.io --force-update helm upgrade -i -n cert-manager --create-namespace cert-manager cert-manager/cert-manager --version "${CERT_MANAGER_VERSION}" --set installCRDs=true From 9ff20c8b3f57e22b489d36db499aa8eff4d481c4 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 5 Jun 2026 15:57:39 -0700 Subject: [PATCH 13/13] Document PVC expansion and update operator chart changelog Note the allowVolumeExpansion / size-only / no-shrink constraints in the SolrCloud CRD docs, and add an operator chart changelog entry for the new PVC and storageclasses RBAC permissions. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/solr-cloud/solr-cloud-crd.md | 4 ++++ helm/solr-operator/Chart.yaml | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/docs/solr-cloud/solr-cloud-crd.md b/docs/solr-cloud/solr-cloud-crd.md index e2671eeb..03051881 100644 --- a/docs/solr-cloud/solr-cloud-crd.md +++ b/docs/solr-cloud/solr-cloud-crd.md @@ -64,6 +64,10 @@ These options can be found in `SolrCloud.spec.dataStorage` Note: Currently, [Kubernetes does not support PVC resizing (expanding) in StatefulSets](https://github.com/kubernetes/enhancements/issues/661). However, the Solr Operator will manage the PVC expansion for users until this is supported by default in Kubernetes. Therefore the `pvcTemplate.spec` can have an update to `pvcTemplate.spec.resources.requests`, but all other fields should be considered immutable. + + The storage size can only be increased (PVCs cannot be shrunk), and the backing [`StorageClass` must allow volume expansion](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#expanding-persistent-volumes-claims) (`allowVolumeExpansion: true`). + When the size is increased, the operator resizes the data PVCs and then performs a rolling restart of the SolrCloud so the new capacity is picked up on each node. + If the storage class does not allow expansion, or the request would shrink the PVCs, the operator emits a warning event on the SolrCloud and leaves the storage unchanged. - **`ephemeral`** There are two types of ephemeral volumes that can be specified. diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml index 81138d09..a214d509 100644 --- a/helm/solr-operator/Chart.yaml +++ b/helm/solr-operator/Chart.yaml @@ -55,6 +55,13 @@ annotations: # Allowed syntax is described at: https://artifacthub.io/docs/topics/annotations/helm/#example # 'kind' accepts values: "added", "changed", "deprecated", "removed", "fixed" and "security" artifacthub.io/changes: | + - kind: added + description: The operator can now resize (expand) persistent data PVCs, which requires new RBAC permissions for persistentvolumeclaims (update/patch) and storageclasses (get/list/watch) + links: + - name: Github Issue + url: https://github.com/apache/solr-operator/issues/709 + - name: Github PR + url: https://github.com/apache/solr-operator/pull/712 - kind: changed description: A container PostStart Hook is no longer used to create the ZooKeeper ChRoot, instead the initContainer will manage this links: