From d33d0c61b6b0b23d749249830e0560fdcdcf0b83 Mon Sep 17 00:00:00 2001 From: ashishch432 Date: Sun, 14 Jun 2026 17:25:32 +0000 Subject: [PATCH 1/4] refactor: use shell-free probe command and remove keeper version probe --- .github/workflows/ci.yaml | 2 +- api/v1alpha1/keepercluster_types.go | 20 - api/v1alpha1/zz_generated.deepcopy.go | 5 - cmd/main.go | 2 +- .../bases/clickhouse.com_keeperclusters.yaml | 635 ------------------ ...khouse-operator.clusterserviceversion.yaml | 11 +- dist/chart-cluster/values.yaml | 7 - .../crd/keeperclusters.clickhouse.com.yaml | 635 ------------------ docs/guides/configuration.mdx | 6 +- docs/reference/api-reference.mdx | 5 - internal/controller/clickhouse/sync.go | 1 - internal/controller/keeper/commands.go | 8 - internal/controller/keeper/controller.go | 16 +- internal/controller/keeper/controller_test.go | 70 +- internal/controller/keeper/sync.go | 46 +- internal/controller/versionprobe.go | 7 +- internal/controller/versionprobe_test.go | 7 +- test/e2e/e2e_suite_test.go | 2 +- test/e2e/keeper_e2e_test.go | 2 +- 19 files changed, 27 insertions(+), 1460 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5438a8c1..f891b75e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -238,7 +238,7 @@ jobs: deploy_target: test-compat-e2e-olm - name: supported-clickhouse-compatibility k8s_image: v1.30.13 - clickhouse_version: "26.3,26.2,26.1,25.8" + clickhouse_version: "26.3,26.3-distroless,26.2,26.1,25.8" deploy_target: test-compat-e2e-manifest - name: operator-upgrade k8s_image: v1.30.13 diff --git a/api/v1alpha1/keepercluster_types.go b/api/v1alpha1/keepercluster_types.go index 9a3bda65..197a76c6 100644 --- a/api/v1alpha1/keepercluster_types.go +++ b/api/v1alpha1/keepercluster_types.go @@ -58,16 +58,6 @@ type KeeperClusterSpec struct { // +optional // +kubebuilder:default:="cluster.local" ClusterDomain string `json:"clusterDomain,omitempty"` - - // UpgradeChannel specifies the release channel for major version upgrade checks. - // When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). - // +optional - // +kubebuilder:validation:Pattern=`^(lts|stable|\d+\.\d+)?$` - UpgradeChannel string `json:"upgradeChannel,omitempty"` - - // VersionProbeTemplate overrides for the version detection Job. - // +optional - VersionProbeTemplate *VersionProbeTemplate `json:"versionProbeTemplate,omitempty"` } // WithDefaults sets default values for KeeperClusterSpec fields. @@ -161,15 +151,6 @@ type KeeperClusterStatus struct { // ObservedGeneration indicates latest generation observed by controller. // +operator-sdk:csv:customresourcedefinitions:type=status ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // Version indicates the version reported by the container image. - // +optional - // +operator-sdk:csv:customresourcedefinitions:type=status - Version string `json:"version,omitempty"` - // VersionProbeRevision is the image hash of the last successful version probe. - // When this matches the current image hash, the cached Version is used directly. - // +optional - // +operator-sdk:csv:customresourcedefinitions:type=status - VersionProbeRevision string `json:"versionProbeRevision,omitempty"` } // KeeperCluster is the Schema for the `keeperclusters` API. @@ -180,7 +161,6 @@ type KeeperClusterStatus struct { // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message" // +kubebuilder:printcolumn:name="ReadyReplicas",type="number",JSONPath=".status.readyReplicas" // +kubebuilder:printcolumn:name="Replicas",type="number",JSONPath=".spec.replicas" -// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".status.version" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +operator-sdk:csv:customresourcedefinitions:resources={{Pod,v1}} // +operator-sdk:csv:customresourcedefinitions:resources={{PersistentVolumeClaim,v1}} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 7dc5dc61..6ffafacc 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -466,11 +466,6 @@ func (in *KeeperClusterSpec) DeepCopyInto(out *KeeperClusterSpec) { (*in).DeepCopyInto(*out) } in.Settings.DeepCopyInto(&out.Settings) - if in.VersionProbeTemplate != nil { - in, out := &in.VersionProbeTemplate, &out.VersionProbeTemplate - *out = new(VersionProbeTemplate) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeeperClusterSpec. diff --git a/cmd/main.go b/cmd/main.go index 33a3a5b3..704ca963 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -201,7 +201,7 @@ func run() error { upgradeChecker = upgrade.NewChecker(updater) } - if err = keeper.SetupWithManager(mgr, zapLogger, upgradeChecker, nil, env.EnablePDB); err != nil { + if err = keeper.SetupWithManager(mgr, zapLogger, nil, env.EnablePDB); err != nil { return fmt.Errorf("unable to setup KeeperCluster controller: %w", err) } diff --git a/config/crd/bases/clickhouse.com_keeperclusters.yaml b/config/crd/bases/clickhouse.com_keeperclusters.yaml index c7157d28..db842ae6 100644 --- a/config/crd/bases/clickhouse.com_keeperclusters.yaml +++ b/config/crd/bases/clickhouse.com_keeperclusters.yaml @@ -30,9 +30,6 @@ spec: - jsonPath: .spec.replicas name: Replicas type: number - - jsonPath: .status.version - name: Version - type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -6156,629 +6153,6 @@ spec: x-kubernetes-map-type: atomic type: object type: object - upgradeChannel: - description: |- - UpgradeChannel specifies the release channel for major version upgrade checks. - When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). - pattern: ^(lts|stable|\d+\.\d+)?$ - type: string - versionProbeTemplate: - description: VersionProbeTemplate overrides for the version detection - Job. - properties: - metadata: - description: Metadata applied to the version probe Job. - properties: - annotations: - additionalProperties: - type: string - description: Annotations are annotations applied to the template - objects. - type: object - labels: - additionalProperties: - type: string - description: Labels are labels applied to the template objects. - type: object - type: object - spec: - description: Specification of the desired behavior of the version - probe Job. - properties: - template: - description: Template describes the pod that will be created - for the version probe Job. - properties: - metadata: - description: Metadata applied to version probe Pods. - properties: - annotations: - additionalProperties: - type: string - description: Annotations are annotations applied to - the template objects. - type: object - labels: - additionalProperties: - type: string - description: Labels are labels applied to the template - objects. - type: object - type: object - spec: - description: Specification of the desired behavior of - the version probe Pod. - properties: - containers: - description: |- - Containers overrides for the version probe Pod. - The name field is optional — the operator fills it with default container. - Additional container with the different name may be specified. - items: - description: |- - VersionProbeContainer defines container-level overrides for the version probe. - Field names and JSON tags match corev1.Container so that SMP merges by name. - properties: - name: - default: version-probe - description: Name of the container. If empty, - the operator sets it to the version probe - container name. - type: string - resources: - description: |- - Resources are the compute resource requirements for the version probe container. - Deep-merged with operator defaults via SMP. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references - one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - securityContext: - description: |- - SecurityContext defines the security options for the version probe container. - Deep-merged with operator defaults via SMP. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent - POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent - POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level - label that applies to the container. - type: string - role: - description: Role is a SELinux role - label that applies to the container. - type: string - type: - description: Type is a SELinux type - label that applies to the container. - type: string - user: - description: User is a SELinux user - label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName - is the name of the GMSA credential - spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeSelector: - additionalProperties: - type: string - description: NodeSelector constrains the version probe - Pod to nodes with matching labels. - type: object - x-kubernetes-map-type: atomic - securityContext: - description: SecurityContext holds pod-level security - attributes for the version probe Pod. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter - to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the - name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - tolerations: - description: Tolerations for the version probe Pod. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - ttlSecondsAfterFinished: - description: TTLSecondsAfterFinished limits the lifetime of - a completed Job. - format: int32 - type: integer - type: object - type: object type: object status: description: KeeperClusterStatus defines the observed state of KeeperCluster. @@ -6868,15 +6242,6 @@ spec: description: UpdateRevision indicates latest requested KeeperCluster spec revision. type: string - version: - description: Version indicates the version reported by the container - image. - type: string - versionProbeRevision: - description: |- - VersionProbeRevision is the image hash of the last successful version probe. - When this matches the current image hash, the cached Version is used directly. - type: string type: object type: object served: true diff --git a/config/manifests/bases/clickhouse-operator.clusterserviceversion.yaml b/config/manifests/bases/clickhouse-operator.clusterserviceversion.yaml index d015c7ff..75c9d63d 100644 --- a/config/manifests/bases/clickhouse-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/clickhouse-operator.clusterserviceversion.yaml @@ -87,8 +87,7 @@ spec: displayName: Version Probe Revision path: versionProbeRevision version: v1alpha1 - - description: KeeperCluster is the Schema for the `keeperclusters` API. - displayName: Keeper Cluster + - displayName: Keeper Cluster kind: KeeperCluster name: keeperclusters.clickhouse.com resources: @@ -141,14 +140,6 @@ spec: revision. displayName: Update Revision path: updateRevision - - description: Version indicates the version reported by the container image. - displayName: Version - path: version - - description: |- - VersionProbeRevision is the image hash of the last successful version probe. - When this matches the current image hash, the cached Version is used directly. - displayName: Version Probe Revision - path: versionProbeRevision version: v1alpha1 description: | The ClickHouse Operator is a Kubernetes operator that automates the deployment, configuration, and management of ClickHouse clusters and ClickHouse Keeper clusters on Kubernetes. diff --git a/dist/chart-cluster/values.yaml b/dist/chart-cluster/values.yaml index e926f9dd..f6b640e5 100644 --- a/dist/chart-cluster/values.yaml +++ b/dist/chart-cluster/values.yaml @@ -151,10 +151,3 @@ keeper: # Configuration parameters for ClickHouse Keeper server. # settings: {} - # UpgradeChannel specifies the release channel for major version upgrade checks. - # When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). - # upgradeChannel: "" - - # VersionProbeTemplate overrides for the version detection Job. - # versionProbeTemplate: {} - diff --git a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml index a7b3d539..ce81a333 100644 --- a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml +++ b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml @@ -33,9 +33,6 @@ spec: - jsonPath: .spec.replicas name: Replicas type: number - - jsonPath: .status.version - name: Version - type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -6159,629 +6156,6 @@ spec: x-kubernetes-map-type: atomic type: object type: object - upgradeChannel: - description: |- - UpgradeChannel specifies the release channel for major version upgrade checks. - When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). - pattern: ^(lts|stable|\d+\.\d+)?$ - type: string - versionProbeTemplate: - description: VersionProbeTemplate overrides for the version detection - Job. - properties: - metadata: - description: Metadata applied to the version probe Job. - properties: - annotations: - additionalProperties: - type: string - description: Annotations are annotations applied to the template - objects. - type: object - labels: - additionalProperties: - type: string - description: Labels are labels applied to the template objects. - type: object - type: object - spec: - description: Specification of the desired behavior of the version - probe Job. - properties: - template: - description: Template describes the pod that will be created - for the version probe Job. - properties: - metadata: - description: Metadata applied to version probe Pods. - properties: - annotations: - additionalProperties: - type: string - description: Annotations are annotations applied to - the template objects. - type: object - labels: - additionalProperties: - type: string - description: Labels are labels applied to the template - objects. - type: object - type: object - spec: - description: Specification of the desired behavior of - the version probe Pod. - properties: - containers: - description: |- - Containers overrides for the version probe Pod. - The name field is optional — the operator fills it with default container. - Additional container with the different name may be specified. - items: - description: |- - VersionProbeContainer defines container-level overrides for the version probe. - Field names and JSON tags match corev1.Container so that SMP merges by name. - properties: - name: - default: version-probe - description: Name of the container. If empty, - the operator sets it to the version probe - container name. - type: string - resources: - description: |- - Resources are the compute resource requirements for the version probe container. - Deep-merged with operator defaults via SMP. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references - one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - securityContext: - description: |- - SecurityContext defines the security options for the version probe container. - Deep-merged with operator defaults via SMP. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent - POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent - POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level - label that applies to the container. - type: string - role: - description: Role is a SELinux role - label that applies to the container. - type: string - type: - description: Type is a SELinux type - label that applies to the container. - type: string - user: - description: User is a SELinux user - label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName - is the name of the GMSA credential - spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeSelector: - additionalProperties: - type: string - description: NodeSelector constrains the version probe - Pod to nodes with matching labels. - type: object - x-kubernetes-map-type: atomic - securityContext: - description: SecurityContext holds pod-level security - attributes for the version probe Pod. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter - to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the - name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - tolerations: - description: Tolerations for the version probe Pod. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - ttlSecondsAfterFinished: - description: TTLSecondsAfterFinished limits the lifetime of - a completed Job. - format: int32 - type: integer - type: object - type: object type: object status: description: KeeperClusterStatus defines the observed state of KeeperCluster. @@ -6871,15 +6245,6 @@ spec: description: UpdateRevision indicates latest requested KeeperCluster spec revision. type: string - version: - description: Version indicates the version reported by the container - image. - type: string - versionProbeRevision: - description: |- - VersionProbeRevision is the image hash of the last successful version probe. - When this matches the current image hash, the cached Version is used directly. - type: string type: object type: object served: true diff --git a/docs/guides/configuration.mdx b/docs/guides/configuration.mdx index 014b25f7..c2b63f47 100644 --- a/docs/guides/configuration.mdx +++ b/docs/guides/configuration.mdx @@ -557,14 +557,14 @@ spec.additionalPorts[0].name: "http" is reserved by the operator ## Version probe and upgrade channel {#version-probe-and-upgrade-channel} -The operator does two independent things with cluster versions: +The operator does two independent things with ClickHouse cluster versions: -1. **Version probe** — a Kubernetes `Job` that runs the container image once to detect the running ClickHouse / Keeper version. The detected version is recorded in `.status.version` and used by other reconciliation steps (e.g. the `External Secret` named-collections key is only required from ClickHouse `25.12`). +1. **Version probe** — a Kubernetes `Job` that runs the ClickHouse container image once to detect the running ClickHouse version. The detected version is recorded in `.status.version` and used by other reconciliation steps (e.g. the `External Secret` named-collections key is only required from ClickHouse `25.12`). 2. **Upgrade channel** — a periodic check against the public ClickHouse release feed (`https://clickhouse.com/data/version_date.tsv`). The operator reports whether a newer version is available via the `VersionUpgraded` status condition. It never upgrades the cluster on its own — the user is in control of the image tag. ### Choosing a release channel {#upgrade-channel-choosing} -`spec.upgradeChannel` selects which set of upstream releases the operator compares against. Same field exists on both `ClickHouseCluster` and `KeeperCluster`. +`ClickHouseCluster` `spec.upgradeChannel` selects which set of upstream releases the operator compares against. ```yaml spec: diff --git a/docs/reference/api-reference.mdx b/docs/reference/api-reference.mdx index 45a6bc55..4419a5bd 100644 --- a/docs/reference/api-reference.mdx +++ b/docs/reference/api-reference.mdx @@ -273,8 +273,6 @@ KeeperClusterSpec defines the desired state of KeeperCluster. | `podDisruptionBudget` | [PodDisruptionBudgetSpec](#poddisruptionbudgetspec) | PodDisruptionBudget configures the PDB created for the Keeper cluster.
When unset, the operator defaults to maxUnavailable=replicas/2
(preserving quorum for a 2F+1 cluster); single-replica clusters use maxUnavailable=1. | false | | | `settings` | [KeeperSettings](#keepersettings) | Configuration parameters for ClickHouse Keeper server. | false | | | `clusterDomain` | string | ClusterDomain is the Kubernetes cluster domain suffix used for DNS resolution. | false | cluster.local | -| `upgradeChannel` | string | UpgradeChannel specifies the release channel for major version upgrade checks.
When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). | false | | -| `versionProbeTemplate` | [VersionProbeTemplate](#versionprobetemplate) | VersionProbeTemplate overrides for the version detection Job. | false | | Appears in: - [KeeperCluster](#keepercluster) @@ -292,8 +290,6 @@ KeeperClusterStatus defines the observed state of KeeperCluster. | `currentRevision` | string | CurrentRevision indicates latest applied KeeperCluster spec revision. | true | | | `updateRevision` | string | UpdateRevision indicates latest requested KeeperCluster spec revision. | true | | | `observedGeneration` | integer | ObservedGeneration indicates latest generation observed by controller. | true | | -| `version` | string | Version indicates the version reported by the container image. | false | | -| `versionProbeRevision` | string | VersionProbeRevision is the image hash of the last successful version probe.
When this matches the current image hash, the cached Version is used directly. | false | | Appears in: - [KeeperCluster](#keepercluster) @@ -474,4 +470,3 @@ The structure mirrors batchv1.JobTemplateSpec, exposing only supported fields. Appears in: - [ClickHouseClusterSpec](#clickhouseclusterspec) -- [KeeperClusterSpec](#keeperclusterspec) diff --git a/internal/controller/clickhouse/sync.go b/internal/controller/clickhouse/sync.go index 10ec9a3a..50378357 100644 --- a/internal/controller/clickhouse/sync.go +++ b/internal/controller/clickhouse/sync.go @@ -401,7 +401,6 @@ func (r *clickhouseReconciler) reconcileExternalSecret(ctx context.Context, log func (r *clickhouseReconciler) reconcileVersionProbe(ctx context.Context, log ctrlutil.Logger) (chctrl.StepResult, error) { probeResult, err := r.VersionProbe(ctx, log, chctrl.VersionProbeConfig{ - Binary: "clickhouse-server", Labels: r.Cluster.Spec.Labels, Annotations: r.Cluster.Spec.Annotations, PodTemplate: r.Cluster.Spec.PodTemplate, diff --git a/internal/controller/keeper/commands.go b/internal/controller/keeper/commands.go index 680096d1..61ceead6 100644 --- a/internal/controller/keeper/commands.go +++ b/internal/controller/keeper/commands.go @@ -29,7 +29,6 @@ var ( type serverStatus struct { ServerState string Followers int - Version string } func getConnection(ctx context.Context, dialer controllerutil.DialContextFunc, hostname string, tlsRequired bool) (net.Conn, error) { @@ -104,13 +103,6 @@ func queryKeeper(ctx context.Context, log controllerutil.Logger, conn net.Conn) ServerState: statMap["zk_server_state"], } - version, err := controllerutil.ParseVersion(statMap["zk_version"]) - if err != nil { - log.Warn("failed to parse keeper version", "raw", statMap["zk_version"], "error", err) - } else { - result.Version = version - } - if result.ServerState == "" { return serverStatus{}, fmt.Errorf("response missing required field 'Mode': %q", string(data)) } diff --git a/internal/controller/keeper/controller.go b/internal/controller/keeper/controller.go index ce1f3850..60bef4b7 100644 --- a/internal/controller/keeper/controller.go +++ b/internal/controller/keeper/controller.go @@ -5,7 +5,6 @@ import ( "fmt" appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -20,7 +19,6 @@ import ( v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1" chctrl "github.com/ClickHouse/clickhouse-operator/internal/controller" "github.com/ClickHouse/clickhouse-operator/internal/controllerutil" - "github.com/ClickHouse/clickhouse-operator/internal/upgrade" webhookv1 "github.com/ClickHouse/clickhouse-operator/internal/webhook/v1alpha1" ) @@ -32,7 +30,6 @@ type ClusterController struct { Recorder events.EventRecorder Logger controllerutil.Logger Webhook webhookv1.KeeperClusterWebhook - Checker *upgrade.Checker Dialer controllerutil.DialContextFunc EnablePDB bool } @@ -47,7 +44,6 @@ type ClusterController struct { // +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get // +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;delete // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch -// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -101,7 +97,6 @@ func (cc *ClusterController) Reconcile(ctx context.Context, req ctrl.Request) (c ResourceManager: chctrl.NewResourceManager(cc, cluster), Dialer: cc.Dialer, - Checker: cc.Checker, EnablePDB: cc.EnablePDB, Cluster: cluster, @@ -126,18 +121,13 @@ func (cc *ClusterController) GetRecorder() events.EventRecorder { return cc.Recorder } -// GetVersionChecker returns the version upgrade Checker. -func (cc *ClusterController) GetVersionChecker() *upgrade.Checker { - return cc.Checker -} - // GetDialer returns the custom dialer, or nil. func (cc *ClusterController) GetDialer() controllerutil.DialContextFunc { return cc.Dialer } // SetupWithManager sets up the controller with the Manager. -func SetupWithManager(mgr ctrl.Manager, log controllerutil.Logger, checker *upgrade.Checker, dialer controllerutil.DialContextFunc, enablePDB bool) error { +func SetupWithManager(mgr ctrl.Manager, log controllerutil.Logger, dialer controllerutil.DialContextFunc, enablePDB bool) error { namedLogger := log.Named("keeper") keeperController := &ClusterController{ @@ -146,7 +136,6 @@ func SetupWithManager(mgr ctrl.Manager, log controllerutil.Logger, checker *upgr Recorder: mgr.GetEventRecorder("keeper-controller"), Logger: namedLogger, Webhook: webhookv1.KeeperClusterWebhook{Log: namedLogger}, - Checker: checker, Dialer: dialer, EnablePDB: enablePDB, } @@ -156,8 +145,7 @@ func SetupWithManager(mgr ctrl.Manager, log controllerutil.Logger, checker *upgr Owns(&appsv1.StatefulSet{}). Owns(&corev1.ConfigMap{}). Owns(&corev1.Service{}). - Owns(&corev1.Pod{}). - Owns(&batchv1.Job{}) + Owns(&corev1.Pod{}) if enablePDB { controllerBuilder = controllerBuilder.Owns(&policyv1.PodDisruptionBudget{}) diff --git a/internal/controller/keeper/controller_test.go b/internal/controller/keeper/controller_test.go index 58dfe334..68064dd2 100644 --- a/internal/controller/keeper/controller_test.go +++ b/internal/controller/keeper/controller_test.go @@ -9,7 +9,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,7 +17,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1" "github.com/ClickHouse/clickhouse-operator/internal/controller/testutil" @@ -41,7 +39,6 @@ var _ = When("reconciling standalone KeeperCluster resource", Ordered, func() { pdbs policyv1.PodDisruptionBudgetList configs corev1.ConfigMapList statefulsets appsv1.StatefulSetList - jobs batchv1.JobList cr = &v1.KeeperCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "standalone", @@ -115,75 +112,12 @@ var _ = When("reconciling standalone KeeperCluster resource", Ordered, func() { Expect(suite.Client.List(ctx, &statefulsets, listOpts)).To(Succeed()) Expect(statefulsets.Items).To(HaveLen(1)) - Expect(suite.Client.List(ctx, &jobs, listOpts)).To(Succeed()) - Expect(jobs.Items).To(HaveLen(1)) - Expect(jobs.Items[0].Labels[controllerutil.LabelRoleKey]).To(Equal(controllerutil.LabelVersionProbe)) - testutil.AssertEvents(recorder.Events, map[string]int{ "ReplicaCreated": 1, "ClusterNotReady": 1, }) }) - It("should propagate version probe overrides to the job", func(ctx context.Context) { - By("updating the CR with version probe overrides") - - updatedCR := cr.DeepCopy() - Expect(suite.Client.Get(ctx, cr.NamespacedName(), updatedCR)).To(Succeed()) - updatedCR.Spec.VersionProbeTemplate = &v1.VersionProbeTemplate{ - Spec: v1.VersionProbeJobSpec{ - Template: v1.VersionProbePodTemplate{ - Metadata: v1.TemplateMeta{ - Annotations: map[string]string{ - "sidecar.istio.io/inject": "false", - }, - Labels: map[string]string{ - "probe-label": "probe-value", - }, - }, - }, - }, - } - updatedCR.Spec.PodTemplate.Tolerations = []corev1.Toleration{ - {Key: "workload", Operator: corev1.TolerationOpEqual, Value: "system", Effect: corev1.TaintEffectNoSchedule}, - } - Expect(suite.Client.Update(ctx, updatedCR)).To(Succeed()) - - // Delete old job so new one is created with overrides. - for _, j := range jobs.Items { - Expect(suite.Client.Delete(ctx, &j, client.PropagationPolicy(metav1.DeletePropagationBackground))).To(Succeed()) - } - - _, err := controller.Reconcile(ctx, ctrl.Request{NamespacedName: cr.NamespacedName()}) - Expect(err).NotTo(HaveOccurred()) - Expect(suite.Client.Get(ctx, cr.NamespacedName(), updatedCR)).To(Succeed()) - - listOpts := controllerutil.AppRequirements(cr.Namespace, cr.SpecificName()) - Expect(suite.Client.List(ctx, &jobs, listOpts)).To(Succeed()) - Expect(jobs.Items).To(HaveLen(1)) - - By("verifying annotations on Pod template only") - Expect(jobs.Items[0].Spec.Template.Annotations).To(HaveKeyWithValue("sidecar.istio.io/inject", "false")) - - By("verifying probe-specific labels on Pod template only") - Expect(jobs.Items[0].Spec.Template.Labels).To(HaveKeyWithValue("probe-label", "probe-value")) - - By("verifying operator-reserved labels are preserved") - Expect(jobs.Items[0].Labels[controllerutil.LabelRoleKey]).To(Equal(controllerutil.LabelVersionProbe)) - Expect(jobs.Items[0].Labels[controllerutil.LabelAppKey]).To(Equal(cr.SpecificName())) - - By("verifying scheduling fields inherited from PodTemplate") - Expect(jobs.Items[0].Spec.Template.Spec.Tolerations).To(ContainElement(corev1.Toleration{ - Key: "workload", Operator: corev1.TolerationOpEqual, Value: "system", Effect: corev1.TaintEffectNoSchedule, - })) - - testutil.AssertEvents(recorder.Events, map[string]int{ - "HorizontalScaleBlocked": 1, - }) - - cr = updatedCR.DeepCopy() - }) - It("should propagate meta attributes for every resource", func() { expectedOwnerRef := metav1.OwnerReference{ Kind: "KeeperCluster", @@ -266,6 +200,10 @@ var _ = When("reconciling standalone KeeperCluster resource", Ordered, func() { Expect(updatedCR.Status.UpdateRevision).NotTo(Equal(updatedCR.Status.CurrentRevision)) Expect(updatedCR.Status.ConfigurationRevision).NotTo(Equal(cr.Status.ConfigurationRevision)) Expect(updatedCR.Status.StatefulSetRevision).NotTo(Equal(cr.Status.StatefulSetRevision)) + + testutil.AssertEvents(recorder.Events, map[string]int{ + "HorizontalScaleBlocked": 1, + }) }) It("should add extra config in configmap", func(ctx context.Context) { diff --git a/internal/controller/keeper/sync.go b/internal/controller/keeper/sync.go index 151ac686..c8385db6 100644 --- a/internal/controller/keeper/sync.go +++ b/internal/controller/keeper/sync.go @@ -21,7 +21,6 @@ import ( v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1" chctrl "github.com/ClickHouse/clickhouse-operator/internal/controller" ctrlutil "github.com/ClickHouse/clickhouse-operator/internal/controllerutil" - "github.com/ClickHouse/clickhouse-operator/internal/upgrade" ) type replicaState struct { @@ -80,14 +79,12 @@ type keeperReconciler struct { chctrl.ResourceManager Dialer ctrlutil.DialContextFunc - Checker *upgrade.Checker EnablePDB bool Cluster *v1.KeeperCluster ReplicaState map[v1.KeeperReplicaID]replicaState - versionProbe chctrl.VersionProbeResult - revs chctrl.RevisionState + revs chctrl.RevisionState // Computed by reconcileActiveReplicaStatus HorizontalScaleAllowed bool } @@ -95,14 +92,15 @@ type keeperReconciler struct { func (r *keeperReconciler) sync(ctx context.Context, log ctrlutil.Logger) (ctrl.Result, error) { log.Info("Enter Keeper Reconcile", "spec", r.Cluster.Spec, "status", r.Cluster.Status) + meta.RemoveStatusCondition(&r.Cluster.Status.Conditions, v1.ConditionTypeVersionInSync) + meta.RemoveStatusCondition(&r.Cluster.Status.Conditions, v1.ConditionTypeVersionUpgraded) + r.SetUnknownConditions(v1.ConditionReasonStepFailed, "Reconcile stopped before condition evaluation", []v1.ConditionType{ v1.ConditionTypeReplicaStartupSucceeded, v1.ConditionTypeHealthy, v1.ConditionTypeClusterSizeAligned, v1.ConditionTypeConfigurationInSync, - v1.ConditionTypeVersionInSync, - v1.ConditionTypeVersionUpgraded, v1.ConditionTypeReady, v1.KeeperConditionTypeScaleAllowed, }) @@ -155,7 +153,7 @@ func (r *keeperReconciler) sync(ctx context.Context, log ctrlutil.Logger) (ctrl. return result, nil } -func (r *keeperReconciler) reconcileClusterRevisions(ctx context.Context, log ctrlutil.Logger) (chctrl.StepResult, error) { +func (r *keeperReconciler) reconcileClusterRevisions(_ context.Context, log ctrlutil.Logger) (chctrl.StepResult, error) { if r.Cluster.Status.ObservedGeneration != r.Cluster.Generation { r.Cluster.Status.ObservedGeneration = r.Cluster.Generation log.Debug(fmt.Sprintf("observed new CR generation %d", r.Cluster.Generation)) @@ -203,33 +201,6 @@ func (r *keeperReconciler) reconcileClusterRevisions(ctx context.Context, log ct } } - probeResult, err := r.VersionProbe(ctx, log, chctrl.VersionProbeConfig{ - Binary: "clickhouse-keeper", - Labels: r.Cluster.Spec.Labels, - Annotations: r.Cluster.Spec.Annotations, - PodTemplate: r.Cluster.Spec.PodTemplate, - ContainerTemplate: r.Cluster.Spec.ContainerTemplate, - VersionProbe: r.Cluster.Spec.VersionProbeTemplate, - CachedVersion: r.Cluster.Status.Version, - CachedRevision: r.Cluster.Status.VersionProbeRevision, - }) - if err != nil { - return chctrl.StepResult{}, fmt.Errorf("run version probe: %w", err) - } - - r.versionProbe = probeResult - if probeResult.Completed() { - r.Cluster.Status.Version = probeResult.Version - r.Cluster.Status.VersionProbeRevision = probeResult.Revision - } - - if r.Checker != nil { - cond, event := chctrl.GetUpgradeCondition(*r.Checker, r.versionProbe, r.Cluster.Spec.UpgradeChannel) - r.SetCondition(cond, event...) - } else { - meta.RemoveStatusCondition(r.Cluster.GetStatus().GetConditions(), v1.ConditionTypeVersionUpgraded) - } - return chctrl.StepContinue(), nil } @@ -594,7 +565,6 @@ func (r *keeperReconciler) evaluateReplicaConditions() { var errorIDs, notReadyIDs, notUpdatedIDs []string replicasByMode := map[string][]v1.KeeperReplicaID{} - replicaVersions := map[string]string{} r.Cluster.Status.ReadyReplicas = 0 for id, replica := range r.ReplicaState { @@ -614,17 +584,11 @@ func (r *keeperReconciler) evaluateReplicaConditions() { if replica.HasDiff(r.revs) || !replica.Updated() { notUpdatedIDs = append(notUpdatedIDs, idStr) } - - replicaVersions[idStr] = replica.Status.Version } r.SetCondition(chctrl.ReplicaStartupCondition(errorIDs)) r.SetCondition(chctrl.HealthyCondition(notReadyIDs)) r.SetCondition(chctrl.ConfigSyncCondition(nil, notUpdatedIDs, nil)) - { - cond, event := chctrl.GetVersionSyncCondition(r.versionProbe, replicaVersions, len(notUpdatedIDs) > 0) - r.SetCondition(cond, event...) - } // Ready condition — keeper-specific logic. exists := len(r.ReplicaState) diff --git a/internal/controller/versionprobe.go b/internal/controller/versionprobe.go index 1aac3b62..9cdd23e7 100644 --- a/internal/controller/versionprobe.go +++ b/internal/controller/versionprobe.go @@ -26,12 +26,12 @@ const ( DefaultProbeCPURequest = "250m" DefaultProbeMemoryLimit = "256Mi" DefaultProbeMemoryRequest = "256Mi" + versionProbeBinary = "/usr/bin/clickhouse" + versionProbeQuery = "INSERT INTO FUNCTION file('/dev/termination-log', 'RawBLOB', 'version String') SELECT version()" ) // VersionProbeConfig holds parameters for the version probe Job. type VersionProbeConfig struct { - // Name of the binary to run. - Binary string // Labels to apply to the Job, inherited from the cluster spec. Labels map[string]string // Annotations to apply to the Job, inherited from the cluster spec. @@ -280,7 +280,8 @@ func (rm *ResourceManager) buildVersionProbeJob(cfg VersionProbeConfig, revision SecurityContext: DefaultContainerSecurityContext(), TerminationMessagePolicy: corev1.TerminationMessageReadFile, TerminationMessagePath: corev1.TerminationMessagePathDefault, - Command: []string{"sh", "-c", fmt.Sprintf("%s --version > %s 2>&1", cfg.Binary, corev1.TerminationMessagePathDefault)}, + Command: []string{versionProbeBinary}, + Args: []string{"local", "--query", versionProbeQuery}, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(DefaultProbeCPURequest), diff --git a/internal/controller/versionprobe_test.go b/internal/controller/versionprobe_test.go index 2c6e73f4..081dccc6 100644 --- a/internal/controller/versionprobe_test.go +++ b/internal/controller/versionprobe_test.go @@ -54,7 +54,8 @@ func baseJob() batchv1.Job { { Name: v1.VersionProbeContainerName, Image: "clickhouse/clickhouse-server:latest", - Command: []string{"sh", "-c", "clickhouse-server --version"}, + Command: []string{versionProbeBinary}, + Args: []string{"local", "--query", versionProbeQuery}, SecurityContext: &corev1.SecurityContext{ RunAsNonRoot: new(true), }, @@ -189,7 +190,8 @@ var _ = Describe("patchResource with jobSchema (version probe overrides)", func( By("verifying container command is preserved") Expect(container.Image).To(Equal("clickhouse/clickhouse-server:latest")) - Expect(container.Command).To(Equal([]string{"sh", "-c", "clickhouse-server --version"})) + Expect(container.Command).To(Equal([]string{versionProbeBinary})) + Expect(container.Args).To(Equal([]string{"local", "--query", versionProbeQuery})) }) It("should deep-merge securityContext via SMP", func() { @@ -311,7 +313,6 @@ func (f *fakeController) GetRecorder() events.EventRecorder { return f.recorder // probeCfg returns a VersionProbeConfig with the given image and cache values. func probeCfg(image, cachedVersion, cachedRevision string) VersionProbeConfig { return VersionProbeConfig{ - Binary: "clickhouse-server", ContainerTemplate: v1.ContainerTemplateSpec{ Image: v1.ContainerImage{Repository: image, Tag: "latest"}, }, diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 13afffac..45847759 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -172,7 +172,7 @@ var _ = BeforeSuite(func(ctx context.Context) { upgradeChecker := upgrade.NewChecker(updater) podDialer = testutil.NewPortForwardDialer(config) - Expect(keeper.SetupWithManager(mgr, zapLogger, upgradeChecker, podDialer, true)).To(Succeed()) + Expect(keeper.SetupWithManager(mgr, zapLogger, podDialer, true)).To(Succeed()) Expect(clickhouse.SetupWithManager(mgr, zapLogger, upgradeChecker, podDialer, true)).To(Succeed()) // +kubebuilder:scaffold:builder diff --git a/test/e2e/keeper_e2e_test.go b/test/e2e/keeper_e2e_test.go index 399a030d..2c7069b6 100644 --- a/test/e2e/keeper_e2e_test.go +++ b/test/e2e/keeper_e2e_test.go @@ -60,7 +60,7 @@ var _ = Describe("Keeper controller", Label("keeper"), func() { WaitKeeperUpdatedAndReady(ctx, &cr, 3*time.Minute, true) ExpectWithOffset(1, k8sClient.Get(ctx, cr.NamespacedName(), &cr)).To(Succeed()) - Expect(cr.Status.Version).To(HavePrefix(cr.Spec.ContainerTemplate.Image.Tag)) + KeeperRWChecks(ctx, &cr, &checks) }, Entry("update log level", v1.KeeperClusterSpec{Settings: v1.KeeperSettings{ From 06c10784c251ca596ca17c41f3bc29348ff9266a Mon Sep 17 00:00:00 2001 From: ashishch432 Date: Sun, 14 Jun 2026 19:35:58 +0000 Subject: [PATCH 2/4] revert aggressive cleanup of live keeper version --- internal/controller/keeper/commands.go | 8 +++ internal/controller/keeper/controller_test.go | 7 +++ internal/controller/keeper/sync.go | 59 ++++++++++++++++++- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/internal/controller/keeper/commands.go b/internal/controller/keeper/commands.go index 61ceead6..680096d1 100644 --- a/internal/controller/keeper/commands.go +++ b/internal/controller/keeper/commands.go @@ -29,6 +29,7 @@ var ( type serverStatus struct { ServerState string Followers int + Version string } func getConnection(ctx context.Context, dialer controllerutil.DialContextFunc, hostname string, tlsRequired bool) (net.Conn, error) { @@ -103,6 +104,13 @@ func queryKeeper(ctx context.Context, log controllerutil.Logger, conn net.Conn) ServerState: statMap["zk_server_state"], } + version, err := controllerutil.ParseVersion(statMap["zk_version"]) + if err != nil { + log.Warn("failed to parse keeper version", "raw", statMap["zk_version"], "error", err) + } else { + result.Version = version + } + if result.ServerState == "" { return serverStatus{}, fmt.Errorf("response missing required field 'Mode': %q", string(data)) } diff --git a/internal/controller/keeper/controller_test.go b/internal/controller/keeper/controller_test.go index 68064dd2..a4e83be6 100644 --- a/internal/controller/keeper/controller_test.go +++ b/internal/controller/keeper/controller_test.go @@ -9,8 +9,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -112,6 +114,11 @@ var _ = When("reconciling standalone KeeperCluster resource", Ordered, func() { Expect(suite.Client.List(ctx, &statefulsets, listOpts)).To(Succeed()) Expect(statefulsets.Items).To(HaveLen(1)) + var jobs batchv1.JobList + Expect(suite.Client.List(ctx, &jobs, listOpts)).To(Succeed()) + Expect(jobs.Items).To(BeEmpty()) + Expect(meta.FindStatusCondition(cr.Status.Conditions, v1.ConditionTypeVersionUpgraded)).To(BeNil()) + testutil.AssertEvents(recorder.Events, map[string]int{ "ReplicaCreated": 1, "ClusterNotReady": 1, diff --git a/internal/controller/keeper/sync.go b/internal/controller/keeper/sync.go index c8385db6..73d4f911 100644 --- a/internal/controller/keeper/sync.go +++ b/internal/controller/keeper/sync.go @@ -7,6 +7,7 @@ import ( "math" "slices" "strconv" + "strings" "time" "gopkg.in/yaml.v2" @@ -92,7 +93,6 @@ type keeperReconciler struct { func (r *keeperReconciler) sync(ctx context.Context, log ctrlutil.Logger) (ctrl.Result, error) { log.Info("Enter Keeper Reconcile", "spec", r.Cluster.Spec, "status", r.Cluster.Status) - meta.RemoveStatusCondition(&r.Cluster.Status.Conditions, v1.ConditionTypeVersionInSync) meta.RemoveStatusCondition(&r.Cluster.Status.Conditions, v1.ConditionTypeVersionUpgraded) r.SetUnknownConditions(v1.ConditionReasonStepFailed, "Reconcile stopped before condition evaluation", @@ -101,6 +101,7 @@ func (r *keeperReconciler) sync(ctx context.Context, log ctrlutil.Logger) (ctrl. v1.ConditionTypeHealthy, v1.ConditionTypeClusterSizeAligned, v1.ConditionTypeConfigurationInSync, + v1.ConditionTypeVersionInSync, v1.ConditionTypeReady, v1.KeeperConditionTypeScaleAllowed, }) @@ -565,6 +566,7 @@ func (r *keeperReconciler) evaluateReplicaConditions() { var errorIDs, notReadyIDs, notUpdatedIDs []string replicasByMode := map[string][]v1.KeeperReplicaID{} + replicaVersions := map[string]string{} r.Cluster.Status.ReadyReplicas = 0 for id, replica := range r.ReplicaState { @@ -584,11 +586,17 @@ func (r *keeperReconciler) evaluateReplicaConditions() { if replica.HasDiff(r.revs) || !replica.Updated() { notUpdatedIDs = append(notUpdatedIDs, idStr) } + + if replica.Status.Version != "" { + replicaVersions[idStr] = replica.Status.Version + } } r.SetCondition(chctrl.ReplicaStartupCondition(errorIDs)) r.SetCondition(chctrl.HealthyCondition(notReadyIDs)) r.SetCondition(chctrl.ConfigSyncCondition(nil, notUpdatedIDs, nil)) + versionCond, versionEvents := keeperVersionSyncCondition(replicaVersions, len(notUpdatedIDs) > 0) + r.SetCondition(versionCond, versionEvents...) // Ready condition — keeper-specific logic. exists := len(r.ReplicaState) @@ -667,6 +675,55 @@ func (r *keeperReconciler) evaluateReplicaConditions() { ) } +func keeperVersionSyncCondition(replicaVersions map[string]string, isUpdating bool) (metav1.Condition, []chctrl.EventSpec) { + newCond := func(status metav1.ConditionStatus, reason v1.ConditionReason, message string) metav1.Condition { + return metav1.Condition{ + Type: v1.ConditionTypeVersionInSync, + Status: status, + Reason: reason, + Message: message, + } + } + + if len(replicaVersions) == 0 { + return newCond(metav1.ConditionUnknown, v1.ConditionReasonVersionPending, "No Keeper replica version has been observed yet"), nil + } + + versions := map[string]struct{}{} + + var observed []string + for id, version := range replicaVersions { + versions[version] = struct{}{} + observed = append(observed, fmt.Sprintf("%s: %s", id, version)) + } + + if len(versions) == 1 { + var version string + for _, v := range replicaVersions { + version = v + break + } + + return newCond(metav1.ConditionTrue, v1.ConditionReasonVersionMatch, + "All observed Keeper replicas report version "+version), nil + } + + slices.Sort(observed) + cond := newCond(metav1.ConditionFalse, v1.ConditionReasonVersionMismatch, + "Keeper replica versions differ: "+strings.Join(observed, ", ")) + + if isUpdating { + return cond, nil + } + + return cond, []chctrl.EventSpec{{ + Type: corev1.EventTypeWarning, + Reason: v1.EventReasonVersionDiverge, + Action: v1.EventActionVersionCheck, + Message: cond.Message, + }} +} + func (r *keeperReconciler) updateReplica(ctx context.Context, log ctrlutil.Logger, replicaID v1.KeeperReplicaID) (*ctrl.Result, error) { log = log.With("replica_id", replicaID) log.Info("updating replica") From a40b68dba2748ce6d1dec9c28b4489a445e3ba50 Mon Sep 17 00:00:00 2001 From: ashishch432 Date: Mon, 15 Jun 2026 18:09:57 +0000 Subject: [PATCH 3/4] address review comments --- api/v1alpha1/keepercluster_types.go | 23 + api/v1alpha1/zz_generated.deepcopy.go | 5 + cmd/main.go | 2 +- .../bases/clickhouse.com_keeperclusters.yaml | 638 ++++++++++++++++++ ...khouse-operator.clusterserviceversion.yaml | 11 +- dist/chart-cluster/values.yaml | 9 + .../crd/keeperclusters.clickhouse.com.yaml | 638 ++++++++++++++++++ docs/guides/configuration.mdx | 150 ++-- docs/reference/api-reference.mdx | 5 + internal/controller/keeper/controller.go | 11 +- internal/controller/keeper/sync.go | 48 +- internal/controller/keeper/sync_test.go | 96 +++ test/e2e/e2e_suite_test.go | 2 +- 13 files changed, 1547 insertions(+), 91 deletions(-) diff --git a/api/v1alpha1/keepercluster_types.go b/api/v1alpha1/keepercluster_types.go index 197a76c6..885a4d4d 100644 --- a/api/v1alpha1/keepercluster_types.go +++ b/api/v1alpha1/keepercluster_types.go @@ -58,6 +58,18 @@ type KeeperClusterSpec struct { // +optional // +kubebuilder:default:="cluster.local" ClusterDomain string `json:"clusterDomain,omitempty"` + + // UpgradeChannel specifies the release channel for major version upgrade checks. + // When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). + // +optional + // +kubebuilder:validation:Pattern=`^(lts|stable|\d+\.\d+)?$` + UpgradeChannel string `json:"upgradeChannel,omitempty"` + + // VersionProbeTemplate overrides for the version detection Job. + // + // Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. + // +optional + VersionProbeTemplate *VersionProbeTemplate `json:"versionProbeTemplate,omitempty"` } // WithDefaults sets default values for KeeperClusterSpec fields. @@ -151,6 +163,16 @@ type KeeperClusterStatus struct { // ObservedGeneration indicates latest generation observed by controller. // +operator-sdk:csv:customresourcedefinitions:type=status ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // Version indicates the version reported by the Keeper server. + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=status + Version string `json:"version,omitempty"` + // VersionProbeRevision is the image hash of the last successful version probe. + // + // Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=status + VersionProbeRevision string `json:"versionProbeRevision,omitempty"` } // KeeperCluster is the Schema for the `keeperclusters` API. @@ -161,6 +183,7 @@ type KeeperClusterStatus struct { // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message" // +kubebuilder:printcolumn:name="ReadyReplicas",type="number",JSONPath=".status.readyReplicas" // +kubebuilder:printcolumn:name="Replicas",type="number",JSONPath=".spec.replicas" +// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".status.version" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +operator-sdk:csv:customresourcedefinitions:resources={{Pod,v1}} // +operator-sdk:csv:customresourcedefinitions:resources={{PersistentVolumeClaim,v1}} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6ffafacc..7dc5dc61 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -466,6 +466,11 @@ func (in *KeeperClusterSpec) DeepCopyInto(out *KeeperClusterSpec) { (*in).DeepCopyInto(*out) } in.Settings.DeepCopyInto(&out.Settings) + if in.VersionProbeTemplate != nil { + in, out := &in.VersionProbeTemplate, &out.VersionProbeTemplate + *out = new(VersionProbeTemplate) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeeperClusterSpec. diff --git a/cmd/main.go b/cmd/main.go index 704ca963..33a3a5b3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -201,7 +201,7 @@ func run() error { upgradeChecker = upgrade.NewChecker(updater) } - if err = keeper.SetupWithManager(mgr, zapLogger, nil, env.EnablePDB); err != nil { + if err = keeper.SetupWithManager(mgr, zapLogger, upgradeChecker, nil, env.EnablePDB); err != nil { return fmt.Errorf("unable to setup KeeperCluster controller: %w", err) } diff --git a/config/crd/bases/clickhouse.com_keeperclusters.yaml b/config/crd/bases/clickhouse.com_keeperclusters.yaml index db842ae6..4e9bad54 100644 --- a/config/crd/bases/clickhouse.com_keeperclusters.yaml +++ b/config/crd/bases/clickhouse.com_keeperclusters.yaml @@ -30,6 +30,9 @@ spec: - jsonPath: .spec.replicas name: Replicas type: number + - jsonPath: .status.version + name: Version + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -6153,6 +6156,631 @@ spec: x-kubernetes-map-type: atomic type: object type: object + upgradeChannel: + description: |- + UpgradeChannel specifies the release channel for major version upgrade checks. + When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). + pattern: ^(lts|stable|\d+\.\d+)?$ + type: string + versionProbeTemplate: + description: |- + VersionProbeTemplate overrides for the version detection Job. + + Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. + properties: + metadata: + description: Metadata applied to the version probe Job. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are annotations applied to the template + objects. + type: object + labels: + additionalProperties: + type: string + description: Labels are labels applied to the template objects. + type: object + type: object + spec: + description: Specification of the desired behavior of the version + probe Job. + properties: + template: + description: Template describes the pod that will be created + for the version probe Job. + properties: + metadata: + description: Metadata applied to version probe Pods. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are annotations applied to + the template objects. + type: object + labels: + additionalProperties: + type: string + description: Labels are labels applied to the template + objects. + type: object + type: object + spec: + description: Specification of the desired behavior of + the version probe Pod. + properties: + containers: + description: |- + Containers overrides for the version probe Pod. + The name field is optional — the operator fills it with default container. + Additional container with the different name may be specified. + items: + description: |- + VersionProbeContainer defines container-level overrides for the version probe. + Field names and JSON tags match corev1.Container so that SMP merges by name. + properties: + name: + default: version-probe + description: Name of the container. If empty, + the operator sets it to the version probe + container name. + type: string + resources: + description: |- + Resources are the compute resource requirements for the version probe container. + Deep-merged with operator defaults via SMP. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + SecurityContext defines the security options for the version probe container. + Deep-merged with operator defaults via SMP. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeSelector: + additionalProperties: + type: string + description: NodeSelector constrains the version probe + Pod to nodes with matching labels. + type: object + x-kubernetes-map-type: atomic + securityContext: + description: SecurityContext holds pod-level security + attributes for the version probe Pod. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + tolerations: + description: Tolerations for the version probe Pod. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + ttlSecondsAfterFinished: + description: TTLSecondsAfterFinished limits the lifetime of + a completed Job. + format: int32 + type: integer + type: object + type: object type: object status: description: KeeperClusterStatus defines the observed state of KeeperCluster. @@ -6242,6 +6870,16 @@ spec: description: UpdateRevision indicates latest requested KeeperCluster spec revision. type: string + version: + description: Version indicates the version reported by the Keeper + server. + type: string + versionProbeRevision: + description: |- + VersionProbeRevision is the image hash of the last successful version probe. + + Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. + type: string type: object type: object served: true diff --git a/config/manifests/bases/clickhouse-operator.clusterserviceversion.yaml b/config/manifests/bases/clickhouse-operator.clusterserviceversion.yaml index 75c9d63d..470f8913 100644 --- a/config/manifests/bases/clickhouse-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/clickhouse-operator.clusterserviceversion.yaml @@ -87,7 +87,8 @@ spec: displayName: Version Probe Revision path: versionProbeRevision version: v1alpha1 - - displayName: Keeper Cluster + - description: KeeperCluster is the Schema for the `keeperclusters` API. + displayName: Keeper Cluster kind: KeeperCluster name: keeperclusters.clickhouse.com resources: @@ -140,6 +141,14 @@ spec: revision. displayName: Update Revision path: updateRevision + - description: Version indicates the version reported by the Keeper server. + displayName: Version + path: version + - description: |- + VersionProbeRevision is the image hash of the last successful version probe. + Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. + displayName: Version Probe Revision + path: versionProbeRevision version: v1alpha1 description: | The ClickHouse Operator is a Kubernetes operator that automates the deployment, configuration, and management of ClickHouse clusters and ClickHouse Keeper clusters on Kubernetes. diff --git a/dist/chart-cluster/values.yaml b/dist/chart-cluster/values.yaml index f6b640e5..91b606ee 100644 --- a/dist/chart-cluster/values.yaml +++ b/dist/chart-cluster/values.yaml @@ -151,3 +151,12 @@ keeper: # Configuration parameters for ClickHouse Keeper server. # settings: {} + # UpgradeChannel specifies the release channel for major version upgrade checks. + # When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). + # upgradeChannel: "" + + # VersionProbeTemplate overrides for the version detection Job. + # + # Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. + # versionProbeTemplate: {} + diff --git a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml index ce81a333..666c2fd5 100644 --- a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml +++ b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml @@ -33,6 +33,9 @@ spec: - jsonPath: .spec.replicas name: Replicas type: number + - jsonPath: .status.version + name: Version + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -6156,6 +6159,631 @@ spec: x-kubernetes-map-type: atomic type: object type: object + upgradeChannel: + description: |- + UpgradeChannel specifies the release channel for major version upgrade checks. + When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). + pattern: ^(lts|stable|\d+\.\d+)?$ + type: string + versionProbeTemplate: + description: |- + VersionProbeTemplate overrides for the version detection Job. + + Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. + properties: + metadata: + description: Metadata applied to the version probe Job. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are annotations applied to the template + objects. + type: object + labels: + additionalProperties: + type: string + description: Labels are labels applied to the template objects. + type: object + type: object + spec: + description: Specification of the desired behavior of the version + probe Job. + properties: + template: + description: Template describes the pod that will be created + for the version probe Job. + properties: + metadata: + description: Metadata applied to version probe Pods. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are annotations applied to + the template objects. + type: object + labels: + additionalProperties: + type: string + description: Labels are labels applied to the template + objects. + type: object + type: object + spec: + description: Specification of the desired behavior of + the version probe Pod. + properties: + containers: + description: |- + Containers overrides for the version probe Pod. + The name field is optional — the operator fills it with default container. + Additional container with the different name may be specified. + items: + description: |- + VersionProbeContainer defines container-level overrides for the version probe. + Field names and JSON tags match corev1.Container so that SMP merges by name. + properties: + name: + default: version-probe + description: Name of the container. If empty, + the operator sets it to the version probe + container name. + type: string + resources: + description: |- + Resources are the compute resource requirements for the version probe container. + Deep-merged with operator defaults via SMP. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + SecurityContext defines the security options for the version probe container. + Deep-merged with operator defaults via SMP. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeSelector: + additionalProperties: + type: string + description: NodeSelector constrains the version probe + Pod to nodes with matching labels. + type: object + x-kubernetes-map-type: atomic + securityContext: + description: SecurityContext holds pod-level security + attributes for the version probe Pod. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + tolerations: + description: Tolerations for the version probe Pod. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + ttlSecondsAfterFinished: + description: TTLSecondsAfterFinished limits the lifetime of + a completed Job. + format: int32 + type: integer + type: object + type: object type: object status: description: KeeperClusterStatus defines the observed state of KeeperCluster. @@ -6245,6 +6873,16 @@ spec: description: UpdateRevision indicates latest requested KeeperCluster spec revision. type: string + version: + description: Version indicates the version reported by the Keeper + server. + type: string + versionProbeRevision: + description: |- + VersionProbeRevision is the image hash of the last successful version probe. + + Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. + type: string type: object type: object served: true diff --git a/docs/guides/configuration.mdx b/docs/guides/configuration.mdx index c2b63f47..7210c373 100644 --- a/docs/guides/configuration.mdx +++ b/docs/guides/configuration.mdx @@ -158,12 +158,12 @@ For ClickHouse clusters with more than one shard, **one PDB is created per shard The operator picks safe defaults based on the cluster size so that a fresh `apply` already protects against accidental quorum loss. -| Resource | Topology | Default PDB | -|---|---|---| -| `ClickHouseCluster` | `replicas: 1` (single-replica shard) | `maxUnavailable: 1` — disruption is allowed for a single-node cluster so that node drains are not blocked | -| `ClickHouseCluster` | `replicas: 2+` (multi-replica shard) | `minAvailable: 1` — at least one replica per shard must stay up | -| `KeeperCluster` | `replicas: 1` | `maxUnavailable: 1` — disruption is allowed for a single-node cluster so that node drains are not blocked | -| `KeeperCluster` | `replicas: 3+` | `maxUnavailable: replicas/2` — preserves the RAFT quorum for a `2F+1` cluster (3 replicas tolerate 1 down, 5 replicas tolerate 2 down) | +| Resource | Topology | Default PDB | +|---------------------|--------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| `ClickHouseCluster` | `replicas: 1` (single-replica shard) | `maxUnavailable: 1` — disruption is allowed for a single-node cluster so that node drains are not blocked | +| `ClickHouseCluster` | `replicas: 2+` (multi-replica shard) | `minAvailable: 1` — at least one replica per shard must stay up | +| `KeeperCluster` | `replicas: 1` | `maxUnavailable: 1` — disruption is allowed for a single-node cluster so that node drains are not blocked | +| `KeeperCluster` | `replicas: 3+` | `maxUnavailable: replicas/2` — preserves the RAFT quorum for a `2F+1` cluster (3 replicas tolerate 1 down, 5 replicas tolerate 2 down) | For a 3-shard ClickHouseCluster with `replicas: 3`, the operator creates three PDBs, one per shard, each with `minAvailable: 1`. @@ -205,11 +205,11 @@ spec: `spec.podDisruptionBudget.policy` lets you choose **how aggressively** the operator manages PDBs: -| Policy | Behavior | -|---|---| -| `Enabled` (default) | The operator creates and updates the PDB on every reconcile. This is the safe production default. | -| `Disabled` | The operator does **not** create PDBs and **deletes** any existing ones with matching labels. Useful for development clusters where every voluntary disruption should be allowed. | -| `Ignored` | The operator neither creates nor deletes PDBs. Existing PDBs are left alone. Use this when another system (e.g. policy admission, GitOps tool) owns PDB management for you. | +| Policy | Behavior | +|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Enabled` (default) | The operator creates and updates the PDB on every reconcile. This is the safe production default. | +| `Disabled` | The operator does **not** create PDBs and **deletes** any existing ones with matching labels. Useful for development clusters where every voluntary disruption should be allowed. | +| `Ignored` | The operator neither creates nor deletes PDBs. Existing PDBs are left alone. Use this when another system (e.g. policy admission, GitOps tool) owns PDB management for you. | Example — disable PDB management completely on a development cluster: @@ -385,12 +385,12 @@ The referenced Secret must reside in the **same namespace** as the ClickHouseClu The Secret must contain the following keys: -| Key | Format | When required | -|---|---|---| -| `interserver-password` | plaintext password | Always | -| `management-password` | plaintext password | Always | -| `keeper-identity` | `clickhouse:` | Always | -| `cluster-secret` | plaintext password | Always | +| Key | Format | When required | +|-------------------------|--------------------------------------------|----------------------------| +| `interserver-password` | plaintext password | Always | +| `management-password` | plaintext password | Always | +| `keeper-identity` | `clickhouse:` | Always | +| `cluster-secret` | plaintext password | Always | | `named-collections-key` | hex-encoded 16-byte AES key (32 hex chars) | ClickHouse `>= 25.12` only | A complete Secret looks like this: @@ -414,10 +414,10 @@ stringData: `spec.externalSecret.policy` controls how the operator handles missing required keys: -| Policy | Behavior on missing keys | -|---|---| -| `Observe` (default) | Reconciliation is **blocked** until every required key is present. The operator reports each missing key — and the format hint for it — via the `ExternalSecretValid` condition (with reason `ExternalSecretInvalid`) and a `Warning` event. | -| `Manage` | The operator **generates** any missing required keys and writes them back to the same Secret. Useful for bootstrapping: create an empty Secret, let the operator fill it, then optionally tighten access. The operator still never deletes the Secret. | +| Policy | Behavior on missing keys | +|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Observe` (default) | Reconciliation is **blocked** until every required key is present. The operator reports each missing key — and the format hint for it — via the `ExternalSecretValid` condition (with reason `ExternalSecretInvalid`) and a `Warning` event. | +| `Manage` | The operator **generates** any missing required keys and writes them back to the same Secret. Useful for bootstrapping: create an empty Secret, let the operator fill it, then optionally tighten access. The operator still never deletes the Secret. | Even with `policy: Manage` the Secret must already exist in the namespace — the operator never creates the Secret itself, it only writes generated keys into an existing one. If the referenced Secret is missing, reconciliation is blocked with the `ExternalSecretNotFound` reason regardless of policy. @@ -442,11 +442,11 @@ kubectl get clickhousecluster sample -o jsonpath='{.status.conditions}' | jq Possible reasons: -| `reason` | Meaning | Fix | -|---|---|---| -| `ExternalSecretNotFound` | The referenced Secret does not exist in the namespace. | Create the Secret, or fix `spec.externalSecret.name`. | -| `ExternalSecretInvalid` | The Secret exists but lacks required keys (only with `Observe`). The message lists each missing key together with its expected format. | Add the missing keys, or switch to `policy: Manage`. | -| `ExternalSecretValid` | All required keys are present and the operator is using the Secret. | — | +| `reason` | Meaning | Fix | +|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------| +| `ExternalSecretNotFound` | The referenced Secret does not exist in the namespace. | Create the Secret, or fix `spec.externalSecret.name`. | +| `ExternalSecretInvalid` | The Secret exists but lacks required keys (only with `Observe`). The message lists each missing key together with its expected format. | Add the missing keys, or switch to `policy: Manage`. | +| `ExternalSecretValid` | All required keys are present and the operator is using the Secret. | — | The operator requeues reconciliation while the Secret is invalid, so once you add the missing keys the next reconcile picks them up automatically — no need to bounce pods. @@ -517,36 +517,36 @@ kubectl exec sample-clickhouse-0-0-0 -- \ ### Field constraints {#additional-ports-constraints} -| Field | Rule | -|---|---| +| Field | Rule | +|--------|------------------------------------------------------------------------------------------------------------------------------------------| | `name` | Must match the DNS_LABEL pattern `^[a-z]([-a-z0-9]*[a-z0-9])?$`, max 63 characters. Uniqueness is enforced by the CRD as a list-map key. | -| `port` | Integer in `[1, 65535]`. The webhook rejects duplicate port numbers within the list. | +| `port` | Integer in `[1, 65535]`. The webhook rejects duplicate port numbers within the list. | ### Reserved ports and names {#additional-ports-reserved} The validating webhook rejects `additionalPorts` entries that would collide with ports the operator binds itself. All TLS-related ports are reserved **unconditionally** so that flipping `spec.settings.tls.enabled` later cannot break a previously valid cluster. -| Port | Reserved for | -|---|---| -| `8123` | HTTP | -| `8443` | HTTPS | -| `9000` | native TCP | -| `9440` | native TLS | -| `9009` | interserver | -| `9001` | management | +| Port | Reserved for | +|--------|--------------------| +| `8123` | HTTP | +| `8443` | HTTPS | +| `9000` | native TCP | +| `9440` | native TLS | +| `9009` | interserver | +| `9001` | management | | `9363` | Prometheus metrics | The following names are also rejected — they are the operator's internal protocol-type identifiers (not the human-readable aliases): -| Name | -|---| -| `http` | +| Name | +|---------------| +| `http` | | `http-secure` | -| `tcp` | -| `tcp-secure` | +| `tcp` | +| `tcp-secure` | | `interserver` | -| `management` | -| `prometheus` | +| `management` | +| `prometheus` | A rejected request produces an error such as: @@ -557,14 +557,14 @@ spec.additionalPorts[0].name: "http" is reserved by the operator ## Version probe and upgrade channel {#version-probe-and-upgrade-channel} -The operator does two independent things with ClickHouse cluster versions: +The operator does two independent things with cluster versions: -1. **Version probe** — a Kubernetes `Job` that runs the ClickHouse container image once to detect the running ClickHouse version. The detected version is recorded in `.status.version` and used by other reconciliation steps (e.g. the `External Secret` named-collections key is only required from ClickHouse `25.12`). +1. **Version reporting** — for `ClickHouseCluster`, a Kubernetes `Job` runs the container image once to detect the running ClickHouse version; for `KeeperCluster`, the operator reads the server-reported version from running replicas. The detected version is recorded in `.status.version` and used by other reconciliation steps (e.g. the `External Secret` named-collections key is only required from ClickHouse `25.12`). 2. **Upgrade channel** — a periodic check against the public ClickHouse release feed (`https://clickhouse.com/data/version_date.tsv`). The operator reports whether a newer version is available via the `VersionUpgraded` status condition. It never upgrades the cluster on its own — the user is in control of the image tag. ### Choosing a release channel {#upgrade-channel-choosing} -`ClickHouseCluster` `spec.upgradeChannel` selects which set of upstream releases the operator compares against. +`spec.upgradeChannel` selects which set of upstream releases the operator compares against. Same field exists on both `ClickHouseCluster` and `KeeperCluster`. ```yaml spec: @@ -573,12 +573,12 @@ spec: Allowed values (validated by the CRD with the pattern `^(lts|stable|\d+\.\d+)?$`): -| Value | Behavior | -|---|---| -| _empty_ (default) | The operator proposes only **minor** updates within the currently-running major.minor line. A cluster on `25.8.3.1` will be told about `25.8.4.x` but not `25.9.x`. | -| `stable` | Tracks the upstream `stable` channel — the latest release that ClickHouse Inc. flags as stable on the main release line. Receives major upgrades sooner than the `lts` channel. | -| `lts` | Tracks the upstream `lts` channel — long-term support releases. Receives major upgrades less frequently, with longer support windows. | -| `25.8` (or any `.`) | Pins the channel to a specific major.minor line. Major upgrades beyond it are not proposed even if a newer version exists upstream. | +| Value | Behavior | +|-----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| _empty_ (default) | The operator proposes only **minor** updates within the currently-running major.minor line. A cluster on `25.8.3.1` will be told about `25.8.4.x` but not `25.9.x`. | +| `stable` | Tracks the upstream `stable` channel — the latest release that ClickHouse Inc. flags as stable on the main release line. Receives major upgrades sooner than the `lts` channel. | +| `lts` | Tracks the upstream `lts` channel — long-term support releases. Receives major upgrades less frequently, with longer support windows. | +| `25.8` (or any `.`) | Pins the channel to a specific major.minor line. Major upgrades beyond it are not proposed even if a newer version exists upstream. | For production, pinning the channel to an explicit `.` (e.g. `25.8`) is generally preferred. It locks the cluster to the intended major release line and lets the operator surface a `WrongReleaseChannel` warning if any replica somehow drifts onto a different major — which matters especially when the image is referenced by a digest (`@sha256:...`) rather than by a human-readable tag. The empty default is fine for development clusters where major-version jumps are not a concern. @@ -586,18 +586,18 @@ For production, pinning the channel to an explicit `.` (e.g. `25.8 Two conditions surface the result of the probe and the upgrade check: -| Condition | Reason | Meaning | -|---|---|---| -| `VersionInSync` | `VersionMatch` | All replicas report the same version as the image | -| `VersionInSync` | `VersionMismatch` | Replicas are running different versions. The warning event is suppressed during a planned rolling upgrade. It typically surfaces when a mutable image tag has been pinned (for example `latest` or a bare major like `26.3`) and the underlying registry has shifted between pulls, so different replicas ended up on different patches of the same tag. | -| `VersionInSync` | `VersionPending` | Version probe Job has not finished yet | -| `VersionInSync` | `VersionProbeFailed` | Probe Job failed; the operator cannot determine the running version | -| `VersionUpgraded` | `UpToDate` | The cluster is on the latest version available in the selected channel | -| `VersionUpgraded` | `MinorUpdateAvailable` | A newer patch is available in the same `major.minor` line | -| `VersionUpgraded` | `MajorUpdateAvailable` | A newer `major.minor` is available within the chosen channel | -| `VersionUpgraded` | `VersionOutdated` | The running version is out of date and will no longer receive fixes from the selected channel — typically because the major line has been dropped from `lts` or `stable` upstream | -| `VersionUpgraded` | `WrongReleaseChannel` | The running image does not belong to the selected `upgradeChannel`. Example: a cluster running `26.5` with `upgradeChannel: lts`, since `26.5` is not part of the upstream `lts` line. | -| `VersionUpgraded` | `UpgradeCheckFailed` | The operator could not reach the upstream release feed | +| Condition | Reason | Meaning | +|-------------------|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `VersionInSync` | `VersionMatch` | All replicas report the same version as the image | +| `VersionInSync` | `VersionMismatch` | Replicas are running different versions. This reason is suppressed during a planned rolling upgrade. It typically surfaces when a mutable image tag has been pinned (for example `latest` or a bare major like `26.3`) and the underlying registry has shifted between pulls, so different replicas ended up on different patches of the same tag. | +| `VersionInSync` | `VersionPending` | Version probe Job has not finished yet, or no Keeper replica version has been observed yet | +| `VersionInSync` | `VersionProbeFailed` | ClickHouse probe Job failed; the operator cannot determine the running version | +| `VersionUpgraded` | `UpToDate` | The cluster is on the latest version available in the selected channel | +| `VersionUpgraded` | `MinorUpdateAvailable` | A newer patch is available in the same `major.minor` line | +| `VersionUpgraded` | `MajorUpdateAvailable` | A newer `major.minor` is available within the chosen channel | +| `VersionUpgraded` | `VersionOutdated` | The running version is out of date and will no longer receive fixes from the selected channel — typically because the major line has been dropped from `lts` or `stable` upstream | +| `VersionUpgraded` | `WrongReleaseChannel` | The running image does not belong to the selected `upgradeChannel`. Example: a cluster running `26.5` with `upgradeChannel: lts`, since `26.5` is not part of the upstream `lts` line. | +| `VersionUpgraded` | `UpgradeCheckFailed` | The operator could not reach the upstream release feed | Inspect them with: @@ -637,9 +637,9 @@ The container name `version-probe` is the operator's default — the entry under Two flags on the operator manager control the upgrade-check loop globally: -| Flag | Default | Effect | -|---|---|---| -| `--version-update-interval` | `24h` | How often the operator re-fetches the upstream version list | +| Flag | Default | Effect | +|-----------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| `--version-update-interval` | `24h` | How often the operator re-fetches the upstream version list | | `--disable-version-update-checks` | `false` | Disables the upgrade checker entirely. The `VersionUpgraded` condition is not set, and no outbound HTTP traffic to `clickhouse.com` is generated | Set `--disable-version-update-checks=true` in air-gapped environments or when egress to `clickhouse.com` is not allowed. @@ -763,13 +763,13 @@ spec: count: 50 # Default: 50. Number of rotated files to keep ``` -| Field | Default | Description | -|---|---|---| -| `logToFile` | `true` | When `false`, the operator drops the file targets and the server logs only to the container console. | -| `jsonLogs` | `false` | When `true`, the operator adds `formatting.type: json` so each line is a JSON object. | -| `level` | `trace` | Log verbosity. One of `test`, `trace`, `debug`, `information`, `notice`, `warning`, `error`, `critical`, `fatal`. | -| `size` | `1000M` | Maximum size of a single log file before rotation. | -| `count` | `50` | Number of rotated log files the server retains. | +| Field | Default | Description | +|-------------|---------|-------------------------------------------------------------------------------------------------------------------| +| `logToFile` | `true` | When `false`, the operator drops the file targets and the server logs only to the container console. | +| `jsonLogs` | `false` | When `true`, the operator adds `formatting.type: json` so each line is a JSON object. | +| `level` | `trace` | Log verbosity. One of `test`, `trace`, `debug`, `information`, `notice`, `warning`, `error`, `critical`, `fatal`. | +| `size` | `1000M` | Maximum size of a single log file before rotation. | +| `count` | `50` | Number of rotated log files the server retains. | The operator always keeps console logging on so that `kubectl logs` works, and layers file logging on top when `logToFile` is `true`. A cluster with the defaults renders this `logger` block: diff --git a/docs/reference/api-reference.mdx b/docs/reference/api-reference.mdx index 4419a5bd..751af4ae 100644 --- a/docs/reference/api-reference.mdx +++ b/docs/reference/api-reference.mdx @@ -273,6 +273,8 @@ KeeperClusterSpec defines the desired state of KeeperCluster. | `podDisruptionBudget` | [PodDisruptionBudgetSpec](#poddisruptionbudgetspec) | PodDisruptionBudget configures the PDB created for the Keeper cluster.
When unset, the operator defaults to maxUnavailable=replicas/2
(preserving quorum for a 2F+1 cluster); single-replica clusters use maxUnavailable=1. | false | | | `settings` | [KeeperSettings](#keepersettings) | Configuration parameters for ClickHouse Keeper server. | false | | | `clusterDomain` | string | ClusterDomain is the Kubernetes cluster domain suffix used for DNS resolution. | false | cluster.local | +| `upgradeChannel` | string | UpgradeChannel specifies the release channel for major version upgrade checks.
When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). | false | | +| `versionProbeTemplate` | [VersionProbeTemplate](#versionprobetemplate) | VersionProbeTemplate overrides for the version detection Job.
Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. | false | | Appears in: - [KeeperCluster](#keepercluster) @@ -290,6 +292,8 @@ KeeperClusterStatus defines the observed state of KeeperCluster. | `currentRevision` | string | CurrentRevision indicates latest applied KeeperCluster spec revision. | true | | | `updateRevision` | string | UpdateRevision indicates latest requested KeeperCluster spec revision. | true | | | `observedGeneration` | integer | ObservedGeneration indicates latest generation observed by controller. | true | | +| `version` | string | Version indicates the version reported by the Keeper server. | false | | +| `versionProbeRevision` | string | VersionProbeRevision is the image hash of the last successful version probe.
Deprecated: Keeper version probe Jobs are not used; this field is retained for backward compatibility. | false | | Appears in: - [KeeperCluster](#keepercluster) @@ -470,3 +474,4 @@ The structure mirrors batchv1.JobTemplateSpec, exposing only supported fields. Appears in: - [ClickHouseClusterSpec](#clickhouseclusterspec) +- [KeeperClusterSpec](#keeperclusterspec) diff --git a/internal/controller/keeper/controller.go b/internal/controller/keeper/controller.go index 60bef4b7..5d52ef1d 100644 --- a/internal/controller/keeper/controller.go +++ b/internal/controller/keeper/controller.go @@ -19,6 +19,7 @@ import ( v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1" chctrl "github.com/ClickHouse/clickhouse-operator/internal/controller" "github.com/ClickHouse/clickhouse-operator/internal/controllerutil" + "github.com/ClickHouse/clickhouse-operator/internal/upgrade" webhookv1 "github.com/ClickHouse/clickhouse-operator/internal/webhook/v1alpha1" ) @@ -30,6 +31,7 @@ type ClusterController struct { Recorder events.EventRecorder Logger controllerutil.Logger Webhook webhookv1.KeeperClusterWebhook + Checker *upgrade.Checker Dialer controllerutil.DialContextFunc EnablePDB bool } @@ -97,6 +99,7 @@ func (cc *ClusterController) Reconcile(ctx context.Context, req ctrl.Request) (c ResourceManager: chctrl.NewResourceManager(cc, cluster), Dialer: cc.Dialer, + Checker: cc.Checker, EnablePDB: cc.EnablePDB, Cluster: cluster, @@ -121,13 +124,18 @@ func (cc *ClusterController) GetRecorder() events.EventRecorder { return cc.Recorder } +// GetVersionChecker returns the version upgrade Checker. +func (cc *ClusterController) GetVersionChecker() *upgrade.Checker { + return cc.Checker +} + // GetDialer returns the custom dialer, or nil. func (cc *ClusterController) GetDialer() controllerutil.DialContextFunc { return cc.Dialer } // SetupWithManager sets up the controller with the Manager. -func SetupWithManager(mgr ctrl.Manager, log controllerutil.Logger, dialer controllerutil.DialContextFunc, enablePDB bool) error { +func SetupWithManager(mgr ctrl.Manager, log controllerutil.Logger, checker *upgrade.Checker, dialer controllerutil.DialContextFunc, enablePDB bool) error { namedLogger := log.Named("keeper") keeperController := &ClusterController{ @@ -136,6 +144,7 @@ func SetupWithManager(mgr ctrl.Manager, log controllerutil.Logger, dialer contro Recorder: mgr.GetEventRecorder("keeper-controller"), Logger: namedLogger, Webhook: webhookv1.KeeperClusterWebhook{Log: namedLogger}, + Checker: checker, Dialer: dialer, EnablePDB: enablePDB, } diff --git a/internal/controller/keeper/sync.go b/internal/controller/keeper/sync.go index 73d4f911..efd6e974 100644 --- a/internal/controller/keeper/sync.go +++ b/internal/controller/keeper/sync.go @@ -22,6 +22,7 @@ import ( v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1" chctrl "github.com/ClickHouse/clickhouse-operator/internal/controller" ctrlutil "github.com/ClickHouse/clickhouse-operator/internal/controllerutil" + "github.com/ClickHouse/clickhouse-operator/internal/upgrade" ) type replicaState struct { @@ -80,6 +81,7 @@ type keeperReconciler struct { chctrl.ResourceManager Dialer ctrlutil.DialContextFunc + Checker *upgrade.Checker EnablePDB bool Cluster *v1.KeeperCluster @@ -93,8 +95,6 @@ type keeperReconciler struct { func (r *keeperReconciler) sync(ctx context.Context, log ctrlutil.Logger) (ctrl.Result, error) { log.Info("Enter Keeper Reconcile", "spec", r.Cluster.Spec, "status", r.Cluster.Status) - meta.RemoveStatusCondition(&r.Cluster.Status.Conditions, v1.ConditionTypeVersionUpgraded) - r.SetUnknownConditions(v1.ConditionReasonStepFailed, "Reconcile stopped before condition evaluation", []v1.ConditionType{ v1.ConditionTypeReplicaStartupSucceeded, @@ -102,6 +102,7 @@ func (r *keeperReconciler) sync(ctx context.Context, log ctrlutil.Logger) (ctrl. v1.ConditionTypeClusterSizeAligned, v1.ConditionTypeConfigurationInSync, v1.ConditionTypeVersionInSync, + v1.ConditionTypeVersionUpgraded, v1.ConditionTypeReady, v1.KeeperConditionTypeScaleAllowed, }) @@ -597,6 +598,7 @@ func (r *keeperReconciler) evaluateReplicaConditions() { r.SetCondition(chctrl.ConfigSyncCondition(nil, notUpdatedIDs, nil)) versionCond, versionEvents := keeperVersionSyncCondition(replicaVersions, len(notUpdatedIDs) > 0) r.SetCondition(versionCond, versionEvents...) + r.evaluateUpgradeCondition(replicaVersions) // Ready condition — keeper-specific logic. exists := len(r.ReplicaState) @@ -675,6 +677,21 @@ func (r *keeperReconciler) evaluateReplicaConditions() { ) } +func (r *keeperReconciler) evaluateUpgradeCondition(replicaVersions map[string]string) { + version, ok := keeperObservedVersion(replicaVersions) + if ok { + r.Cluster.Status.Version = version + } + + if r.Checker == nil || !ok { + meta.RemoveStatusCondition(r.Cluster.GetStatus().GetConditions(), v1.ConditionTypeVersionUpgraded) + return + } + + cond, event := chctrl.GetUpgradeCondition(*r.Checker, chctrl.VersionProbeResult{Version: version}, r.Cluster.Spec.UpgradeChannel) + r.SetCondition(cond, event...) +} + func keeperVersionSyncCondition(replicaVersions map[string]string, isUpdating bool) (metav1.Condition, []chctrl.EventSpec) { newCond := func(status metav1.ConditionStatus, reason v1.ConditionReason, message string) metav1.Condition { return metav1.Condition{ @@ -689,21 +706,12 @@ func keeperVersionSyncCondition(replicaVersions map[string]string, isUpdating bo return newCond(metav1.ConditionUnknown, v1.ConditionReasonVersionPending, "No Keeper replica version has been observed yet"), nil } - versions := map[string]struct{}{} - var observed []string for id, version := range replicaVersions { - versions[version] = struct{}{} observed = append(observed, fmt.Sprintf("%s: %s", id, version)) } - if len(versions) == 1 { - var version string - for _, v := range replicaVersions { - version = v - break - } - + if version, ok := keeperObservedVersion(replicaVersions); ok { return newCond(metav1.ConditionTrue, v1.ConditionReasonVersionMatch, "All observed Keeper replicas report version "+version), nil } @@ -724,6 +732,22 @@ func keeperVersionSyncCondition(replicaVersions map[string]string, isUpdating bo }} } +func keeperObservedVersion(replicaVersions map[string]string) (string, bool) { + var result string + for _, version := range replicaVersions { + if result == "" { + result = version + continue + } + + if version != result { + return "", false + } + } + + return result, result != "" +} + func (r *keeperReconciler) updateReplica(ctx context.Context, log ctrlutil.Logger, replicaID v1.KeeperReplicaID) (*ctrl.Result, error) { log = log.With("replica_id", replicaID) log.Info("updating replica") diff --git a/internal/controller/keeper/sync_test.go b/internal/controller/keeper/sync_test.go index 773aaaf0..d5f64f27 100644 --- a/internal/controller/keeper/sync_test.go +++ b/internal/controller/keeper/sync_test.go @@ -5,12 +5,14 @@ import ( "errors" "net" "reflect" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -23,6 +25,7 @@ import ( v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1" "github.com/ClickHouse/clickhouse-operator/internal/controller" util "github.com/ClickHouse/clickhouse-operator/internal/controllerutil" + "github.com/ClickHouse/clickhouse-operator/internal/upgrade" ) var _ = Describe("UpdateReplica", Ordered, func() { @@ -175,6 +178,99 @@ var _ = Describe("UpdateReplica", Ordered, func() { }) }) +var _ = Describe("Keeper version status", func() { + It("should report matching live replica versions", func() { + _, rec, cancelEvents := setupReconciler() + defer cancelEvents() + + rec.ReplicaState = map[v1.KeeperReplicaID]replicaState{ + 0: {Status: serverStatus{Version: "25.8.2.1"}}, + 1: {Status: serverStatus{Version: "25.8.2.1"}}, + } + + rec.evaluateReplicaConditions() + + Expect(rec.Cluster.Status.Version).To(Equal("25.8.2.1")) + cond := meta.FindStatusCondition(rec.Cluster.Status.Conditions, v1.ConditionTypeVersionInSync) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(v1.ConditionReasonVersionMatch)) + Expect(meta.FindStatusCondition(rec.Cluster.Status.Conditions, v1.ConditionTypeVersionUpgraded)).To(BeNil()) + }) + + It("should keep last known version when no live replica version is observed", func() { + _, rec, cancelEvents := setupReconciler() + defer cancelEvents() + + rec.Cluster.Status.Version = "25.8.2.1" + rec.ReplicaState = map[v1.KeeperReplicaID]replicaState{ + 0: {Status: serverStatus{}}, + } + + rec.evaluateReplicaConditions() + + Expect(rec.Cluster.Status.Version).To(Equal("25.8.2.1")) + cond := meta.FindStatusCondition(rec.Cluster.Status.Conditions, v1.ConditionTypeVersionInSync) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionUnknown)) + Expect(cond.Reason).To(Equal(v1.ConditionReasonVersionPending)) + }) + + It("should keep last known aggregate version when live replica versions differ", func() { + _, rec, cancelEvents := setupReconciler() + defer cancelEvents() + + rec.Cluster.Status.Version = "25.8.2.1" + rec.ReplicaState = map[v1.KeeperReplicaID]replicaState{ + 0: {Status: serverStatus{Version: "25.8.2.1"}}, + 1: {Status: serverStatus{Version: "25.8.3.1"}}, + } + + rec.evaluateReplicaConditions() + + Expect(rec.Cluster.Status.Version).To(Equal("25.8.2.1")) + cond := meta.FindStatusCondition(rec.Cluster.Status.Conditions, v1.ConditionTypeVersionInSync) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(v1.ConditionReasonVersionMismatch)) + }) + + It("should use the agreed live version for upgrade checks", func(ctx context.Context) { + log, rec, cancelEvents := setupReconciler() + defer cancelEvents() + + updater := upgrade.NewReleaseUpdater(&upgrade.StaticFetcher{Releases: map[string][]upgrade.ClickHouseVersion{ + upgrade.ChannelStable: { + {Major: 25, Minor: 8, Patch: 2, Build: 1}, + {Major: 25, Minor: 8, Patch: 3, Build: 1}, + }, + }}, time.Hour, log) + rec.Checker = upgrade.NewChecker(updater) + rec.ReplicaState = map[v1.KeeperReplicaID]replicaState{ + 0: {Status: serverStatus{Version: "25.8.2.1"}}, + 1: {Status: serverStatus{Version: "25.8.2.1"}}, + } + + updCtx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + defer GinkgoRecover() + + Expect(updater.Start(updCtx)).To(Succeed()) + }() + + Eventually(updater.GetReleasesData).ShouldNot(BeNil()) + + rec.evaluateReplicaConditions() + + cond := meta.FindStatusCondition(rec.Cluster.Status.Conditions, v1.ConditionTypeVersionUpgraded) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(v1.ConditionReasonMinorUpdateAvailable)) + }) +}) + func mustGet[T client.Object](ctx context.Context, c client.Client, key types.NamespacedName) T { var result T diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 45847759..13afffac 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -172,7 +172,7 @@ var _ = BeforeSuite(func(ctx context.Context) { upgradeChecker := upgrade.NewChecker(updater) podDialer = testutil.NewPortForwardDialer(config) - Expect(keeper.SetupWithManager(mgr, zapLogger, podDialer, true)).To(Succeed()) + Expect(keeper.SetupWithManager(mgr, zapLogger, upgradeChecker, podDialer, true)).To(Succeed()) Expect(clickhouse.SetupWithManager(mgr, zapLogger, upgradeChecker, podDialer, true)).To(Succeed()) // +kubebuilder:scaffold:builder From 3c0f791c79cd565f40c67dffa8804b04bc70fe7b Mon Sep 17 00:00:00 2001 From: Pervakov Grigorii Date: Tue, 16 Jun 2026 18:00:12 +0200 Subject: [PATCH 4/4] simplify keeper version conditions --- internal/controller/keeper/sync.go | 86 ++++++++---------------------- test/e2e/keeper_e2e_test.go | 2 +- 2 files changed, 24 insertions(+), 64 deletions(-) diff --git a/internal/controller/keeper/sync.go b/internal/controller/keeper/sync.go index efd6e974..48e52c72 100644 --- a/internal/controller/keeper/sync.go +++ b/internal/controller/keeper/sync.go @@ -7,7 +7,6 @@ import ( "math" "slices" "strconv" - "strings" "time" "gopkg.in/yaml.v2" @@ -596,9 +595,7 @@ func (r *keeperReconciler) evaluateReplicaConditions() { r.SetCondition(chctrl.ReplicaStartupCondition(errorIDs)) r.SetCondition(chctrl.HealthyCondition(notReadyIDs)) r.SetCondition(chctrl.ConfigSyncCondition(nil, notUpdatedIDs, nil)) - versionCond, versionEvents := keeperVersionSyncCondition(replicaVersions, len(notUpdatedIDs) > 0) - r.SetCondition(versionCond, versionEvents...) - r.evaluateUpgradeCondition(replicaVersions) + r.evaluateVersionConditions(len(notUpdatedIDs) > 0) // Ready condition — keeper-specific logic. exists := len(r.ReplicaState) @@ -677,75 +674,38 @@ func (r *keeperReconciler) evaluateReplicaConditions() { ) } -func (r *keeperReconciler) evaluateUpgradeCondition(replicaVersions map[string]string) { - version, ok := keeperObservedVersion(replicaVersions) - if ok { - r.Cluster.Status.Version = version - } - - if r.Checker == nil || !ok { - meta.RemoveStatusCondition(r.Cluster.GetStatus().GetConditions(), v1.ConditionTypeVersionUpgraded) - return - } +func (r *keeperReconciler) evaluateVersionConditions(isUpdating bool) { + versionByReplica := map[string]string{} + countByVersion := map[string]int{} + commonVersion := "" - cond, event := chctrl.GetUpgradeCondition(*r.Checker, chctrl.VersionProbeResult{Version: version}, r.Cluster.Spec.UpgradeChannel) - r.SetCondition(cond, event...) -} - -func keeperVersionSyncCondition(replicaVersions map[string]string, isUpdating bool) (metav1.Condition, []chctrl.EventSpec) { - newCond := func(status metav1.ConditionStatus, reason v1.ConditionReason, message string) metav1.Condition { - return metav1.Condition{ - Type: v1.ConditionTypeVersionInSync, - Status: status, - Reason: reason, - Message: message, + for i, s := range r.ReplicaState { + if s.Status.Version != "" { + versionByReplica[strconv.FormatInt(int64(i), 10)] = s.Status.Version + countByVersion[s.Status.Version]++ + commonVersion = s.Status.Version } } - if len(replicaVersions) == 0 { - return newCond(metav1.ConditionUnknown, v1.ConditionReasonVersionPending, "No Keeper replica version has been observed yet"), nil - } - - var observed []string - for id, version := range replicaVersions { - observed = append(observed, fmt.Sprintf("%s: %s", id, version)) + // Record the version only when all observed replicas agree, otherwise keep the last known one. + if len(countByVersion) == 1 { + r.Cluster.Status.Version = commonVersion } - if version, ok := keeperObservedVersion(replicaVersions); ok { - return newCond(metav1.ConditionTrue, v1.ConditionReasonVersionMatch, - "All observed Keeper replicas report version "+version), nil + probe := chctrl.VersionProbeResult{ + Version: r.Cluster.Status.Version, + Pending: len(versionByReplica) == 0, } - slices.Sort(observed) - cond := newCond(metav1.ConditionFalse, v1.ConditionReasonVersionMismatch, - "Keeper replica versions differ: "+strings.Join(observed, ", ")) - - if isUpdating { - return cond, nil - } - - return cond, []chctrl.EventSpec{{ - Type: corev1.EventTypeWarning, - Reason: v1.EventReasonVersionDiverge, - Action: v1.EventActionVersionCheck, - Message: cond.Message, - }} -} - -func keeperObservedVersion(replicaVersions map[string]string) (string, bool) { - var result string - for _, version := range replicaVersions { - if result == "" { - result = version - continue - } + cond, event := chctrl.GetVersionSyncCondition(probe, versionByReplica, isUpdating) + r.SetCondition(cond, event...) - if version != result { - return "", false - } + if r.Checker != nil { + cond, event = chctrl.GetUpgradeCondition(*r.Checker, probe, r.Cluster.Spec.UpgradeChannel) + r.SetCondition(cond, event...) + } else { + meta.RemoveStatusCondition(r.Cluster.GetStatus().GetConditions(), v1.ConditionTypeVersionUpgraded) } - - return result, result != "" } func (r *keeperReconciler) updateReplica(ctx context.Context, log ctrlutil.Logger, replicaID v1.KeeperReplicaID) (*ctrl.Result, error) { diff --git a/test/e2e/keeper_e2e_test.go b/test/e2e/keeper_e2e_test.go index 2c7069b6..399a030d 100644 --- a/test/e2e/keeper_e2e_test.go +++ b/test/e2e/keeper_e2e_test.go @@ -60,7 +60,7 @@ var _ = Describe("Keeper controller", Label("keeper"), func() { WaitKeeperUpdatedAndReady(ctx, &cr, 3*time.Minute, true) ExpectWithOffset(1, k8sClient.Get(ctx, cr.NamespacedName(), &cr)).To(Succeed()) - + Expect(cr.Status.Version).To(HavePrefix(cr.Spec.ContainerTemplate.Image.Tag)) KeeperRWChecks(ctx, &cr, &checks) }, Entry("update log level", v1.KeeperClusterSpec{Settings: v1.KeeperSettings{