fix: track users reload separetely of server config

internal/controller/clickhouse/commands.go

@@ -88,8 +88,13 @@ func (cmd *commander) Close() {
 }
 
 type replicaProbe struct {
-	Version              string
-	ReloadConfigRevision string
+	Version                   string
+	ReloadConfigRevision      string
+	UsersReloadConfigRevision string
+}
+
+func (p replicaProbe) Reloaded(rev string) bool {
+	return p.ReloadConfigRevision == rev && p.UsersReloadConfigRevision == rev
 }
 
 // Probe reads the replica server version and the latest applied reload-safe config revision.
@@ -103,11 +108,13 @@ func (cmd *commander) Probe(ctx context.Context, id v1.ClickHouseReplicaID) (rep
 
 	row := conn.QueryRow(ctx,
 		"SELECT version(),"+
-			" ifNull((SELECT collection[?] FROM system.named_collections WHERE name = ?), '')"+
+			" ifNull((SELECT collection[?] FROM system.named_collections WHERE name = ?), ''),"+
+			" ifNull((SELECT trim(BOTH '''' FROM value) FROM system.settings_profile_elements WHERE profile_name = ? AND setting_name = ?), '')"+
 			" SETTINGS format_display_secrets_in_show_and_select=1",
 		OperatorConfigRevisionField, OperatorNamedCollectionName,
+		OperatorSettingsProfileName, OperatorReloadMarkerSettingName,
 	)
-	if err := row.Scan(&probe.Version, &probe.ReloadConfigRevision); err != nil {
+	if err := row.Scan(&probe.Version, &probe.ReloadConfigRevision, &probe.UsersReloadConfigRevision); err != nil {
 		return replicaProbe{}, fmt.Errorf("probe replica %s: %w", id, err)
 	}
 

internal/controller/clickhouse/commands_test.go

@@ -75,6 +75,7 @@ named_collections:
   __operator:
     config_revision: %s
 display_secrets_in_show_and_select: true
+custom_settings_prefixes: custom_
 `, replica, keeperHostname, keeper.PortNative, strings.Join(replicas, ","), testConfigRevision))
 }
 
@@ -84,11 +85,15 @@ func generateUsersConfig() *strings.Reader {
     no_password
   operator:
     password: %s
-    profile: default
+    profile: __operator
     quota: default
     grants:
       query: GRANT ALL ON *.*
-`, testPassword))
+profiles:
+  __operator:
+    profile: default
+    custom_operator_reload_revision: "'%s'"
+`, testPassword, testConfigRevision))
 }
 
 var _ = Describe("commander", Ordered, Label("integration"), func() {
@@ -229,6 +234,7 @@ var _ = Describe("commander", Ordered, Label("integration"), func() {
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(probe.Version).To(MatchRegexp(`^\d+\.\d+\.\d+\.\d+$`))
 				g.Expect(probe.ReloadConfigRevision).To(Equal(testConfigRevision))
+				g.Expect(probe.UsersReloadConfigRevision).To(Equal(testConfigRevision))
 			}, "1m", "100ms").Should(Succeed())
 		}
 	})

internal/controller/clickhouse/config.go

@@ -263,8 +263,10 @@ type clusterConfigParams struct {
 }
 
 const (
-	OperatorNamedCollectionName = "__operator"
-	OperatorConfigRevisionField = "config_revision"
+	OperatorNamedCollectionName     = "__operator"
+	OperatorConfigRevisionField     = "config_revision"
+	OperatorSettingsProfileName     = "__operator"
+	OperatorReloadMarkerSettingName = "custom_operator_reload_revision"
 )
 
 type macro struct {
@@ -404,6 +406,8 @@ type userConfigParams struct {
 	DefaultProfileName       string
 	OperatorUserName         string
 	OperatorUserPasswordHash string
+	OperatorProfileName      string
+	ReloadConfigRevision     string
 }
 
 func userConfigGenerator(tmpl *template.Template, r *clickhouseReconciler, _ v1.ClickHouseReplicaID) (string, error) {
@@ -422,6 +426,8 @@ func userConfigGenerator(tmpl *template.Template, r *clickhouseReconciler, _ v1.
 		DefaultProfileName:       DefaultProfileName,
 		OperatorUserName:         OperatorManagementUsername,
 		OperatorUserPasswordHash: controllerutil.Sha256Hash(r.secret.Data[SecretKeyManagementPassword]),
+		OperatorProfileName:      OperatorSettingsProfileName,
+		ReloadConfigRevision:     r.revs.ReloadConfigRevision,
 	}
 
 	builder := strings.Builder{}

internal/controller/clickhouse/sync.go

@@ -70,7 +70,7 @@ func (r replicaState) UpdateStage(rev chctrl.RevisionState) chctrl.ReplicaUpdate
 		return chctrl.StageHasDiff
 	}
 
-	if !r.Ready() || r.ReloadConfigRevision != rev.ReloadConfigRevision {
+	if !r.Ready() || !r.Reloaded(rev.ReloadConfigRevision) {
 		return chctrl.StageNotReadyUpToDate
 	}
 
@@ -462,7 +462,7 @@ func (r *clickhouseReconciler) reconcileActiveReplicaStatus(ctx context.Context,
 				log.Debug("failed to probe replica", "replica_id", id, "error", err)
 			}
 
-			if cfg, ok := configMaps[id]; ok && probe.ReloadConfigRevision != cfg.Annotations[ctrlutil.AnnotationReloadableConfigHash] {
+			if cfg, ok := configMaps[id]; ok && !probe.Reloaded(cfg.Annotations[ctrlutil.AnnotationReloadableConfigHash]) {
 				if reloadErr = r.commander.ReloadConfig(ctx, id); reloadErr != nil {
 					log.Debug("replica config reload failed", "replica_id", id, "error", reloadErr)
 				} else if probe, err = r.commander.Probe(ctx, id); err != nil {
@@ -600,7 +600,7 @@ func (r *clickhouseReconciler) reconcileClusterRevisions(ctx context.Context, lo
 				reloadErr = append(reloadErr, id.String())
 			}
 
-			if replica.ReloadConfigRevision != r.revs.ReloadConfigRevision {
+			if !replica.Reloaded(r.revs.ReloadConfigRevision) {
 				notReloaded = append(notReloaded, id.String())
 			}
 		}

internal/controller/clickhouse/templates/base.yaml.tmpl

@@ -18,3 +18,4 @@ openSSL:
 
 {{- /* Special settings, required for default config */}}
 display_secrets_in_show_and_select: true
+custom_settings_prefixes: custom_

internal/controller/clickhouse/templates/users.yaml.tmpl

@@ -11,13 +11,16 @@ users:
     grants:
       - query: "GRANT ALL ON *.* WITH GRANT OPTION"
   {{ .OperatorUserName }}:
-    profile: {{ .DefaultProfileName }}
+    profile: {{ .OperatorProfileName }}
     quota: default
     password_sha256_hex: {{ .OperatorUserPasswordHash }}
     grants: {{/* TODO restrict */}}
       - query: "GRANT ALL ON *.*"
 profiles:
   {{ .DefaultProfileName }}:
     log_queries: 1
+  {{ .OperatorProfileName }}:
+    profile: {{ .DefaultProfileName }}
+    custom_operator_reload_revision: "'{{ .ReloadConfigRevision }}'"
 quotas:
   default: