From 554dbead38baff041ab724f4101db9d7dc773275 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Fri, 12 Jun 2026 15:19:50 +0300 Subject: [PATCH 1/4] Document SeaweedFS S3 secrets and restore Helm-driven OpenBao access keys. OpenBao paths stay legacy minio-* for compatibility; comments and values.yaml clarify SeaweedFS. Static API/console access key names use Values again so install-time overrides work without raw Helm reaching manage-secrets. --- docs/manual_helm_install/components/s3.md | 9 +++++---- .../templates/openbao-secret-definitions.yaml | 15 ++++++++------- sources/openbao-config/0.1.0/values.yaml | 6 +++--- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/manual_helm_install/components/s3.md b/docs/manual_helm_install/components/s3.md index 3ed15de0..df893a34 100644 --- a/docs/manual_helm_install/components/s3.md +++ b/docs/manual_helm_install/components/s3.md @@ -14,10 +14,11 @@ authenticate with are created via ExternalSecrets from OpenBao (in full cluster-forge deployments) or via `kubectl create secret generic` (in manual installations). No static `secrets-*.yaml` file is required. -**Note on credential compatibility:** For backward compatibility with MinIO, -the secret keys in OpenBao still reference `minio-*` paths (e.g., -`secrets/data/minio-api-access-key`). These same credentials are used by -SeaweedFS via the `seaweedfs-s3-config` secret. +**Note on OpenBao path names:** In-cluster object storage is **SeaweedFS** +(filer S3). OpenBao KV entries still use the historical `secrets/minio-*` paths +(e.g. `secrets/data/minio-api-access-key`) so existing ExternalSecrets and +charts (AIRM, Keycloak, etc.) keep working unchanged. The SeaweedFS filer reads +those values via the `seaweedfs-s3-config` Secret. These instructions assume you have the cluster-forge sources available locally and that `SOURCES_DIR` points at the `sources/` directory: diff --git a/sources/openbao-config/0.1.0/templates/openbao-secret-definitions.yaml b/sources/openbao-config/0.1.0/templates/openbao-secret-definitions.yaml index 7c3ceae8..380dac7d 100644 --- a/sources/openbao-config/0.1.0/templates/openbao-secret-definitions.yaml +++ b/sources/openbao-config/0.1.0/templates/openbao-secret-definitions.yaml @@ -83,21 +83,22 @@ data: secrets/keycloak-cnpg-superuser-password|random||16 # ============================================================================= - # MINIO SECRETS + # SEAWEEDFS S3 (filer). OpenBao paths still use legacy "minio-*" names for + # compatibility with ExternalSecrets, AIRM, and Keycloak references. # ============================================================================= - # MinIO API access credentials - secrets/minio-api-access-key|static|api-default-user|0 + # Filer S3 API access (SeaweedFS s3.json ApiUser); override via --set minio.apiAccessKey=... + secrets/minio-api-access-key|static|{{ .Values.minio.apiAccessKey }}|0 secrets/minio-api-secret-key|random||16 - # MinIO console access credentials - secrets/minio-console-access-key|static|default-user|0 + # Filer S3 console identities (SeaweedFS s3.json Console); override via --set minio.consoleAccessKey=... + secrets/minio-console-access-key|static|{{ .Values.minio.consoleAccessKey }}|0 secrets/minio-console-secret-key|random||16 - # MinIO root admin password + # SeaweedFS admin UI password (ExternalSecret seaweed-admin-es-secret) secrets/minio-root-password|random||16 - # MinIO additional secrets + # OIDC / Keycloak client material for object-store console flows (legacy names) secrets/minio-client-secret|random||16 secrets/minio-openid-url|static|https://kc.{{ .Values.domain }}/realms/airm/.well-known/openid-configuration|0 diff --git a/sources/openbao-config/0.1.0/values.yaml b/sources/openbao-config/0.1.0/values.yaml index c7529bd6..db64f87f 100644 --- a/sources/openbao-config/0.1.0/values.yaml +++ b/sources/openbao-config/0.1.0/values.yaml @@ -1,8 +1,8 @@ domain: # to be filled by cluster-forge app +# In-cluster object storage is SeaweedFS (filer S3). OpenBao KV paths remain +# `secrets/minio-*` for backward compatibility. Static access key names below +# are rendered into openbao-secret-definitions (override with --set minio.*). minio: - # Seed values for MinIO access keys stored in OpenBao. Override at install - # time (`--set minio.apiAccessKey=...`) when integrating with an - # externally-provisioned MinIO whose existing keys differ from these defaults. apiAccessKey: api-default-user consoleAccessKey: default-user \ No newline at end of file From 9db4abd17c9767424ffdcc2193def508ffb2655a Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Fri, 12 Jun 2026 15:25:49 +0300 Subject: [PATCH 2/4] Tune SeaweedFS ExternalSecrets and avoid Argo PVC sync deadlock. PreSync hooks, faster refresh, and comments on secrets. Remove sync-wave from the Seaweed CR so PVCs are not gated behind WaitForFirstConsumer when Argo evaluates application health. --- .../templates/seaweedfs-es-admin.yaml | 4 +- .../templates/seaweedfs-es-s3-config.yaml | 8 +- .../templates/seaweedfs-seaweed.yaml | 78 +++++++++---------- 3 files changed, 49 insertions(+), 41 deletions(-) diff --git a/sources/seaweedfs-config/templates/seaweedfs-es-admin.yaml b/sources/seaweedfs-config/templates/seaweedfs-es-admin.yaml index ff260b51..9e1a4e4b 100644 --- a/sources/seaweedfs-config/templates/seaweedfs-es-admin.yaml +++ b/sources/seaweedfs-config/templates/seaweedfs-es-admin.yaml @@ -5,15 +5,17 @@ metadata: namespace: seaweedfs-instance annotations: argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/sync-wave: "-1" spec: secretStoreRef: kind: ClusterSecretStore name: openbao-secret-store - refreshInterval: 5m + refreshInterval: 1m target: name: seaweedfs-admin-secret data: - secretKey: admin-password remoteRef: + # Legacy OpenBao path name; backs SeaweedFS admin UI, not MinIO. key: secrets/data/minio-root-password property: value diff --git a/sources/seaweedfs-config/templates/seaweedfs-es-s3-config.yaml b/sources/seaweedfs-config/templates/seaweedfs-es-s3-config.yaml index 79f2561d..86b86737 100644 --- a/sources/seaweedfs-config/templates/seaweedfs-es-s3-config.yaml +++ b/sources/seaweedfs-config/templates/seaweedfs-es-s3-config.yaml @@ -4,12 +4,17 @@ metadata: name: seaweedfs-es-s3-config namespace: seaweedfs-instance annotations: + # PreSync + sync-wave: ensure OpenBao-backed Secret exists (and Argo marks this + # resource Healthy per ExternalSecret health rules) before the Seaweed CR syncs. argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/sync-wave: "-2" spec: secretStoreRef: kind: ClusterSecretStore name: openbao-secret-store - refreshInterval: 5m + # Shorter interval so the filer picks up OpenBao updates soon after the secret + # manager reconciles static keys (default openbao cron is every 5m). + refreshInterval: 1m target: name: seaweedfs-s3-config template: @@ -43,6 +48,7 @@ spec: } ] } + # OpenBao paths use legacy "minio-*" keys (shared with AIRM / Keycloak); values feed SeaweedFS only. data: - secretKey: API_ACCESS_KEY remoteRef: diff --git a/sources/seaweedfs-config/templates/seaweedfs-seaweed.yaml b/sources/seaweedfs-config/templates/seaweedfs-seaweed.yaml index e207a6bd..30a6213f 100644 --- a/sources/seaweedfs-config/templates/seaweedfs-seaweed.yaml +++ b/sources/seaweedfs-config/templates/seaweedfs-seaweed.yaml @@ -1,40 +1,40 @@ -{{- with .Values.seaweed }} -apiVersion: seaweed.seaweedfs.com/v1 -kind: Seaweed -metadata: - name: seaweed - namespace: seaweedfs-instance -spec: - image: chrislusf/seaweedfs:latest - volumeServerDiskCount: 1 - hostSuffix: plat-dev-1.silogen.ai - master: - replicas: {{ .master.replicas }} - volumeSizeLimitMB: 1024 - volume: - replicas: {{ .volume.replicas }} - storageClassName: {{ .storageClassName }} - requests: - storage: {{ .volume.requests.storage }} - # Topology configuration for rack/datacenter-aware placement - rack: "rack1" - dataCenter: "dc1" - filer: - replicas: {{ .filer.replicas }} - volumes: - - name: s3-metadata - persistentVolumeClaim: - claimName: pvc-seaweed-s3-metadata - volumeMounts: - - mountPath: /data/ - name: s3-metadata - s3: - enabled: true - configSecret: - name: seaweedfs-s3-config - key: s3.json - config: | - [leveldb2] - enabled = true - dir = "/data/filerldb2" +{{- with .Values.seaweed }} +apiVersion: seaweed.seaweedfs.com/v1 +kind: Seaweed +metadata: + name: seaweed + namespace: seaweedfs-instance +spec: + image: chrislusf/seaweedfs:latest + volumeServerDiskCount: 1 + hostSuffix: plat-dev-1.silogen.ai + master: + replicas: {{ .master.replicas }} + volumeSizeLimitMB: 1024 + volume: + replicas: {{ .volume.replicas }} + storageClassName: {{ .storageClassName }} + requests: + storage: {{ .volume.requests.storage }} + # Topology configuration for rack/datacenter-aware placement + rack: "rack1" + dataCenter: "dc1" + filer: + replicas: {{ .filer.replicas }} + volumes: + - name: s3-metadata + persistentVolumeClaim: + claimName: pvc-seaweed-s3-metadata + volumeMounts: + - mountPath: /data/ + name: s3-metadata + s3: + enabled: true + configSecret: + name: seaweedfs-s3-config + key: s3.json + config: | + [leveldb2] + enabled = true + dir = "/data/filerldb2" {{- end}} \ No newline at end of file From 8cbdebd9f0454a55a7f39b727914c39554ef992e Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Fri, 12 Jun 2026 15:25:52 +0300 Subject: [PATCH 3/4] Fix root Argo apps for Kyverno storage policies and SeaweedFS ordering. Use jqPathExpressions for kyverno-policies-storage-local-path, set seaweedfs-config syncWave, and remove duplicate or malformed app entries from medium/small overlays. --- root/values.yaml | 17 ++++++++++++++++- root/values_medium.yaml | 11 ----------- root/values_small.yaml | 12 ------------ 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/root/values.yaml b/root/values.yaml index 925fcd94..a5a3dff1 100644 --- a/root/values.yaml +++ b/root/values.yaml @@ -686,6 +686,20 @@ apps: namespace: kyverno path: kyverno-policies/base syncWave: -20 + # Used only when enabledApps includes this name (small/medium). Kyverno may + # default fields on ClusterPolicy; ignore to avoid perpetual OutOfSync. + kyverno-policies-storage-local-path: + namespace: kyverno + path: kyverno-policies/storage-local-path + syncWave: -20 + ignoreDifferences: + - group: kyverno.io + kind: ClusterPolicy + jqPathExpressions: + # jsonPointers do not support wildcards; Kyverno sets these on rules. + - .spec.rules[].skipBackgroundRequests + - .spec.rules[].validate.allowExistingViolations + - .status metallb: namespace: default path: metallb/v0.15.2 @@ -722,7 +736,8 @@ apps: path: seaweedfs-config namespace: seaweedfs-instance valuesFile: values.yaml - syncWave: 0 + # After seaweedfs-operator (0): CRD/webhook ready before Seaweed + ExternalSecrets. + syncWave: 10 openbao: ignoreDifferences: - group: "apps" diff --git a/root/values_medium.yaml b/root/values_medium.yaml index 7989c02b..427542df 100644 --- a/root/values_medium.yaml +++ b/root/values_medium.yaml @@ -52,17 +52,6 @@ enabledApps: - rabbitmq apps: - # Modular Kyverno policy applications (only the storage-local-path addition) - kyverno-policies-storage-local-path: - namespace: kyverno - path: kyverno-policies/storage-local-path - syncWave: -20 - ignoreDifferences: - - group: kyverno.io - kind: ClusterPolicy - jsonPointers: - - /spec/rules/*/skipBackgroundRequests - - /spec/rules/*/validate/allowExistingViolations aiwb-infra-cnpg: valuesObject: instances: 1 diff --git a/root/values_small.yaml b/root/values_small.yaml index 818ffd24..99f6fa57 100644 --- a/root/values_small.yaml +++ b/root/values_small.yaml @@ -114,18 +114,6 @@ apps: requests: cpu: "250m" memory: "512Mi" - kyverno-policies-storage-local-path: - ignoreDifferences: [] - namespace: kyverno - path: kyverno-policies/storage-local-path - source: clusterForge - syncOptions: - - CreateNamespace=true - syncWave: - - group: kyverno.io - kind: ClusterPolicy - wave: 26 # Deploy after base policies - seaweedfs-config: valuesObject: seaweed: From 6237dfdeefbd82af1e326337fedbf1131743bf78 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Fri, 12 Jun 2026 15:25:53 +0300 Subject: [PATCH 4/4] Order Keycloak CNPG secrets and workload for Argo sync waves. PreSync on CNPG ExternalSecrets and sync-wave annotations on Cluster and Deployment so secrets exist before the database and app roll out. --- .../keycloak-config/templates/keycloak-cluster.yaml | 13 +++++++++++++ .../es-keycloak-cnpg-superuser-credentials.yaml | 4 ++++ .../es-keycloak-cnpg-user-credentials.yaml | 3 +++ sources/keycloak-old/templates/keycloak-cnpg.yaml | 3 +++ .../keycloak-old/templates/keycloak-deployment.yaml | 3 +++ 5 files changed, 26 insertions(+) diff --git a/sources/keycloak-config/templates/keycloak-cluster.yaml b/sources/keycloak-config/templates/keycloak-cluster.yaml index 0421fda3..60ac23e8 100644 --- a/sources/keycloak-config/templates/keycloak-cluster.yaml +++ b/sources/keycloak-config/templates/keycloak-cluster.yaml @@ -1,10 +1,17 @@ --- +# CNPG bootstrap reads keycloak-cnpg-user (and superuser) Secrets at initdb time. +# Run these ExternalSecrets as PreSync hooks so OpenBao-backed Secrets exist and are +# Healthy before the Cluster resource applies (avoids password mismatch vs Keycloak). apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: name: keycloak-cnpg-superuser-credentials namespace: keycloak + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/sync-wave: "-3" spec: + refreshInterval: 1h secretStoreRef: kind: ClusterSecretStore name: openbao-secret-store @@ -27,6 +34,9 @@ kind: ExternalSecret metadata: name: keycloak-cnpg-user-credentials namespace: keycloak + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/sync-wave: "-2" spec: data: - remoteRef: @@ -50,6 +60,9 @@ kind: Cluster metadata: name: keycloak-cnpg namespace: keycloak + annotations: + # After PreSync CNPG ExternalSecrets; before HTTPRoute/other wave-0 extras if any. + argocd.argoproj.io/sync-wave: "1" spec: affinity: enablePodAntiAffinity: true diff --git a/sources/keycloak-old/templates/es-keycloak-cnpg-superuser-credentials.yaml b/sources/keycloak-old/templates/es-keycloak-cnpg-superuser-credentials.yaml index 5607cef9..c0ff98ed 100644 --- a/sources/keycloak-old/templates/es-keycloak-cnpg-superuser-credentials.yaml +++ b/sources/keycloak-old/templates/es-keycloak-cnpg-superuser-credentials.yaml @@ -5,7 +5,11 @@ kind: ExternalSecret metadata: name: keycloak-cnpg-superuser-credentials namespace: keycloak + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/sync-wave: "-3" spec: + refreshInterval: 1h secretStoreRef: kind: ClusterSecretStore name: openbao-secret-store diff --git a/sources/keycloak-old/templates/es-keycloak-cnpg-user-credentials.yaml b/sources/keycloak-old/templates/es-keycloak-cnpg-user-credentials.yaml index 2ec99573..952af40b 100644 --- a/sources/keycloak-old/templates/es-keycloak-cnpg-user-credentials.yaml +++ b/sources/keycloak-old/templates/es-keycloak-cnpg-user-credentials.yaml @@ -5,6 +5,9 @@ kind: ExternalSecret metadata: name: {{ .Values.postgresql.userSecretName }}-credentials namespace: keycloak + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/sync-wave: "-2" spec: data: - remoteRef: diff --git a/sources/keycloak-old/templates/keycloak-cnpg.yaml b/sources/keycloak-old/templates/keycloak-cnpg.yaml index de9f927b..d1f4d11a 100644 --- a/sources/keycloak-old/templates/keycloak-cnpg.yaml +++ b/sources/keycloak-old/templates/keycloak-cnpg.yaml @@ -5,6 +5,9 @@ kind: Cluster metadata: name: keycloak-cnpg namespace: keycloak + annotations: + # After PreSync CNPG ExternalSecrets; before Keycloak Deployment (wave 5) so initdb uses final secrets. + argocd.argoproj.io/sync-wave: "1" spec: affinity: enablePodAntiAffinity: true diff --git a/sources/keycloak-old/templates/keycloak-deployment.yaml b/sources/keycloak-old/templates/keycloak-deployment.yaml index 69593e95..04f1760e 100644 --- a/sources/keycloak-old/templates/keycloak-deployment.yaml +++ b/sources/keycloak-old/templates/keycloak-deployment.yaml @@ -6,6 +6,9 @@ metadata: app: keycloak name: keycloak namespace: keycloak + annotations: + # After CNPG Cluster (wave 1) so JDBC connects once Postgres is up with bootstrap secrets. + argocd.argoproj.io/sync-wave: "5" spec: replicas: 1 selector: