feat: architect observability stack with Garage S3 backend

apps.yaml

@@ -204,7 +204,7 @@ catalog:
     vaultSecrets:
       createAuth: true
       role: openbao-secrets-operator
-      namespace: production
+      namespace: monitoring
       audiences:
         - vault
       secrets:
@@ -227,7 +227,7 @@ catalog:
     vaultSecrets:
       createAuth: true
       role: openbao-secrets-operator
-      namespace: production
+      namespace: monitoring
       audiences:
         - vault
       secrets:
@@ -281,6 +281,30 @@ catalog:
   garage-mem:
     path: bootstrap/mc/garage-mem
 
+  observability:
+    path: apps/base/observability
+    annotations:
+      argocd.argoproj.io/sync-wave: "20"
+    vaultSecrets:
+      createAuth: true
+      role: openbao-secrets-operator
+      namespace: production
+      audiences:
+        - vault
+      secrets:
+        - name: thanos-objstore-secret
+          mount: k8s
+          path: "observability/objstore"
+          destination: "thanos-objstore-secret"
+        - name: loki-bucket-secret
+          mount: k8s
+          path: "observability/loki-bucket"
+          destination: "loki-bucket-secret"
+    syncPolicy:
+      syncOptions:
+        - CreateNamespace=true
+        - ServerSideApply=true
+
   prometheus-operator-crds:
     annotations:
       argocd.argoproj.io/sync-wave: "10"

apps/base/observability/Chart.yaml

@@ -0,0 +1,13 @@
+apiVersion: v2
+name: observability
+description: Observability stack (Prometheus, Thanos, Loki) backed by Garage S3
+type: application
+version: 0.1.0
+appVersion: "1.0.0"
+dependencies:
+  - name: kube-prometheus-stack
+    version: "61.3.2"
+    repository: "https://prometheus-community.github.io/helm-charts"
+  - name: loki
+    version: "6.6.3"
+    repository: "https://grafana.github.io/helm-charts"

apps/base/observability/values.yaml

@@ -0,0 +1,79 @@
+loki:
+  deploymentMode: SingleBinary
+  fullnameOverride: loki
+  loki:
+    auth_enabled: false
+    common:
+      replication_factor: 1
+      storage:
+        s3:
+          endpoint: http://garage.garage.svc.clusterset.local:3900
+          bucketnames: loki-chunks
+          region: us-east-1
+          insecure: true
+          s3forcepathstyle: true
+          access_key_id: "${S3_ACCESS_KEY}"
+          secret_access_key: "${S3_SECRET_KEY}"
+    schema_config:
+      configs:
+        - from: "2024-04-01"
+          store: tsdb
+          object_store: s3
+          schema: v13
+          index:
+            prefix: index_
+            period: 24h
+    ingester:
+      chunk_target_size: 1572864
+      max_chunk_age: 2h
+      chunk_idle_period: 1h
+
+  # Disable minio if enabled by default
+  minio:
+    enabled: false
+
+  # Service Account for IAM (if used) or just ensure it runs
+  serviceAccount:
+    create: true
+
+  singleBinary:
+    extraEnv:
+      - name: S3_ACCESS_KEY
+        valueFrom:
+          secretKeyRef:
+            name: loki-bucket-secret
+            key: access_key
+      - name: S3_SECRET_KEY
+        valueFrom:
+          secretKeyRef:
+            name: loki-bucket-secret
+            key: secret_key
+
+kube-prometheus-stack:
+  crds:
+    enabled: false
+  grafana:
+    enabled: true
+    additionalDataSources:
+      - name: Loki
+        type: loki
+        url: http://loki.monitoring.svc.cluster.local:3100
+        access: proxy
+        jsonData:
+          maxLines: 1000
+
+  prometheus:
+    prometheusSpec:
+      retention: 6h
+      enableAdminAPI: true
+      storageSpec:
+        volumeClaimTemplate:
+          spec:
+            accessModes: ["ReadWriteOnce"]
+            resources:
+              requests:
+                storage: 50Gi
+      thanos:
+        objectStorageConfig:
+          name: thanos-objstore-secret
+          key: objstore.yml

clusters/cc.yaml

@@ -57,6 +57,8 @@ apps:
     namespace: monitoring
   - name: grafana-alloy
     namespace: monitoring
+  - name: observability
+    namespace: monitoring
   - name: karmada-operator
     namespace: karmada-system
   - name: karmada-instance

clusters/mc.yaml

@@ -98,6 +98,20 @@ apps:
             extraVolumeMounts:
               - name: snapshots
                 mountPath: /snapshots
+            ingress:
+              s3:
+                api:
+                  enabled: true
+                  ingressClassName: tailscale
+                  hosts:
+                    - host: s3.moonwake.io
+                      paths:
+                        - path: /
+                          pathType: Prefix
+                  tls:
+                    - secretName: garage-ingress-cert
+                      hosts:
+                        - s3.moonwake.io
         - name: KUSTOMIZE_PATCH
           value: |
             - op: add