Cloudfleet

.pre-commit-config.yaml

@@ -0,0 +1,23 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.6.9
+  hooks:
+  - id: ruff-format
+    name: ruff format yaml
+    files: \.(yaml|yml)$
+    args: [ --config=line-length=120 ]
+  - id: ruff
+    name: ruff lint yaml
+    files: \.(yaml|yml)$
+    args: [ --fix, --config=line-length=120 ]
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.6.0
+  hooks:
+  - id: trailing-whitespace
+    files: \.(yaml|yml)$
+  - id: end-of-file-fixer
+    files: \.(yaml|yml)$
+  - id: check-yaml
+    args: [ --multi, --unsafe ]
+  - id: check-added-large-files

argo/apps/100-traefik/traefik.yaml → argo/apps/100-traefik/application.yaml

@@ -5,14 +5,14 @@ metadata:
   namespace: argocd
   # Add finalizer to ensure that Helm release is deleted before the app
   finalizers:
-    - argocd.argoproj.io/resources-finalizer # Use domain-qualified finalizer
+  - argocd.argoproj.io/resources-finalizer # Use domain-qualified finalizer
 spec:
   project: default
   source:
     # Source is the Git repository containing this Application manifest and the wrapper chart
     repoURL: https://github.com/datamindedbe/playground-data-platform-stack.git
     path: argo/apps/100-traefik # Path to the wrapper chart directory within the Git repo
-    targetRevision: HEAD # Or your specific branch/tag
+    targetRevision: cloudfleet # Or your specific branch/tag
 
     # Helm configuration for the wrapper chart
     helm:
@@ -29,4 +29,4 @@ spec:
       prune: true
       selfHeal: true
     syncOptions:
-      - CreateNamespace=true # Ensure the traefik namespace is created
+    - CreateNamespace=true # Ensure the traefik namespace is created

argo/apps/100-traefik/values.yaml

@@ -8,9 +8,31 @@ traefik:
       enabled: true
       # Use the default entrypoint for the dashboard
       entryPoints:
-        - websecure
+      - websecure
       # Define the matching rule for the dashboard route
       matchRule: Host(`traefik.localhost`)
+  ports:
+    web:
+      transport:
+        respondingTimeouts:
+          readTimeout: 3600s
+          writeTimeout: 3600s
+          idleTimeout: 3600s
+        keepAliveMaxTime: 3600s
+        keepAliveMaxRequests: 1000
+    websecure:
+      transport:
+        respondingTimeouts:
+          readTimeout: 3600s
+          writeTimeout: 3600s
+          idleTimeout: 3600s
+        keepAliveMaxTime: 3600s
+        keepAliveMaxRequests: 1000
+  logs:
+    general:
+      level: DEBUG
+    access:
+      enabled: true
 
   # Enable Kubernetes providers
   providers:

argo/apps/101-hetzner-csi/Chart.yaml

@@ -0,0 +1,13 @@
+apiVersion: v2
+name: hetzner-csi-wrapper
+description: A wrapper Helm chart to deploy Hetzner CSI with custom values.
+version: 0.1.0 # Version of this wrapper chart
+appVersion: "v25.0.0" # Corresponds to the Traefik chart version we depend on
+
+dependencies:
+- name: hcloud-csi
+  version: "v2.15.0" # The version of the Traefik chart to use
+  repository: https://charts.hetzner.cloud # The repository of the dependency
+  # We need to map the values from our local values.yaml to the subchart.
+  # By default, values under a key matching the dependency name are passed.
+  # So, values for 'traefik' in our values.yaml will go to the traefik subchart.

argo/apps/101-hetzner-csi/application.yaml

@@ -0,0 +1,31 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: hetzner-csi
+  namespace: argocd
+  # Add finalizer to ensure that Helm release is deleted before the app
+  finalizers:
+  - argocd.argoproj.io/resources-finalizer # Use domain-qualified finalizer
+spec:
+  project: default
+  source:
+    # Source is the Git repository containing this Application manifest and the wrapper chart
+    repoURL: https://github.com/datamindedbe/playground-data-platform-stack.git
+    path: argo/apps/101-hetzner-csi # Path to the wrapper chart directory within the Git repo
+    targetRevision: cloudfleet # Or your specific branch/tag
+
+    # Helm configuration for the wrapper chart
+    helm:
+      # releaseName is optional here, defaults based on app name
+      releaseName: hetzner-csi
+      # Values file is implicitly values.yaml within the source path
+      # valueFiles: # Not needed if using default values.yaml
+      #  - values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system # Deploy Traefik into its own namespace
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions: []

argo/apps/101-hetzner-csi/templates/hcloud-secret.yaml

@@ -0,0 +1,18 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hcloud-token
+  namespace: kube-system
+spec:
+  refreshInterval: 300s
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  target:
+    name: hcloud
+    creationPolicy: Owner
+  data:
+  - secretKey: token
+    remoteRef:
+      key: cloud/hetzner
+      property: roottoken

argo/apps/101-hetzner-csi/values.yaml

@@ -0,0 +1,20 @@
+# Values for the traefik-wrapper chart
+
+# Values passed to the 'traefik' subchart (dependency)
+traefik:
+  # Enable dashboard access (consider security implications for production)
+  ingressRoute:
+    dashboard:
+      enabled: true
+      # Use the default entrypoint for the dashboard
+      entryPoints:
+        - websecure
+      # Define the matching rule for the dashboard route
+      matchRule: Host(`traefik.localhost`)
+
+  # Enable Kubernetes providers
+  providers:
+    kubernetesCRD:
+      enabled: true # Enable CRD provider (for IngressRoute, ServersTransport, etc.)
+    kubernetesIngress:
+      enabled: true

argo/apps/102-cert-manager/Chart.yaml

@@ -0,0 +1,10 @@
+apiVersion: v2
+name: cert-manager-wrapper
+description: A wrapper to deploy cert-manager with custom values.
+version: 0.1.0 # Version of this wrapper chart
+appVersion: "0.18.0" # Corresponds to the chart version we depend on (check for latest stable)
+
+dependencies:
+- name: cert-manager
+  version: "1.18.0" # Specify the Airflow chart version
+  repository: https://charts.jetstack.io # Official Jetstack repository for cert-manager

argo/apps/102-cert-manager/application.yaml

@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cert-manager
+  namespace: argocd
+  finalizers:
+  - argocd.argoproj.io/resources-finalizer # Use domain-qualified finalizer
+spec:
+  project: default
+  source:
+    # Source is the Git repository containing this Application manifest and the wrapper chart
+    repoURL: https://github.com/datamindedbe/playground-data-platform-stack.git # Ensure this is your repo URL
+    path: argo/apps/102-cert-manager # Path to the wrapper chart directory within the Git repo
+    targetRevision: cloudfleet # Or your specific branch/tag
+
+    # Helm configuration for the wrapper chart
+    helm:
+      releaseName: cert-manager
+      # Values file is implicitly values.yaml within the source path
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: operators # Deploy Airflow into the services namespace
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true # Ensure the services namespace is created

argo/apps/102-cert-manager/templates/issuer-prd.yaml

@@ -0,0 +1,15 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    # server: https://acme-staging-v02.api.letsencrypt.org/directory
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: jonathan.merlevede@dataminded.com
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    solvers:
+    - http01:
+        ingress:
+          class: traefik

argo/apps/102-cert-manager/templates/issuer-staging.yaml

@@ -0,0 +1,14 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    email: jonathan.merlevede@dataminded.com
+    privateKeySecretRef:
+      name: letsencrypt-staging
+    solvers:
+    - http01:
+        ingress:
+          class: traefik

argo/apps/102-cert-manager/values.yaml

@@ -0,0 +1,6 @@
+cert-manager:
+  crds:
+    enabled: true # Enable CRDs installation
+    keep: true # Keep CRDs after uninstalling the chart
+  prometheus:
+    enabled: false

argo/apps/150-argocd-ingress/Chart.yaml

@@ -0,0 +1,9 @@
+apiVersion: v2
+name: argocd-ingress
+description: A Helm chart for ArgoCD ingress configuration
+type: application
+version: 0.1.0
+appVersion: "1.0.0"
+maintainers:
+- name: Dataminded
+  email: jonathan.merlevede@dataminded.com

argo/apps/150-argocd-ingress/application.yaml

@@ -0,0 +1,22 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-ingress
+  namespace: argocd
+  finalizers:
+  - argocd.argoproj.io/resources-finalizer
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/datamindedbe/playground-data-platform-stack.git
+    path: argo/apps/150-argocd-ingress
+    targetRevision: cloudfleet
+    helm:
+      releaseName: argocd-ingress
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

argo/apps/150-argocd-ingress/templates/certificate.yaml

@@ -0,0 +1,11 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: argocd-cert
+spec:
+  secretName: argocd-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  dnsNames:
+    - argocd.cloudfleet.platform.5ha.re

argo/apps/150-argocd-ingress/templates/ingress.yaml

@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: argocd-server-ingress
+  namespace: argocd
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+    ingress.kubernetes.io/ssl-redirect: "false"
+spec:
+  rules:
+  - host: argocd.cloudfleet.platform.5ha.re
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: argocd-server
+            port:
+              number: 80
+  tls:
+  - secretName: argocd-tls

argo/apps/180-zot/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+name: zot-wrapper
+description: A wrapper Helm chart to deploy zot registry with custom values and S3 storage.
+version: 0.1.0 # Version of this wrapper chart
+appVersion: "v2.1.4" # Corresponds to the zot version we deploy
+
+dependencies:
+- name: zot
+  version: "0.1.72" # The version of the zot chart to use
+  repository: http://zotregistry.dev/helm-charts # The repository of the dependency
+  # Values under the 'zot' key in our values.yaml will be passed to the subchart.

argo/apps/180-zot/application.yaml

@@ -0,0 +1,27 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: zot
+  namespace: argocd
+  # Add finalizer to ensure that Helm release is deleted before the app
+  finalizers:
+  - argocd.argoproj.io/resources-finalizer # Use domain-qualified finalizer
+spec:
+  project: default
+  source:
+    # Source is the Git repository containing this Application manifest and the wrapper chart
+    repoURL: https://github.com/datamindedbe/playground-data-platform-stack.git
+    path: argo/apps/180-zot # Path to the zot wrapper chart directory
+    targetRevision: cloudfleet # Or your specific branch/tag
+    # Helm configuration for the wrapper chart
+    helm:
+      releaseName: zot # Helm release name
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: services # Deploy zot into the services namespace
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true # Ensure the services namespace is created

argo/apps/180-zot/templates/certificate.yaml

@@ -0,0 +1,11 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: zot-cert
+spec:
+  secretName: zot-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  dnsNames:
+    - zot.cloudfleet.platform.5ha.re

argo/apps/180-zot/templates/s3-credentials.yaml

@@ -0,0 +1,26 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: zot-s3-credentials
+spec:
+  refreshInterval: 300s # Refresh every 5 minutes
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  data:
+   - secretKey: AWS_ACCESS_KEY_ID # Standard AWS SDK environment variable
+     remoteRef:
+       key: platform/s3-credentials  # Path in Vault
+       property: ACCESS_KEY_ID  # Property to extract from the Vault secret
+   - secretKey: AWS_SECRET_ACCESS_KEY # Standard AWS SDK environment variable
+     remoteRef:
+       key: platform/s3-credentials  # Path in Vault
+       property: SECRET_ACCESS_KEY  # Property to extract from the Vault secret
+   - secretKey: AWS_REGION # Standard AWS SDK environment variable
+     remoteRef:
+       key: platform/s3-credentials  # Path in Vault
+       property: REGION  # Property to extract from the Vault secret
+   - secretKey: AWS_ENDPOINT_URL # Standard AWS SDK environment variable for custom endpoints
+     remoteRef:
+       key: platform/s3-credentials  # Path in Vault
+       property: ENDPOINT  # Property to extract from the Vault secret