From 12ce5350e7c4f5c51f64ab5bc9a9fcc63d6ee476 Mon Sep 17 00:00:00 2001
From: stxkxs <stxkxs@users.noreply.github.com>
Date: Mon, 25 May 2026 20:31:33 -0700
Subject: [PATCH] fix(addons): grafana-agent River env() syntax +
 bitnamilegacy/kubectl
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two unrelated bugs surfaced when bringing up a fresh cluster against
this repo:

─── grafana-agent: sys.env() is not a valid River function ──────────

Three call sites in addons/observability/grafana-agent/values.yaml
used `sys.env(...)` to read env-var-projected IRSA + AMP config
(AMP_REMOTE_WRITE_URL, AWS_REGION, CLUSTER_NAME). Grafana Agent's
River stdlib exposes `env(name)`, not `sys.env(name)` — the agent
fails at config-load with `component "sys.env" does not exist or
is out of scope`, and the DaemonSet pods CrashLoopBackOff before
they can ship a single metric.

─── kyverno + velero: bitnami/kubectl hooks are unpullable ──────────

Bitnami moved its public Docker Hub images to the `bitnamilegacy/`
namespace in 2025; the `bitnami/kubectl` repo no longer publishes
new tags, and `bitnami/kubectl:latest` 404s on pull. Both kyverno
(cleanupJobs.admissionReports + clusterAdmissionReports) and
velero (kubectl helm hook) were still pointing at the old repo,
so their hook pods land in ImagePullBackOff and the cleanup never
runs.

Switched both to `bitnamilegacy/kubectl:1.33.4-debian-12-r0` — a
specific tag verified to exist on the legacy registry. Pinning by
version (vs `:latest`) is also safer for helm hooks, which run on
upgrade and shouldn't pick up surprise image changes.

Comments updated to explain WHY this repo exists (Bitnami's public
namespace migration) so the next person who hits an unpullable
`bitnami/*` image elsewhere knows where to look.
---
 addons/observability/grafana-agent/values.yaml |  6 +++---
 addons/operations/velero/values.yaml           |  8 +++++---
 addons/security/kyverno/values.yaml            | 13 ++++++++-----
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/addons/observability/grafana-agent/values.yaml b/addons/observability/grafana-agent/values.yaml
index 8ab7248..d1ab2ce 100644
--- a/addons/observability/grafana-agent/values.yaml
+++ b/addons/observability/grafana-agent/values.yaml
@@ -72,13 +72,13 @@ agent:
       // expects service "aps" and the workspace's region.
       prometheus.remote_write "amp" {
         endpoint {
-          url = sys.env("AMP_REMOTE_WRITE_URL")
+          url = env("AMP_REMOTE_WRITE_URL")
           sigv4 {
-            region = sys.env("AWS_REGION")
+            region = env("AWS_REGION")
           }
         }
         external_labels = {
-          cluster = sys.env("CLUSTER_NAME"),
+          cluster = env("CLUSTER_NAME"),
         }
       }
 
diff --git a/addons/operations/velero/values.yaml b/addons/operations/velero/values.yaml
index f074719..8a40dda 100644
--- a/addons/operations/velero/values.yaml
+++ b/addons/operations/velero/values.yaml
@@ -51,11 +51,13 @@ tolerations:
   - key: CriticalAddonsOnly
     operator: Exists
 
-# Override kubectl image for helm hooks (bitnamilegacy/kubectl:1.35 doesn't exist)
+# Override kubectl image for helm hooks. Bitnami migrated public images to
+# `bitnamilegacy/`; pin to a real existing tag (latest isn't published on
+# the legacy repo).
 kubectl:
   image:
-    repository: docker.io/bitnami/kubectl
-    tag: latest
+    repository: docker.io/bitnamilegacy/kubectl
+    tag: 1.33.4-debian-12-r0
 
 # Default backup schedules
 schedules:
diff --git a/addons/security/kyverno/values.yaml b/addons/security/kyverno/values.yaml
index d4de878..d5cd352 100644
--- a/addons/security/kyverno/values.yaml
+++ b/addons/security/kyverno/values.yaml
@@ -66,15 +66,18 @@ features:
 webhooksCleanup:
   enabled: true
 
-# Override kubectl image for helm hooks (bitnami/kubectl:1.32.3 doesn't exist)
+# Override kubectl image for helm hooks. Bitnami migrated public images to
+# `bitnamilegacy/` (the `bitnami/kubectl` repo on Docker Hub is no longer
+# published). Pin to a real, existing tag — `latest` on the legacy repo
+# isn't reliably maintained either.
 cleanupJobs:
   admissionReports:
     image:
       registry: docker.io
-      repository: bitnami/kubectl
-      tag: latest
+      repository: bitnamilegacy/kubectl
+      tag: 1.33.4-debian-12-r0
   clusterAdmissionReports:
     image:
       registry: docker.io
-      repository: bitnami/kubectl
-      tag: latest
+      repository: bitnamilegacy/kubectl
+      tag: 1.33.4-debian-12-r0