From c0620112070adf939492e4d3f45f363a298c40a1 Mon Sep 17 00:00:00 2001
From: Bora Oztekin <boztekin@nvidia.com>
Date: Wed, 22 Jul 2026 21:45:19 +0000
Subject: [PATCH 1/6] feat(self-managed): wire autoscaler observability

Signed-off-by: Bora Oztekin <boztekin@nvidia.com>
---
 deploy/helm/function-autoscaler/values.yaml   |  20 ++
 .../charts/nvcf-default-monitors/values.yaml  |  13 +-
 .../observability/environments/base.yaml      |   2 +
 deploy/stacks/self-managed/Makefile.dist      |   2 +-
 .../self-managed/environments/base.yaml       |  16 ++
 deploy/stacks/self-managed/global.yaml.gotmpl |  96 +++++++++-
 .../helmfile.d/03-observability.yaml.gotmpl   | 174 ++++++++++++++++++
 7 files changed, 320 insertions(+), 3 deletions(-)

diff --git a/deploy/helm/function-autoscaler/values.yaml b/deploy/helm/function-autoscaler/values.yaml
index db8eec8f7..d15a2b63f 100644
--- a/deploy/helm/function-autoscaler/values.yaml
+++ b/deploy/helm/function-autoscaler/values.yaml
@@ -97,6 +97,9 @@ functionautoscaler:
     - name: probe
       containerPort: 8181
       protocol: TCP
+    - name: metrics
+      containerPort: 41338
+      protocol: TCP
 
   # Stable service DNS. Override for multiple releases in one namespace.
   service:
@@ -110,6 +113,10 @@ functionautoscaler:
         port: 8181
         targetPort: probe
         protocol: TCP
+      - name: metrics
+        port: 41338
+        targetPort: metrics
+        protocol: TCP
 
   # Additional volumes on the output Deployment definition.
   # The projected service account token lets the Vault Agent authenticate to
@@ -140,3 +147,16 @@ functionautoscaler:
     # Read the secrets file rendered by the Vault Agent. Override this value
     # when mounting secrets manually.
     SECRETS_PATH: "/vault/secrets/secrets.json"
+
+    # Self-managed in-cluster defaults. Consuming stacks should override these
+    # only when service names, namespaces, or auth mode differ.
+    CASSANDRA__CONTACT_POINTS: "cassandra.cassandra-system.svc.cluster.local"
+    CASSANDRA__PORT: "9042"
+    CASSANDRA__KEYSPACE: "nvcf_autoscaler"
+    CASSANDRA__SSL__ENABLED: "false"
+    TIMESERIES_DB__TIMESERIES_DB_URL: "http://vmsingle.monitoring.svc.cluster.local:8428"
+    TIMESERIES_DB__DISABLE_AUTH: "true"
+    TIMESERIES_DB__ENV: "stg"
+    TIMESERIES_DB__IGNORE_ENV: "true"
+    NVCF_API__NVCF_API_GRPC_ADDRESS: "http://api.nvcf.svc.cluster.local:9090"
+    NVCF_API__DISABLE_AUTH: "true"
diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml
index 1605cd7ee..845114034 100644
--- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml
+++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml
@@ -45,26 +45,37 @@ controlPlane:
       selector:
         app.kubernetes.io/instance: state-metrics
         app.kubernetes.io/name: helm-nvcf-state-metrics
+    - name: function-autoscaler
+      enabled: true
+      selector:
+        app.kubernetes.io/instance: function-autoscaler
+        app.kubernetes.io/name: helm-nvcf-function-autoscaler
     - name: grpc-proxy
       enabled: true
       selector:
         app.kubernetes.io/instance: grpc-proxy
     - name: api
       enabled: true
+      path: /actuator/prometheus
+      port: management-port
       selector:
         app.kubernetes.io/name: helm-nvcf-api
     - name: sis
       enabled: true
       namespaces:
         - sis
+      path: /actuator/prometheus
+      port: management-port
       selector:
         app.kubernetes.io/name: helm-nvcf-sis
     - name: llm-api-gateway
       enabled: true
       selector:
         app.kubernetes.io/instance: llm-api-gateway
-    # Off until the Invocation Service exporter is enabled.
+    # Off until the packaged Invocation Service chart can configure
+    # server.metrics.exporters with a pod-reachable Prometheus endpoint.
     - name: invocation-service
       enabled: false
       selector:
         app.kubernetes.io/instance: invocation-service
+        app.kubernetes.io/name: helm-nvcf-invocation-service
diff --git a/deploy/stacks/observability/environments/base.yaml b/deploy/stacks/observability/environments/base.yaml
index 11f3b91a5..81afa88bc 100644
--- a/deploy/stacks/observability/environments/base.yaml
+++ b/deploy/stacks/observability/environments/base.yaml
@@ -114,6 +114,8 @@ defaultMonitors:
   enabled: false
   commonLabels:
     nvcf.nvidia.com/observability-target: "true"
+  controlPlane:
+    enabled: true
   dcgm:
     enabled: false
     namespaceSelector:
diff --git a/deploy/stacks/self-managed/Makefile.dist b/deploy/stacks/self-managed/Makefile.dist
index 5b99749e9..e281fe1e0 100644
--- a/deploy/stacks/self-managed/Makefile.dist
+++ b/deploy/stacks/self-managed/Makefile.dist
@@ -32,7 +32,7 @@ REQUIRED_HELMFILE_MINOR := 1
 REQUIRED_HELM_MAJOR := 3
 
 # Namespaces created by the NVCF stack (used by destroy target)
-NAMESPACES := cassandra-system nats-system vault-system ncp nvcf api-keys ess sis
+NAMESPACES := cassandra-system nats-system vault-system ncp nvcf api-keys ess sis monitoring
 
 # Optional kubeconfig file - only used if defined
 KUBECONFIG_FILE ?=
diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index a73d69869..4e2e5810f 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -260,6 +260,22 @@ addons:
     tolerations: []
     affinity: {}
 
+# Function Autoscaler workload (stage 03). Disabled by default; requires the
+# stage-01 Cassandra/OpenBao migrations and the observability stack's VictoriaMetrics.
+functionAutoscaler:
+  enabled: false
+  # Use the in-repo chart for pre-publish testing. Set empty and use
+  # chartVersion once helm-nvcf-function-autoscaler is published to OCI.
+  chartPath: "../../../helm/function-autoscaler"
+  chartVersion: "0.0.0"
+  image:
+    tag: "v1.18.5"
+    pullPolicy: IfNotPresent
+  # Optional overrides for chart-owned defaults.
+  timeseriesDb: {}
+  nvcfApi: {}
+  cassandra: {}
+
 stateMetrics:
   enabled: false
   # Disable ServiceMonitor for environments without Prometheus Operator
diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index df17ad4c7..c6998afa4 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -336,9 +336,29 @@ api:
     NVCF_REGISTRIES_RECOGNIZED_RESOURCE_NGC_OAUTH2_GROUP_SCOPE: "ngc"
     # Observability
     MANAGEMENT_TRACING_ENABLED: {{ .Values.global.observability.tracing.enabled | quote }}
+    MANAGEMENT_PROMETHEUS_METRICS_EXPORT_ENABLED: {{ .Values.global.observability.metrics.enabled | quote }}
+    MANAGEMENT_METRICS_ENABLE_ALL: {{ .Values.global.observability.metrics.enabled | quote }}
     {{- if .Values.global.observability.tracing.enabled }}
     MANAGEMENT_OTLP_TRACING_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}/v1/traces"
     {{- end }}
+  service:
+    ports:
+      - name: http
+        port: 8080
+        targetPort: http
+        protocol: TCP
+      - name: grpc
+        port: 9090
+        targetPort: grpc
+        protocol: TCP
+      - name: management-port
+        port: 8181
+        targetPort: management-port
+        protocol: TCP
+      - name: metrics
+        port: 9464
+        targetPort: metrics
+        protocol: TCP
 
 invocation:
   fullnameOverride: invocation-service
@@ -565,7 +585,7 @@ adminIssuerProxy:
     path: "/v1/admin/keys"
 
 stateMetrics:
-  enabled: false
+  enabled: {{ dig "stateMetrics" "enabled" false .Values }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
     {{- toYaml .Values.global.imagePullSecrets | nindent 4 }}
@@ -583,6 +603,80 @@ stateMetrics:
   serviceMonitor:
     enabled: false
 
+{{- $observabilityNamespace := dig "observability" "namespace" "monitoring" .Values }}
+{{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" $observabilityNamespace .Values }}
+{{- $functionAutoscalerValues := dig "functionAutoscaler" (dig "functionautoscaler" dict .Values) .Values }}
+{{- $functionAutoscalerImageValues := dig "image" dict $functionAutoscalerValues }}
+{{- $functionAutoscalerTimeseriesDbValues := dig "timeseriesDb" dict $functionAutoscalerValues }}
+{{- $functionAutoscalerNvcfApiValues := dig "nvcfApi" dict $functionAutoscalerValues }}
+{{- $functionAutoscalerCassandraValues := dig "cassandra" dict $functionAutoscalerValues }}
+{{- $functionAutoscalerCassandraSslValues := dig "ssl" dict $functionAutoscalerCassandraValues }}
+{{- $functionAutoscalerTsdbURL := (dig "timeseriesDbUrl" "" $functionAutoscalerValues) | default (dig "url" "" $functionAutoscalerTimeseriesDbValues) }}
+{{- if and (not $functionAutoscalerTsdbURL) (ne $victoriaMetricsNamespace "monitoring") }}
+{{- $functionAutoscalerTsdbURL = printf "http://vmsingle.%s.svc.cluster.local:8428" $victoriaMetricsNamespace }}
+{{- end }}
+{{- $functionAutoscalerCassandraContactPoints := (dig "cassandraContactPoints" "" $functionAutoscalerValues) | default (dig "contactPoints" "" $functionAutoscalerCassandraValues) }}
+{{- $functionAutoscalerGrpcAddress := (dig "nvcfApiGrpcAddress" "" $functionAutoscalerValues) | default (dig "grpcAddress" "" $functionAutoscalerNvcfApiValues) }}
+{{- $functionAutoscalerOauthURL := (dig "oauth2TokenApiAddress" "" $functionAutoscalerValues) | default (dig "oauth2TokenApiAddress" "" $functionAutoscalerNvcfApiValues) }}
+{{- $functionAutoscalerImageRepository := (dig "repository" "" $functionAutoscalerImageValues) | default (printf "%s/nvcf-autoscaler-service" .Values.global.image.repository) }}
+{{- $functionAutoscalerHasEnvOverrides := or $functionAutoscalerTsdbURL $functionAutoscalerCassandraContactPoints $functionAutoscalerGrpcAddress $functionAutoscalerOauthURL (hasKey $functionAutoscalerTimeseriesDbValues "disableAuth") (hasKey $functionAutoscalerTimeseriesDbValues "env") (hasKey $functionAutoscalerTimeseriesDbValues "ignoreEnv") (hasKey $functionAutoscalerCassandraValues "port") (hasKey $functionAutoscalerCassandraValues "keyspace") (hasKey $functionAutoscalerCassandraSslValues "enabled") (hasKey $functionAutoscalerNvcfApiValues "disableAuth") }}
+functionautoscaler:
+  {{- if .Values.global.imagePullSecrets }}
+  imagePullSecrets:
+    {{- toYaml .Values.global.imagePullSecrets | nindent 4 }}
+  {{- end }}
+  image:
+    registry: {{ .Values.global.image.registry }}
+    repository: {{ $functionAutoscalerImageRepository }}
+    {{- with dig "tag" "" $functionAutoscalerImageValues }}
+    tag: {{ . | quote }}
+    {{- end }}
+    {{- with dig "pullPolicy" "" $functionAutoscalerImageValues }}
+    pullPolicy: {{ . | quote }}
+    {{- end }}
+  {{- with include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) }}
+  {{- . | nindent 2 }}
+  {{- end }}
+  {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
+  {{- . | nindent 2 }}
+  {{- end }}
+  {{- if $functionAutoscalerHasEnvOverrides }}
+  env:
+    {{- with $functionAutoscalerTsdbURL }}
+    TIMESERIES_DB__TIMESERIES_DB_URL: {{ $functionAutoscalerTsdbURL | quote }}
+    {{- end }}
+    {{- if hasKey $functionAutoscalerTimeseriesDbValues "disableAuth" }}
+    TIMESERIES_DB__DISABLE_AUTH: {{ get $functionAutoscalerTimeseriesDbValues "disableAuth" | quote }}
+    {{- end }}
+    {{- with dig "env" "" $functionAutoscalerTimeseriesDbValues }}
+    TIMESERIES_DB__ENV: {{ . | quote }}
+    {{- end }}
+    {{- if hasKey $functionAutoscalerTimeseriesDbValues "ignoreEnv" }}
+    TIMESERIES_DB__IGNORE_ENV: {{ get $functionAutoscalerTimeseriesDbValues "ignoreEnv" | quote }}
+    {{- end }}
+    {{- with $functionAutoscalerCassandraContactPoints }}
+    CASSANDRA__CONTACT_POINTS: {{ $functionAutoscalerCassandraContactPoints | quote }}
+    {{- end }}
+    {{- with dig "port" "" $functionAutoscalerCassandraValues }}
+    CASSANDRA__PORT: {{ . | quote }}
+    {{- end }}
+    {{- with dig "keyspace" "" $functionAutoscalerCassandraValues }}
+    CASSANDRA__KEYSPACE: {{ . | quote }}
+    {{- end }}
+    {{- if hasKey $functionAutoscalerCassandraSslValues "enabled" }}
+    CASSANDRA__SSL__ENABLED: {{ get $functionAutoscalerCassandraSslValues "enabled" | quote }}
+    {{- end }}
+    {{- with $functionAutoscalerGrpcAddress }}
+    NVCF_API__NVCF_API_GRPC_ADDRESS: {{ $functionAutoscalerGrpcAddress | quote }}
+    {{- end }}
+    {{- with $functionAutoscalerOauthURL }}
+    NVCF_API__OAUTH2_TOKEN_API_ADDRESS: {{ $functionAutoscalerOauthURL | quote }}
+    {{- end }}
+    {{- if hasKey $functionAutoscalerNvcfApiValues "disableAuth" }}
+    NVCF_API__DISABLE_AUTH: {{ get $functionAutoscalerNvcfApiValues "disableAuth" | quote }}
+    {{- end }}
+  {{- end }}
+
 reval:
   fullnameOverride: reval
   {{- if .Values.global.imagePullSecrets }}
diff --git a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
index 0370fb678..8576940f2 100644
--- a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
+++ b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
@@ -1,11 +1,51 @@
 environments:
   default:
     values:
+      - ../../observability/environments/base.yaml
       - ../environments/base.yaml
       - ../environments/{{ requiredEnv "HELMFILE_ENV" }}.yaml
 
 ---
 
+{{- $mode := dig "observability" "mode" "disabled" .Values }}
+{{- if not (has $mode (list "install" "existing" "disabled")) }}
+{{- fail (printf "observability.mode must be install, existing, or disabled, got %q" $mode) }}
+{{- end }}
+{{- $installMode := eq $mode "install" }}
+{{- $existingMode := eq $mode "existing" }}
+{{- $observabilityNamespace := dig "observability" "namespace" "monitoring" .Values }}
+{{- $controlPlaneCollectorEnabled := and $installMode (dig "controlPlaneCollector" "enabled" false .Values) }}
+{{- $defaultMonitorsEnabled := and (or $installMode $existingMode) (dig "defaultMonitors" "enabled" false .Values) }}
+{{- $prometheusOperatorCrdsEnabled := and $installMode (dig "prometheusOperatorCrds" "enabled" false .Values) }}
+{{- $opentelemetryOperatorEnabled := and $installMode (dig "opentelemetryOperator" "enabled" false .Values) }}
+{{- $victoriaMetricsEnabled := and $installMode (dig "victoriaMetrics" "enabled" false .Values) }}
+{{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" $observabilityNamespace .Values }}
+{{- $bundledRemoteWriteEndpoint := printf "http://vmsingle.%s.svc.cluster.local:8428/api/v1/write" $victoriaMetricsNamespace }}
+{{- $configuredRemoteWriteEndpoint := dig "controlPlaneCollector" "config" "exporters" "prometheusremotewrite" "endpoint" "" .Values }}
+{{- if and $controlPlaneCollectorEnabled $victoriaMetricsEnabled (ne $configuredRemoteWriteEndpoint $bundledRemoteWriteEndpoint) }}
+{{- fail (printf "controlPlaneCollector.config.exporters.prometheusremotewrite.endpoint must be %q when victoriaMetrics.enabled=true, got %q" $bundledRemoteWriteEndpoint $configuredRemoteWriteEndpoint) }}
+{{- end }}
+{{- $functionAutoscalerValues := dig "functionAutoscaler" (dig "functionautoscaler" dict .Values) .Values }}
+{{- $functionAutoscalerNvcfApiValues := dig "nvcfApi" dict $functionAutoscalerValues }}
+{{- $functionAutoscalerEnabled := dig "enabled" false $functionAutoscalerValues }}
+{{- $functionAutoscalerChartPath := dig "chartPath" "" $functionAutoscalerValues }}
+{{- $functionAutoscalerChart := $functionAutoscalerChartPath | default "nvcf/helm-nvcf-function-autoscaler" }}
+{{- $functionAutoscalerChartVersion := dig "chartVersion" "0.0.0" $functionAutoscalerValues }}
+{{- $functionAutoscalerOauthURL := (dig "oauth2TokenApiAddress" "" $functionAutoscalerValues) | default (dig "nvcfApi" "oauth2TokenApiAddress" "" $functionAutoscalerValues) }}
+{{- $functionAutoscalerAuthEnabled := and (hasKey $functionAutoscalerNvcfApiValues "disableAuth") (not (get $functionAutoscalerNvcfApiValues "disableAuth")) }}
+{{- if and $functionAutoscalerEnabled (eq $mode "disabled") }}
+{{- fail "functionAutoscaler.enabled requires observability.mode to be install or existing" }}
+{{- end }}
+{{- if and $functionAutoscalerEnabled (not $functionAutoscalerChartPath) (or (not $functionAutoscalerChartVersion) (eq $functionAutoscalerChartVersion "0.0.0")) }}
+{{- fail "functionAutoscaler.chartVersion must be set to the published helm-nvcf-function-autoscaler chart version when functionAutoscaler.enabled=true and functionAutoscaler.chartPath is empty" }}
+{{- end }}
+{{- if and $functionAutoscalerEnabled (not (dig "stateMetrics" "enabled" false .Values)) }}
+{{- fail "functionAutoscaler.enabled requires stateMetrics.enabled=true" }}
+{{- end }}
+{{- if and $functionAutoscalerEnabled $functionAutoscalerAuthEnabled (not $functionAutoscalerOauthURL) }}
+{{- fail "functionAutoscaler.nvcfApi.oauth2TokenApiAddress is required when functionAutoscaler.enabled=true and nvcfApi.disableAuth=false" }}
+{{- end }}
+
 repositories:
   - name: nvcf
     url: {{ .Values.global.helm.sources.registry }}/{{ .Values.global.helm.sources.repository }}
@@ -31,6 +71,108 @@ templates:
 
 releases:
 
+  {{- if $prometheusOperatorCrdsEnabled }}
+  - name: prometheus-operator-crds
+    chart: nvcf/prometheus-operator-crds
+    version: {{ dig "prometheusOperatorCrds" "version" "30.0.0" .Values | quote }}
+    namespace: {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}
+    values:
+      - {{- toYaml (dig "prometheusOperatorCrds" "values" dict .Values) | nindent 8 }}
+    labels:
+      release-group: observability
+  {{- end }}
+
+  {{- if $opentelemetryOperatorEnabled }}
+  - name: opentelemetry-operator
+    chart: nvcf/opentelemetry-operator
+    version: {{ dig "opentelemetryOperator" "version" "0.114.1" .Values | quote }}
+    namespace: {{ dig "opentelemetryOperator" "namespace" $observabilityNamespace .Values }}
+    {{- if $prometheusOperatorCrdsEnabled }}
+    needs:
+      - {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}/prometheus-operator-crds
+    {{- end }}
+    values:
+      - admissionWebhooks:
+          certManager:
+            enabled: {{ dig "opentelemetryOperator" "admissionWebhooks" "certManager" "enabled" false .Values }}
+          autoGenerateCert:
+            enabled: {{ not (dig "opentelemetryOperator" "admissionWebhooks" "certManager" "enabled" false .Values) }}
+            recreate: {{ dig "opentelemetryOperator" "admissionWebhooks" "autoGenerateCert" "recreate" false .Values }}
+        manager:
+          {{- with dig "opentelemetryOperator" "manager" "collectorImage" dict .Values }}
+          collectorImage:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with dig "opentelemetryOperator" "manager" "targetAllocatorImage" dict .Values }}
+          targetAllocatorImage:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- if .Values.global.imagePullSecrets }}
+        imagePullSecrets:
+          {{- toYaml .Values.global.imagePullSecrets | nindent 10 }}
+        {{- end }}
+    labels:
+      release-group: observability
+  {{- end }}
+
+  {{- if $victoriaMetricsEnabled }}
+  - name: victoria-metrics
+    chart: nvcf/victoria-metrics-single
+    version: {{ dig "victoriaMetrics" "version" "0.38.0" .Values | quote }}
+    namespace: {{ $victoriaMetricsNamespace }}
+    values:
+      - ../../observability/values/victoria-metrics.yaml.gotmpl
+    labels:
+      release-group: observability
+  {{- end }}
+
+  {{- if $controlPlaneCollectorEnabled }}
+  - name: control-plane-collector
+    chart: ../../observability/charts/nvcf-otel-collector
+    namespace: {{ dig "controlPlaneCollector" "namespace" $observabilityNamespace .Values }}
+    {{- if or $opentelemetryOperatorEnabled $prometheusOperatorCrdsEnabled $victoriaMetricsEnabled }}
+    needs:
+      {{- if $opentelemetryOperatorEnabled }}
+      - {{ dig "opentelemetryOperator" "namespace" $observabilityNamespace .Values }}/opentelemetry-operator
+      {{- end }}
+      {{- if $prometheusOperatorCrdsEnabled }}
+      - {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}/prometheus-operator-crds
+      {{- end }}
+      {{- if $victoriaMetricsEnabled }}
+      - {{ $victoriaMetricsNamespace }}/victoria-metrics
+      {{- end }}
+    {{- end }}
+    values:
+      - collectorName: {{ dig "controlPlaneCollector" "collectorName" "nvcf-control-plane" .Values | quote }}
+        image: {{ dig "controlPlaneCollector" "image" "" .Values | quote }}
+        {{- if .Values.global.imagePullSecrets }}
+        imagePullSecrets:
+          {{- toYaml .Values.global.imagePullSecrets | nindent 10 }}
+        {{- end }}
+        targetAllocatorRbac:
+          create: {{ dig "controlPlaneCollector" "targetAllocatorRbac" "create" true .Values }}
+        targetAllocator:
+          {{- toYaml (dig "controlPlaneCollector" "targetAllocator" dict .Values) | nindent 10 }}
+        config:
+          {{- toYaml (dig "controlPlaneCollector" "config" dict .Values) | nindent 10 }}
+    labels:
+      release-group: observability
+  {{- end }}
+
+  {{- if $defaultMonitorsEnabled }}
+  - name: default-monitors
+    chart: ../../observability/charts/nvcf-default-monitors
+    namespace: {{ dig "defaultMonitors" "namespace" $observabilityNamespace .Values }}
+    {{- if $prometheusOperatorCrdsEnabled }}
+    needs:
+      - {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}/prometheus-operator-crds
+    {{- end }}
+    values:
+      - {{- toYaml (dig "defaultMonitors" dict .Values) | nindent 8 }}
+    labels:
+      release-group: observability
+  {{- end }}
+
   # --- Observability Services ---
   - name: state-metrics
     version: 1.0.1
@@ -40,3 +182,35 @@ releases:
       - template: service
     # When enabled, this service will export realtime function information that can be
     # scraped by a Prometheus ServiceMonitor
+
+  {{- if $functionAutoscalerEnabled }}
+  # --- Function Autoscaler workload ---
+  # Deploys the function-autoscaler (chart nvcf/helm-nvcf-function-autoscaler),
+  # disabled by default. SECRETS_PATH and CONFIG come from the chart defaults;
+  # in-cluster endpoints are rendered by global.yaml.gotmpl from
+  # functionAutoscaler values.
+  - name: function-autoscaler
+    chart: {{ $functionAutoscalerChart | quote }}
+    {{- if not $functionAutoscalerChartPath }}
+    version: {{ $functionAutoscalerChartVersion | quote }}
+    {{- end }}
+    namespace: nvcf
+    values:
+      - ../global.yaml.gotmpl
+      - ../secrets/{{ requiredEnv "HELMFILE_ENV" }}-secrets.yaml
+    {{- if or $victoriaMetricsEnabled $controlPlaneCollectorEnabled }}
+    needs:
+      - nvcf/state-metrics
+      {{- if $victoriaMetricsEnabled }}
+      - {{ $victoriaMetricsNamespace }}/victoria-metrics
+      {{- end }}
+      {{- if $controlPlaneCollectorEnabled }}
+      - {{ dig "controlPlaneCollector" "namespace" $observabilityNamespace .Values }}/control-plane-collector
+      {{- end }}
+    {{- else }}
+    needs:
+      - nvcf/state-metrics
+    {{- end }}
+    labels:
+      release-group: observability
+  {{- end }}

From d2c52ed108e509ee56d3e46acb054ec04aa16069 Mon Sep 17 00:00:00 2001
From: Bora Oztekin <boztekin@nvidia.com>
Date: Wed, 22 Jul 2026 23:31:18 +0000
Subject: [PATCH 2/6] fix(self-managed): make autoscaler observability
 distributable

Signed-off-by: Bora Oztekin <boztekin@nvidia.com>
---
 .../observability/environments/base.yaml      |   2 -
 deploy/stacks/self-managed/Makefile.dist      |   2 +-
 .../self-managed/environments/base.yaml       |  34 ++++-
 deploy/stacks/self-managed/global.yaml.gotmpl |  18 ---
 .../helmfile.d/03-observability.yaml.gotmpl   | 142 ++++++++++--------
 5 files changed, 111 insertions(+), 87 deletions(-)

diff --git a/deploy/stacks/observability/environments/base.yaml b/deploy/stacks/observability/environments/base.yaml
index 81afa88bc..11f3b91a5 100644
--- a/deploy/stacks/observability/environments/base.yaml
+++ b/deploy/stacks/observability/environments/base.yaml
@@ -114,8 +114,6 @@ defaultMonitors:
   enabled: false
   commonLabels:
     nvcf.nvidia.com/observability-target: "true"
-  controlPlane:
-    enabled: true
   dcgm:
     enabled: false
     namespaceSelector:
diff --git a/deploy/stacks/self-managed/Makefile.dist b/deploy/stacks/self-managed/Makefile.dist
index e281fe1e0..5b99749e9 100644
--- a/deploy/stacks/self-managed/Makefile.dist
+++ b/deploy/stacks/self-managed/Makefile.dist
@@ -32,7 +32,7 @@ REQUIRED_HELMFILE_MINOR := 1
 REQUIRED_HELM_MAJOR := 3
 
 # Namespaces created by the NVCF stack (used by destroy target)
-NAMESPACES := cassandra-system nats-system vault-system ncp nvcf api-keys ess sis monitoring
+NAMESPACES := cassandra-system nats-system vault-system ncp nvcf api-keys ess sis
 
 # Optional kubeconfig file - only used if defined
 KUBECONFIG_FILE ?=
diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index 4e2e5810f..f208aa8e1 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -125,6 +125,31 @@ global:
       # global.observability.metrics.enabled
       enabled: false
 
+# Optional observability runtime (stage 03). `install` deploys the complete
+# managed metrics path; `existing` assumes a customer-owned backend.
+observability:
+  mode: disabled
+  namespace: monitoring
+
+prometheusOperatorCrds:
+  version: "30.0.0"
+
+opentelemetryOperator:
+  version: "0.114.1"
+
+victoriaMetrics:
+  version: "0.38.0"
+
+controlPlaneCollector:
+  version: "0.1.0"
+
+defaultMonitors:
+  version: "0.1.0"
+  # External observability mode may set enabled: true to install the NVCF-owned
+  # default monitors against customer-managed scrape infrastructure.
+  controlPlane:
+    enabled: true
+
 accounts:
   limits:
     maxFunctions: 10
@@ -261,15 +286,12 @@ addons:
     affinity: {}
 
 # Function Autoscaler workload (stage 03). Disabled by default; requires the
-# stage-01 Cassandra/OpenBao migrations and the observability stack's VictoriaMetrics.
+# stage-01 Cassandra/OpenBao migrations and a Prometheus-compatible TSDB.
 functionAutoscaler:
   enabled: false
-  # Use the in-repo chart for pre-publish testing. Set empty and use
-  # chartVersion once helm-nvcf-function-autoscaler is published to OCI.
-  chartPath: "../../../helm/function-autoscaler"
-  chartVersion: "0.0.0"
+  chartVersion: "1.18.6"
   image:
-    tag: "v1.18.5"
+    tag: "1.18.6"
     pullPolicy: IfNotPresent
   # Optional overrides for chart-owned defaults.
   timeseriesDb: {}
diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index c6998afa4..4276c3afd 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -341,24 +341,6 @@ api:
     {{- if .Values.global.observability.tracing.enabled }}
     MANAGEMENT_OTLP_TRACING_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}/v1/traces"
     {{- end }}
-  service:
-    ports:
-      - name: http
-        port: 8080
-        targetPort: http
-        protocol: TCP
-      - name: grpc
-        port: 9090
-        targetPort: grpc
-        protocol: TCP
-      - name: management-port
-        port: 8181
-        targetPort: management-port
-        protocol: TCP
-      - name: metrics
-        port: 9464
-        targetPort: metrics
-        protocol: TCP
 
 invocation:
   fullnameOverride: invocation-service
diff --git a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
index 8576940f2..6d303f9f5 100644
--- a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
+++ b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
@@ -1,7 +1,6 @@
 environments:
   default:
     values:
-      - ../../observability/environments/base.yaml
       - ../environments/base.yaml
       - ../environments/{{ requiredEnv "HELMFILE_ENV" }}.yaml
 
@@ -14,34 +13,62 @@ environments:
 {{- $installMode := eq $mode "install" }}
 {{- $existingMode := eq $mode "existing" }}
 {{- $observabilityNamespace := dig "observability" "namespace" "monitoring" .Values }}
-{{- $controlPlaneCollectorEnabled := and $installMode (dig "controlPlaneCollector" "enabled" false .Values) }}
-{{- $defaultMonitorsEnabled := and (or $installMode $existingMode) (dig "defaultMonitors" "enabled" false .Values) }}
-{{- $prometheusOperatorCrdsEnabled := and $installMode (dig "prometheusOperatorCrds" "enabled" false .Values) }}
-{{- $opentelemetryOperatorEnabled := and $installMode (dig "opentelemetryOperator" "enabled" false .Values) }}
-{{- $victoriaMetricsEnabled := and $installMode (dig "victoriaMetrics" "enabled" false .Values) }}
 {{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" $observabilityNamespace .Values }}
+{{- $prometheusOperatorCrdsValues := dig "prometheusOperatorCrds" "values" dict .Values }}
+{{- $opentelemetryOperatorValues := dig "opentelemetryOperator" "values" dict .Values }}
+{{- $victoriaMetricsValues := dig "victoriaMetrics" "values" dict .Values }}
+{{- $controlPlaneCollectorValues := dig "controlPlaneCollector" "values" dict .Values }}
+{{- $controlPlaneCollectorRoot := dig "controlPlaneCollector" dict .Values }}
+{{- $defaultMonitorsValues := dig "defaultMonitors" dict .Values }}
+{{- $controlPlaneDefaultMonitorsEnabled := dig "controlPlane" "enabled" true $defaultMonitorsValues }}
+{{- $defaultMonitorsEnabled := or $installMode (and $existingMode (dig "enabled" false $defaultMonitorsValues)) }}
+{{- $victoriaMetricsEnabled := $installMode }}
+{{- $controlPlaneCollectorEnabled := $installMode }}
+{{- $prometheusOperatorCrdsEnabled := $installMode }}
+{{- $opentelemetryOperatorEnabled := $installMode }}
 {{- $bundledRemoteWriteEndpoint := printf "http://vmsingle.%s.svc.cluster.local:8428/api/v1/write" $victoriaMetricsNamespace }}
-{{- $configuredRemoteWriteEndpoint := dig "controlPlaneCollector" "config" "exporters" "prometheusremotewrite" "endpoint" "" .Values }}
-{{- if and $controlPlaneCollectorEnabled $victoriaMetricsEnabled (ne $configuredRemoteWriteEndpoint $bundledRemoteWriteEndpoint) }}
+{{- $configuredRemoteWriteEndpoint := dig "config" "exporters" "prometheusremotewrite" "endpoint" "" $controlPlaneCollectorRoot }}
+{{- if and $controlPlaneCollectorEnabled $victoriaMetricsEnabled $configuredRemoteWriteEndpoint (ne $configuredRemoteWriteEndpoint $bundledRemoteWriteEndpoint) }}
 {{- fail (printf "controlPlaneCollector.config.exporters.prometheusremotewrite.endpoint must be %q when victoriaMetrics.enabled=true, got %q" $bundledRemoteWriteEndpoint $configuredRemoteWriteEndpoint) }}
 {{- end }}
+{{- if and $installMode (hasKey (dig "victoriaMetrics" dict .Values) "enabled") (not (dig "victoriaMetrics" "enabled" true .Values)) }}
+{{- fail "observability.mode=install owns the managed metrics backend; remove victoriaMetrics.enabled=false or use observability.mode=existing with an external TSDB" }}
+{{- end }}
+{{- if and $installMode (hasKey (dig "controlPlaneCollector" dict .Values) "enabled") (not (dig "controlPlaneCollector" "enabled" true .Values)) }}
+{{- fail "observability.mode=install owns the control-plane collector; remove controlPlaneCollector.enabled=false or use observability.mode=existing" }}
+{{- end }}
+{{- if and $installMode (hasKey (dig "defaultMonitors" dict .Values) "enabled") (not (dig "defaultMonitors" "enabled" true .Values)) }}
+{{- fail "observability.mode=install owns the default monitor set; remove defaultMonitors.enabled=false or use observability.mode=existing" }}
+{{- end }}
+{{- if and $installMode (not $controlPlaneDefaultMonitorsEnabled) }}
+{{- fail "observability.mode=install requires defaultMonitors.controlPlane.enabled=true" }}
+{{- end }}
+{{- if and $installMode (hasKey (dig "prometheusOperatorCrds" dict .Values) "enabled") (not (dig "prometheusOperatorCrds" "enabled" true .Values)) }}
+{{- fail "observability.mode=install owns Prometheus Operator CRDs; remove prometheusOperatorCrds.enabled=false or use observability.mode=existing" }}
+{{- end }}
+{{- if and $installMode (hasKey (dig "opentelemetryOperator" dict .Values) "enabled") (not (dig "opentelemetryOperator" "enabled" true .Values)) }}
+{{- fail "observability.mode=install owns the OpenTelemetry Operator; remove opentelemetryOperator.enabled=false or use observability.mode=existing" }}
+{{- end }}
 {{- $functionAutoscalerValues := dig "functionAutoscaler" (dig "functionautoscaler" dict .Values) .Values }}
+{{- $functionAutoscalerTimeseriesDbValues := dig "timeseriesDb" dict $functionAutoscalerValues }}
 {{- $functionAutoscalerNvcfApiValues := dig "nvcfApi" dict $functionAutoscalerValues }}
 {{- $functionAutoscalerEnabled := dig "enabled" false $functionAutoscalerValues }}
-{{- $functionAutoscalerChartPath := dig "chartPath" "" $functionAutoscalerValues }}
-{{- $functionAutoscalerChart := $functionAutoscalerChartPath | default "nvcf/helm-nvcf-function-autoscaler" }}
-{{- $functionAutoscalerChartVersion := dig "chartVersion" "0.0.0" $functionAutoscalerValues }}
-{{- $functionAutoscalerOauthURL := (dig "oauth2TokenApiAddress" "" $functionAutoscalerValues) | default (dig "nvcfApi" "oauth2TokenApiAddress" "" $functionAutoscalerValues) }}
+{{- $functionAutoscalerChartVersion := dig "chartVersion" "" $functionAutoscalerValues }}
+{{- $functionAutoscalerTsdbURL := (dig "timeseriesDbUrl" "" $functionAutoscalerValues) | default (dig "url" "" $functionAutoscalerTimeseriesDbValues) }}
+{{- $functionAutoscalerOauthURL := (dig "oauth2TokenApiAddress" "" $functionAutoscalerValues) | default (dig "oauth2TokenApiAddress" "" $functionAutoscalerNvcfApiValues) }}
 {{- $functionAutoscalerAuthEnabled := and (hasKey $functionAutoscalerNvcfApiValues "disableAuth") (not (get $functionAutoscalerNvcfApiValues "disableAuth")) }}
 {{- if and $functionAutoscalerEnabled (eq $mode "disabled") }}
 {{- fail "functionAutoscaler.enabled requires observability.mode to be install or existing" }}
 {{- end }}
-{{- if and $functionAutoscalerEnabled (not $functionAutoscalerChartPath) (or (not $functionAutoscalerChartVersion) (eq $functionAutoscalerChartVersion "0.0.0")) }}
-{{- fail "functionAutoscaler.chartVersion must be set to the published helm-nvcf-function-autoscaler chart version when functionAutoscaler.enabled=true and functionAutoscaler.chartPath is empty" }}
+{{- if and $functionAutoscalerEnabled (not $functionAutoscalerChartVersion) }}
+{{- fail "functionAutoscaler.chartVersion must be set to the published helm-nvcf-function-autoscaler chart version when functionAutoscaler.enabled=true" }}
 {{- end }}
 {{- if and $functionAutoscalerEnabled (not (dig "stateMetrics" "enabled" false .Values)) }}
 {{- fail "functionAutoscaler.enabled requires stateMetrics.enabled=true" }}
 {{- end }}
+{{- if and $functionAutoscalerEnabled $existingMode (not $functionAutoscalerTsdbURL) }}
+{{- fail "functionAutoscaler.timeseriesDbUrl or functionAutoscaler.timeseriesDb.url is required when observability.mode=existing" }}
+{{- end }}
 {{- if and $functionAutoscalerEnabled $functionAutoscalerAuthEnabled (not $functionAutoscalerOauthURL) }}
 {{- fail "functionAutoscaler.nvcfApi.oauth2TokenApiAddress is required when functionAutoscaler.enabled=true and nvcfApi.disableAuth=false" }}
 {{- end }}
@@ -76,8 +103,10 @@ releases:
     chart: nvcf/prometheus-operator-crds
     version: {{ dig "prometheusOperatorCrds" "version" "30.0.0" .Values | quote }}
     namespace: {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}
+    {{- with $prometheusOperatorCrdsValues }}
     values:
-      - {{- toYaml (dig "prometheusOperatorCrds" "values" dict .Values) | nindent 8 }}
+      - {{- toYaml . | nindent 8 }}
+    {{- end }}
     labels:
       release-group: observability
   {{- end }}
@@ -91,26 +120,16 @@ releases:
     needs:
       - {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}/prometheus-operator-crds
     {{- end }}
+    {{- if or $opentelemetryOperatorValues .Values.global.imagePullSecrets }}
     values:
-      - admissionWebhooks:
-          certManager:
-            enabled: {{ dig "opentelemetryOperator" "admissionWebhooks" "certManager" "enabled" false .Values }}
-          autoGenerateCert:
-            enabled: {{ not (dig "opentelemetryOperator" "admissionWebhooks" "certManager" "enabled" false .Values) }}
-            recreate: {{ dig "opentelemetryOperator" "admissionWebhooks" "autoGenerateCert" "recreate" false .Values }}
-        manager:
-          {{- with dig "opentelemetryOperator" "manager" "collectorImage" dict .Values }}
-          collectorImage:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with dig "opentelemetryOperator" "manager" "targetAllocatorImage" dict .Values }}
-          targetAllocatorImage:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-        {{- if .Values.global.imagePullSecrets }}
-        imagePullSecrets:
+      {{- with $opentelemetryOperatorValues }}
+      - {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.global.imagePullSecrets }}
+      - imagePullSecrets:
           {{- toYaml .Values.global.imagePullSecrets | nindent 10 }}
-        {{- end }}
+      {{- end }}
+    {{- end }}
     labels:
       release-group: observability
   {{- end }}
@@ -121,14 +140,24 @@ releases:
     version: {{ dig "victoriaMetrics" "version" "0.38.0" .Values | quote }}
     namespace: {{ $victoriaMetricsNamespace }}
     values:
-      - ../../observability/values/victoria-metrics.yaml.gotmpl
+      - server:
+          fullnameOverride: vmsingle
+      {{- with $victoriaMetricsValues }}
+      - {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.global.imagePullSecrets }}
+      - global:
+          imagePullSecrets:
+            {{- toYaml .Values.global.imagePullSecrets | nindent 12 }}
+      {{- end }}
     labels:
       release-group: observability
   {{- end }}
 
   {{- if $controlPlaneCollectorEnabled }}
   - name: control-plane-collector
-    chart: ../../observability/charts/nvcf-otel-collector
+    chart: nvcf/nvcf-otel-collector
+    version: {{ dig "controlPlaneCollector" "version" "0.1.0" .Values | quote }}
     namespace: {{ dig "controlPlaneCollector" "namespace" $observabilityNamespace .Values }}
     {{- if or $opentelemetryOperatorEnabled $prometheusOperatorCrdsEnabled $victoriaMetricsEnabled }}
     needs:
@@ -142,33 +171,37 @@ releases:
       - {{ $victoriaMetricsNamespace }}/victoria-metrics
       {{- end }}
     {{- end }}
+    {{- if or $controlPlaneCollectorValues (ne $victoriaMetricsNamespace "monitoring") .Values.global.imagePullSecrets }}
     values:
-      - collectorName: {{ dig "controlPlaneCollector" "collectorName" "nvcf-control-plane" .Values | quote }}
-        image: {{ dig "controlPlaneCollector" "image" "" .Values | quote }}
-        {{- if .Values.global.imagePullSecrets }}
-        imagePullSecrets:
+      {{- if ne $victoriaMetricsNamespace "monitoring" }}
+      - config:
+          exporters:
+            prometheusremotewrite:
+              endpoint: {{ $bundledRemoteWriteEndpoint | quote }}
+      {{- end }}
+      {{- with $controlPlaneCollectorValues }}
+      - {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.global.imagePullSecrets }}
+      - imagePullSecrets:
           {{- toYaml .Values.global.imagePullSecrets | nindent 10 }}
-        {{- end }}
-        targetAllocatorRbac:
-          create: {{ dig "controlPlaneCollector" "targetAllocatorRbac" "create" true .Values }}
-        targetAllocator:
-          {{- toYaml (dig "controlPlaneCollector" "targetAllocator" dict .Values) | nindent 10 }}
-        config:
-          {{- toYaml (dig "controlPlaneCollector" "config" dict .Values) | nindent 10 }}
+      {{- end }}
+    {{- end }}
     labels:
       release-group: observability
   {{- end }}
 
   {{- if $defaultMonitorsEnabled }}
   - name: default-monitors
-    chart: ../../observability/charts/nvcf-default-monitors
+    chart: nvcf/nvcf-default-monitors
+    version: {{ dig "defaultMonitors" "version" "0.1.0" .Values | quote }}
     namespace: {{ dig "defaultMonitors" "namespace" $observabilityNamespace .Values }}
     {{- if $prometheusOperatorCrdsEnabled }}
     needs:
       - {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}/prometheus-operator-crds
     {{- end }}
     values:
-      - {{- toYaml (dig "defaultMonitors" dict .Values) | nindent 8 }}
+      - {{- toYaml $defaultMonitorsValues | nindent 8 }}
     labels:
       release-group: observability
   {{- end }}
@@ -185,20 +218,13 @@ releases:
 
   {{- if $functionAutoscalerEnabled }}
   # --- Function Autoscaler workload ---
-  # Deploys the function-autoscaler (chart nvcf/helm-nvcf-function-autoscaler),
-  # disabled by default. SECRETS_PATH and CONFIG come from the chart defaults;
-  # in-cluster endpoints are rendered by global.yaml.gotmpl from
-  # functionAutoscaler values.
   - name: function-autoscaler
-    chart: {{ $functionAutoscalerChart | quote }}
-    {{- if not $functionAutoscalerChartPath }}
+    chart: nvcf/helm-nvcf-function-autoscaler
     version: {{ $functionAutoscalerChartVersion | quote }}
-    {{- end }}
     namespace: nvcf
     values:
       - ../global.yaml.gotmpl
       - ../secrets/{{ requiredEnv "HELMFILE_ENV" }}-secrets.yaml
-    {{- if or $victoriaMetricsEnabled $controlPlaneCollectorEnabled }}
     needs:
       - nvcf/state-metrics
       {{- if $victoriaMetricsEnabled }}
@@ -207,10 +233,6 @@ releases:
       {{- if $controlPlaneCollectorEnabled }}
       - {{ dig "controlPlaneCollector" "namespace" $observabilityNamespace .Values }}/control-plane-collector
       {{- end }}
-    {{- else }}
-    needs:
-      - nvcf/state-metrics
-    {{- end }}
     labels:
       release-group: observability
   {{- end }}

From 8a6db33ffe3c573d6dcabbb9516e29ab9ee885fa Mon Sep 17 00:00:00 2001
From: Bora Oztekin <boztekin@nvidia.com>
Date: Fri, 24 Jul 2026 17:27:33 +0000
Subject: [PATCH 3/6] fix(self-managed): repair autoscaler observability render

Signed-off-by: Bora Oztekin <boztekin@nvidia.com>
---
 .../charts/nvcf-default-monitors/values.yaml  |  4 +-
 .../self-managed/environments/base.yaml       |  2 +-
 deploy/stacks/self-managed/global.yaml.gotmpl | 58 ++++++++++++-------
 .../helmfile.d/03-observability.yaml.gotmpl   | 25 +++++---
 4 files changed, 57 insertions(+), 32 deletions(-)

diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml
index 845114034..13ad72b7d 100644
--- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml
+++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml
@@ -72,10 +72,8 @@ controlPlane:
       enabled: true
       selector:
         app.kubernetes.io/instance: llm-api-gateway
-    # Off until the packaged Invocation Service chart can configure
-    # server.metrics.exporters with a pod-reachable Prometheus endpoint.
     - name: invocation-service
-      enabled: false
+      enabled: true
       selector:
         app.kubernetes.io/instance: invocation-service
         app.kubernetes.io/name: helm-nvcf-invocation-service
diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index f208aa8e1..608da42cc 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -289,7 +289,7 @@ addons:
 # stage-01 Cassandra/OpenBao migrations and a Prometheus-compatible TSDB.
 functionAutoscaler:
   enabled: false
-  chartVersion: "1.18.6"
+  chartVersion: "0.1.0"
   image:
     tag: "1.18.6"
     pullPolicy: IfNotPresent
diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index 4276c3afd..53356a367 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -587,20 +587,38 @@ stateMetrics:
 
 {{- $observabilityNamespace := dig "observability" "namespace" "monitoring" .Values }}
 {{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" $observabilityNamespace .Values }}
-{{- $functionAutoscalerValues := dig "functionAutoscaler" (dig "functionautoscaler" dict .Values) .Values }}
-{{- $functionAutoscalerImageValues := dig "image" dict $functionAutoscalerValues }}
-{{- $functionAutoscalerTimeseriesDbValues := dig "timeseriesDb" dict $functionAutoscalerValues }}
-{{- $functionAutoscalerNvcfApiValues := dig "nvcfApi" dict $functionAutoscalerValues }}
-{{- $functionAutoscalerCassandraValues := dig "cassandra" dict $functionAutoscalerValues }}
-{{- $functionAutoscalerCassandraSslValues := dig "ssl" dict $functionAutoscalerCassandraValues }}
-{{- $functionAutoscalerTsdbURL := (dig "timeseriesDbUrl" "" $functionAutoscalerValues) | default (dig "url" "" $functionAutoscalerTimeseriesDbValues) }}
+{{- $functionAutoscalerValues := (index .Values "functionAutoscaler") | default (index .Values "functionautoscaler") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerValues) }}
+{{- fail "functionAutoscaler must be a map" }}
+{{- end }}
+{{- $functionAutoscalerImageValues := (index $functionAutoscalerValues "image") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerImageValues) }}
+{{- fail "functionAutoscaler.image must be a map" }}
+{{- end }}
+{{- $functionAutoscalerTimeseriesDbValues := (index $functionAutoscalerValues "timeseriesDb") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerTimeseriesDbValues) }}
+{{- fail "functionAutoscaler.timeseriesDb must be a map" }}
+{{- end }}
+{{- $functionAutoscalerNvcfApiValues := (index $functionAutoscalerValues "nvcfApi") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerNvcfApiValues) }}
+{{- fail "functionAutoscaler.nvcfApi must be a map" }}
+{{- end }}
+{{- $functionAutoscalerCassandraValues := (index $functionAutoscalerValues "cassandra") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerCassandraValues) }}
+{{- fail "functionAutoscaler.cassandra must be a map" }}
+{{- end }}
+{{- $functionAutoscalerCassandraSslValues := (index $functionAutoscalerCassandraValues "ssl") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerCassandraSslValues) }}
+{{- fail "functionAutoscaler.cassandra.ssl must be a map" }}
+{{- end }}
+{{- $functionAutoscalerTsdbURL := (index $functionAutoscalerValues "timeseriesDbUrl") | default (index $functionAutoscalerTimeseriesDbValues "url") | default "" }}
 {{- if and (not $functionAutoscalerTsdbURL) (ne $victoriaMetricsNamespace "monitoring") }}
 {{- $functionAutoscalerTsdbURL = printf "http://vmsingle.%s.svc.cluster.local:8428" $victoriaMetricsNamespace }}
 {{- end }}
-{{- $functionAutoscalerCassandraContactPoints := (dig "cassandraContactPoints" "" $functionAutoscalerValues) | default (dig "contactPoints" "" $functionAutoscalerCassandraValues) }}
-{{- $functionAutoscalerGrpcAddress := (dig "nvcfApiGrpcAddress" "" $functionAutoscalerValues) | default (dig "grpcAddress" "" $functionAutoscalerNvcfApiValues) }}
-{{- $functionAutoscalerOauthURL := (dig "oauth2TokenApiAddress" "" $functionAutoscalerValues) | default (dig "oauth2TokenApiAddress" "" $functionAutoscalerNvcfApiValues) }}
-{{- $functionAutoscalerImageRepository := (dig "repository" "" $functionAutoscalerImageValues) | default (printf "%s/nvcf-autoscaler-service" .Values.global.image.repository) }}
+{{- $functionAutoscalerCassandraContactPoints := (index $functionAutoscalerValues "cassandraContactPoints") | default (index $functionAutoscalerCassandraValues "contactPoints") | default "" }}
+{{- $functionAutoscalerGrpcAddress := (index $functionAutoscalerValues "nvcfApiGrpcAddress") | default (index $functionAutoscalerNvcfApiValues "grpcAddress") | default "" }}
+{{- $functionAutoscalerOauthURL := (index $functionAutoscalerValues "oauth2TokenApiAddress") | default (index $functionAutoscalerNvcfApiValues "oauth2TokenApiAddress") | default "" }}
+{{- $functionAutoscalerImageRepository := (index $functionAutoscalerImageValues "repository") | default (printf "%s/nvcf-autoscaler-service" .Values.global.image.repository) }}
 {{- $functionAutoscalerHasEnvOverrides := or $functionAutoscalerTsdbURL $functionAutoscalerCassandraContactPoints $functionAutoscalerGrpcAddress $functionAutoscalerOauthURL (hasKey $functionAutoscalerTimeseriesDbValues "disableAuth") (hasKey $functionAutoscalerTimeseriesDbValues "env") (hasKey $functionAutoscalerTimeseriesDbValues "ignoreEnv") (hasKey $functionAutoscalerCassandraValues "port") (hasKey $functionAutoscalerCassandraValues "keyspace") (hasKey $functionAutoscalerCassandraSslValues "enabled") (hasKey $functionAutoscalerNvcfApiValues "disableAuth") }}
 functionautoscaler:
   {{- if .Values.global.imagePullSecrets }}
@@ -610,10 +628,10 @@ functionautoscaler:
   image:
     registry: {{ .Values.global.image.registry }}
     repository: {{ $functionAutoscalerImageRepository }}
-    {{- with dig "tag" "" $functionAutoscalerImageValues }}
+    {{- with (index $functionAutoscalerImageValues "tag") }}
     tag: {{ . | quote }}
     {{- end }}
-    {{- with dig "pullPolicy" "" $functionAutoscalerImageValues }}
+    {{- with (index $functionAutoscalerImageValues "pullPolicy") }}
     pullPolicy: {{ . | quote }}
     {{- end }}
   {{- with include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) }}
@@ -628,25 +646,25 @@ functionautoscaler:
     TIMESERIES_DB__TIMESERIES_DB_URL: {{ $functionAutoscalerTsdbURL | quote }}
     {{- end }}
     {{- if hasKey $functionAutoscalerTimeseriesDbValues "disableAuth" }}
-    TIMESERIES_DB__DISABLE_AUTH: {{ get $functionAutoscalerTimeseriesDbValues "disableAuth" | quote }}
+    TIMESERIES_DB__DISABLE_AUTH: {{ index $functionAutoscalerTimeseriesDbValues "disableAuth" | quote }}
     {{- end }}
-    {{- with dig "env" "" $functionAutoscalerTimeseriesDbValues }}
+    {{- with (index $functionAutoscalerTimeseriesDbValues "env") }}
     TIMESERIES_DB__ENV: {{ . | quote }}
     {{- end }}
     {{- if hasKey $functionAutoscalerTimeseriesDbValues "ignoreEnv" }}
-    TIMESERIES_DB__IGNORE_ENV: {{ get $functionAutoscalerTimeseriesDbValues "ignoreEnv" | quote }}
+    TIMESERIES_DB__IGNORE_ENV: {{ index $functionAutoscalerTimeseriesDbValues "ignoreEnv" | quote }}
     {{- end }}
     {{- with $functionAutoscalerCassandraContactPoints }}
     CASSANDRA__CONTACT_POINTS: {{ $functionAutoscalerCassandraContactPoints | quote }}
     {{- end }}
-    {{- with dig "port" "" $functionAutoscalerCassandraValues }}
+    {{- with (index $functionAutoscalerCassandraValues "port") }}
     CASSANDRA__PORT: {{ . | quote }}
     {{- end }}
-    {{- with dig "keyspace" "" $functionAutoscalerCassandraValues }}
+    {{- with (index $functionAutoscalerCassandraValues "keyspace") }}
     CASSANDRA__KEYSPACE: {{ . | quote }}
     {{- end }}
     {{- if hasKey $functionAutoscalerCassandraSslValues "enabled" }}
-    CASSANDRA__SSL__ENABLED: {{ get $functionAutoscalerCassandraSslValues "enabled" | quote }}
+    CASSANDRA__SSL__ENABLED: {{ index $functionAutoscalerCassandraSslValues "enabled" | quote }}
     {{- end }}
     {{- with $functionAutoscalerGrpcAddress }}
     NVCF_API__NVCF_API_GRPC_ADDRESS: {{ $functionAutoscalerGrpcAddress | quote }}
@@ -655,7 +673,7 @@ functionautoscaler:
     NVCF_API__OAUTH2_TOKEN_API_ADDRESS: {{ $functionAutoscalerOauthURL | quote }}
     {{- end }}
     {{- if hasKey $functionAutoscalerNvcfApiValues "disableAuth" }}
-    NVCF_API__DISABLE_AUTH: {{ get $functionAutoscalerNvcfApiValues "disableAuth" | quote }}
+    NVCF_API__DISABLE_AUTH: {{ index $functionAutoscalerNvcfApiValues "disableAuth" | quote }}
     {{- end }}
   {{- end }}
 
diff --git a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
index 6d303f9f5..5e98e33f8 100644
--- a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
+++ b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
@@ -49,14 +49,23 @@ environments:
 {{- if and $installMode (hasKey (dig "opentelemetryOperator" dict .Values) "enabled") (not (dig "opentelemetryOperator" "enabled" true .Values)) }}
 {{- fail "observability.mode=install owns the OpenTelemetry Operator; remove opentelemetryOperator.enabled=false or use observability.mode=existing" }}
 {{- end }}
-{{- $functionAutoscalerValues := dig "functionAutoscaler" (dig "functionautoscaler" dict .Values) .Values }}
-{{- $functionAutoscalerTimeseriesDbValues := dig "timeseriesDb" dict $functionAutoscalerValues }}
-{{- $functionAutoscalerNvcfApiValues := dig "nvcfApi" dict $functionAutoscalerValues }}
-{{- $functionAutoscalerEnabled := dig "enabled" false $functionAutoscalerValues }}
-{{- $functionAutoscalerChartVersion := dig "chartVersion" "" $functionAutoscalerValues }}
-{{- $functionAutoscalerTsdbURL := (dig "timeseriesDbUrl" "" $functionAutoscalerValues) | default (dig "url" "" $functionAutoscalerTimeseriesDbValues) }}
-{{- $functionAutoscalerOauthURL := (dig "oauth2TokenApiAddress" "" $functionAutoscalerValues) | default (dig "oauth2TokenApiAddress" "" $functionAutoscalerNvcfApiValues) }}
-{{- $functionAutoscalerAuthEnabled := and (hasKey $functionAutoscalerNvcfApiValues "disableAuth") (not (get $functionAutoscalerNvcfApiValues "disableAuth")) }}
+{{- $functionAutoscalerValues := (index .Values "functionAutoscaler") | default (index .Values "functionautoscaler") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerValues) }}
+{{- fail "functionAutoscaler must be a map" }}
+{{- end }}
+{{- $functionAutoscalerTimeseriesDbValues := (index $functionAutoscalerValues "timeseriesDb") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerTimeseriesDbValues) }}
+{{- fail "functionAutoscaler.timeseriesDb must be a map" }}
+{{- end }}
+{{- $functionAutoscalerNvcfApiValues := (index $functionAutoscalerValues "nvcfApi") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerNvcfApiValues) }}
+{{- fail "functionAutoscaler.nvcfApi must be a map" }}
+{{- end }}
+{{- $functionAutoscalerEnabled := (index $functionAutoscalerValues "enabled") | default false }}
+{{- $functionAutoscalerChartVersion := (index $functionAutoscalerValues "chartVersion") | default "" }}
+{{- $functionAutoscalerTsdbURL := (index $functionAutoscalerValues "timeseriesDbUrl") | default (index $functionAutoscalerTimeseriesDbValues "url") | default "" }}
+{{- $functionAutoscalerOauthURL := (index $functionAutoscalerValues "oauth2TokenApiAddress") | default (index $functionAutoscalerNvcfApiValues "oauth2TokenApiAddress") | default "" }}
+{{- $functionAutoscalerAuthEnabled := and (hasKey $functionAutoscalerNvcfApiValues "disableAuth") (not (index $functionAutoscalerNvcfApiValues "disableAuth")) }}
 {{- if and $functionAutoscalerEnabled (eq $mode "disabled") }}
 {{- fail "functionAutoscaler.enabled requires observability.mode to be install or existing" }}
 {{- end }}

From 6fb7f83b1e92b1c2db1fc4187a158bd532bb6879 Mon Sep 17 00:00:00 2001
From: Bora Oztekin <boztekin@nvidia.com>
Date: Fri, 24 Jul 2026 19:00:44 +0000
Subject: [PATCH 4/6] Support OpenBao bearer token for autoscaler

Signed-off-by: Bora Oztekin <boztekin@nvidia.com>
---
 .../vault-agent-templates/secrets.json.tmpl   |  3 ++
 .../server/src/nvcf_api/oauth2_client.rs      | 45 +++++++++++++++++++
 .../server/src/secrets/secrets_config.rs      |  2 +
 .../src/secrets/secrets_file_watcher.rs       | 33 ++++++++++++++
 4 files changed, 83 insertions(+)

diff --git a/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl b/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl
index 11a032a73..47e8735d3 100644
--- a/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl
+++ b/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl
@@ -7,6 +7,9 @@
   corresponding app integration is enabled and wired to a secret path.
 */}}
 {
+  {{- with secret `services/nvcf-api/jwt/sign/nvcf-autoscaler-service` }}
+  "nvcfApiToken": "{{ .Data.token }}",
+  {{- end }}
   "kv": {
     "cassandra": {
       {{- with secret "services/nvcf-autoscaler-service/kv/data/cassandra/creds" }}
diff --git a/src/control-plane-services/function-autoscaler/crates/server/src/nvcf_api/oauth2_client.rs b/src/control-plane-services/function-autoscaler/crates/server/src/nvcf_api/oauth2_client.rs
index e6756f037..50b2abd65 100644
--- a/src/control-plane-services/function-autoscaler/crates/server/src/nvcf_api/oauth2_client.rs
+++ b/src/control-plane-services/function-autoscaler/crates/server/src/nvcf_api/oauth2_client.rs
@@ -136,6 +136,10 @@ impl OAuth2Client {
     }
 
     pub async fn get_jwt_token(&self) -> Result<String, Box<dyn std::error::Error>> {
+        if let Some(token) = self.secrets_watcher.get_nvcf_api_token() {
+            return Ok(token);
+        }
+
         // Check if we have a cached valid token
         {
             let cache = self.token_cache.read().await;
@@ -164,6 +168,10 @@ impl OAuth2Client {
         secrets_watcher: &Arc<SecretFileWatcher>,
         http_client: &reqwest_middleware::ClientWithMiddleware,
     ) -> Result<String, Box<dyn std::error::Error>> {
+        if let Some(token) = secrets_watcher.get_nvcf_api_token() {
+            return Ok(token);
+        }
+
         // Get current secrets
         let secrets = secrets_watcher.get_config();
 
@@ -305,6 +313,26 @@ mod tests {
         (temp_dir, secrets_file)
     }
 
+    async fn create_test_secrets_file_with_static_token() -> (TempDir, std::path::PathBuf) {
+        let temp_dir = TempDir::new().unwrap();
+        let secrets_file = temp_dir.path().join("secrets.json");
+
+        let secrets_content = serde_json::json!({
+            "nvcfApiToken": "openbao-rendered-token",
+            "kv": {
+                "nvcf_api": {
+                    "client_id": "test_client_id",
+                    "client_secret": "test_client_secret"
+                }
+            }
+        });
+
+        fs::write(&secrets_file, secrets_content.to_string())
+            .await
+            .unwrap();
+        (temp_dir, secrets_file)
+    }
+
     #[tokio::test]
     async fn test_cached_token() {
         let (_temp_dir, secrets_file) = create_test_secrets_file().await;
@@ -325,6 +353,23 @@ mod tests {
         assert_eq!(creds.client_secret, "test_client_secret");
     }
 
+    #[tokio::test]
+    async fn test_static_nvcf_api_token_takes_precedence() {
+        let (_temp_dir, secrets_file) = create_test_secrets_file_with_static_token().await;
+        let secrets_watcher = SecretFileWatcher::new(&secrets_file).await.unwrap();
+        let oauth2_client = OAuth2Client::new(
+            "http://127.0.0.1:1".to_string(),
+            TEST_SCOPE.to_string(),
+            Arc::new(secrets_watcher),
+        )
+        .unwrap();
+
+        let token = oauth2_client.get_jwt_token().await.unwrap();
+
+        assert_eq!(token, "openbao-rendered-token");
+        assert!(oauth2_client.token_cache.read().await.is_none());
+    }
+
     #[tokio::test]
     async fn test_token_expiration() {
         let (_temp_dir, secrets_file) = create_test_secrets_file().await;
diff --git a/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_config.rs b/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_config.rs
index a72db49fb..cf5603561 100644
--- a/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_config.rs
+++ b/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_config.rs
@@ -19,6 +19,8 @@ use serde::Deserialize;
 
 #[derive(Debug, Deserialize, Clone)]
 pub struct Secrets {
+    #[serde(rename = "nvcfApiToken")]
+    pub nvcf_api_token: Option<String>,
     pub kv: CredentialsData,
 }
 
diff --git a/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_file_watcher.rs b/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_file_watcher.rs
index 4cf47a6a4..9e79b0502 100644
--- a/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_file_watcher.rs
+++ b/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_file_watcher.rs
@@ -157,6 +157,16 @@ impl SecretFileWatcher {
         let guard = self.config.read().unwrap();
         guard.kv.clone()
     }
+
+    pub fn get_nvcf_api_token(&self) -> Option<String> {
+        let guard = self.config.read().unwrap();
+        guard
+            .nvcf_api_token
+            .as_deref()
+            .map(str::trim)
+            .filter(|token| !token.is_empty())
+            .map(str::to_owned)
+    }
 }
 
 #[cfg(test)]
@@ -235,6 +245,29 @@ mod tests {
         test_file.cleanup();
     }
 
+    #[tokio::test]
+    async fn test_secrets_file_watcher_reads_nvcf_api_token() {
+        let test_file = TestSecretFile::new();
+        let creds_with_token = r#"{
+            "nvcfApiToken": "  bearer-token  ",
+            "kv": {
+                "cassandra": {
+                    "username": "test_username",
+                    "password": "test_password"
+                }
+            }
+        }"#;
+        std::fs::create_dir_all(test_file.path.parent().unwrap()).unwrap();
+        fs::write(&test_file.path, creds_with_token).unwrap();
+        let secrets_file_watcher = SecretFileWatcher::new(&test_file.path).await.unwrap();
+
+        assert_eq!(
+            secrets_file_watcher.get_nvcf_api_token().as_deref(),
+            Some("bearer-token")
+        );
+        test_file.cleanup();
+    }
+
     #[tokio::test]
     async fn test_secrets_file_watcher_with_invalid_file() {
         let test_file = TestSecretFile::new();

From f05d143c11300134d0af28600bbe7556dc396b2f Mon Sep 17 00:00:00 2001
From: Bora Oztekin <boztekin@nvidia.com>
Date: Fri, 24 Jul 2026 19:11:55 +0000
Subject: [PATCH 5/6] Allow autoscaler auth with Vault token

Signed-off-by: Bora Oztekin <boztekin@nvidia.com>
---
 deploy/stacks/self-managed/global.yaml.gotmpl        |  8 ++++++++
 .../helmfile.d/03-observability.yaml.gotmpl          | 12 ++++++++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index 53356a367..4744aeb75 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -603,6 +603,10 @@ stateMetrics:
 {{- if not (kindIs "map" $functionAutoscalerNvcfApiValues) }}
 {{- fail "functionAutoscaler.nvcfApi must be a map" }}
 {{- end }}
+{{- $functionAutoscalerVaultValues := (index $functionAutoscalerValues "vault") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerVaultValues) }}
+{{- fail "functionAutoscaler.vault must be a map" }}
+{{- end }}
 {{- $functionAutoscalerCassandraValues := (index $functionAutoscalerValues "cassandra") | default dict }}
 {{- if not (kindIs "map" $functionAutoscalerCassandraValues) }}
 {{- fail "functionAutoscaler.cassandra must be a map" }}
@@ -634,6 +638,10 @@ functionautoscaler:
     {{- with (index $functionAutoscalerImageValues "pullPolicy") }}
     pullPolicy: {{ . | quote }}
     {{- end }}
+  {{- with $functionAutoscalerVaultValues }}
+  vault:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   {{- with include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) }}
   {{- . | nindent 2 }}
   {{- end }}
diff --git a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
index 5e98e33f8..86c310f82 100644
--- a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
+++ b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl
@@ -61,11 +61,19 @@ environments:
 {{- if not (kindIs "map" $functionAutoscalerNvcfApiValues) }}
 {{- fail "functionAutoscaler.nvcfApi must be a map" }}
 {{- end }}
+{{- $functionAutoscalerVaultValues := (index $functionAutoscalerValues "vault") | default dict }}
+{{- if not (kindIs "map" $functionAutoscalerVaultValues) }}
+{{- fail "functionAutoscaler.vault must be a map" }}
+{{- end }}
 {{- $functionAutoscalerEnabled := (index $functionAutoscalerValues "enabled") | default false }}
 {{- $functionAutoscalerChartVersion := (index $functionAutoscalerValues "chartVersion") | default "" }}
 {{- $functionAutoscalerTsdbURL := (index $functionAutoscalerValues "timeseriesDbUrl") | default (index $functionAutoscalerTimeseriesDbValues "url") | default "" }}
 {{- $functionAutoscalerOauthURL := (index $functionAutoscalerValues "oauth2TokenApiAddress") | default (index $functionAutoscalerNvcfApiValues "oauth2TokenApiAddress") | default "" }}
 {{- $functionAutoscalerAuthEnabled := and (hasKey $functionAutoscalerNvcfApiValues "disableAuth") (not (index $functionAutoscalerNvcfApiValues "disableAuth")) }}
+{{- $functionAutoscalerVaultEnabled := true }}
+{{- if hasKey $functionAutoscalerVaultValues "enabled" }}
+{{- $functionAutoscalerVaultEnabled = index $functionAutoscalerVaultValues "enabled" }}
+{{- end }}
 {{- if and $functionAutoscalerEnabled (eq $mode "disabled") }}
 {{- fail "functionAutoscaler.enabled requires observability.mode to be install or existing" }}
 {{- end }}
@@ -78,8 +86,8 @@ environments:
 {{- if and $functionAutoscalerEnabled $existingMode (not $functionAutoscalerTsdbURL) }}
 {{- fail "functionAutoscaler.timeseriesDbUrl or functionAutoscaler.timeseriesDb.url is required when observability.mode=existing" }}
 {{- end }}
-{{- if and $functionAutoscalerEnabled $functionAutoscalerAuthEnabled (not $functionAutoscalerOauthURL) }}
-{{- fail "functionAutoscaler.nvcfApi.oauth2TokenApiAddress is required when functionAutoscaler.enabled=true and nvcfApi.disableAuth=false" }}
+{{- if and $functionAutoscalerEnabled $functionAutoscalerAuthEnabled (not $functionAutoscalerOauthURL) (not $functionAutoscalerVaultEnabled) }}
+{{- fail "functionAutoscaler.nvcfApi.oauth2TokenApiAddress is required when functionAutoscaler.enabled=true, nvcfApi.disableAuth=false, and functionAutoscaler.vault.enabled=false" }}
 {{- end }}
 
 repositories:

From 44af5efe01710c629038c0f3f0b703bb96748148 Mon Sep 17 00:00:00 2001
From: Bora Oztekin <boztekin@nvidia.com>
Date: Fri, 24 Jul 2026 23:27:39 +0000
Subject: [PATCH 6/6] fix(function-autoscaler): gate static API token template

---
 .../scripts/test-render-static-token.sh       | 80 +++++++++++++++++++
 .../configmap-vault-agent-template.yaml       | 10 ++-
 deploy/helm/function-autoscaler/values.yaml   |  7 ++
 .../vault-agent-templates/secrets.json.tmpl   |  4 +-
 4 files changed, 97 insertions(+), 4 deletions(-)
 create mode 100755 deploy/helm/function-autoscaler/scripts/test-render-static-token.sh

diff --git a/deploy/helm/function-autoscaler/scripts/test-render-static-token.sh b/deploy/helm/function-autoscaler/scripts/test-render-static-token.sh
new file mode 100755
index 000000000..909635663
--- /dev/null
+++ b/deploy/helm/function-autoscaler/scripts/test-render-static-token.sh
@@ -0,0 +1,80 @@
+#!/bin/sh
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+chart_dir=${CHART_DIR:-.}
+release_name=${RELEASE_NAME:-function-autoscaler}
+namespace=${NAMESPACE:-nvcf}
+tmpdir=$(mktemp -d)
+
+cleanup() {
+  rm -rf "$tmpdir"
+}
+trap cleanup EXIT
+
+export HELM_CONFIG_HOME=${HELM_CONFIG_HOME:-"$tmpdir/helm-config"}
+export HELM_CACHE_HOME=${HELM_CACHE_HOME:-"$tmpdir/helm-cache"}
+export HELM_DATA_HOME=${HELM_DATA_HOME:-"$tmpdir/helm-data"}
+
+render() {
+  helm template "$release_name" "$chart_dir" \
+    --namespace "$namespace" \
+    --values "$chart_dir/values.yaml" \
+    --set functionautoscaler.image.repository=rs-autoscaler \
+    "$@"
+}
+
+assert_contains() {
+  file=$1
+  pattern=$2
+
+  if ! grep -Fq -- "$pattern" "$file"; then
+    echo "expected rendered output to contain: $pattern" >&2
+    return 1
+  fi
+}
+
+assert_not_contains() {
+  file=$1
+  pattern=$2
+
+  if grep -Fq -- "$pattern" "$file"; then
+    echo "expected rendered output not to contain: $pattern" >&2
+    return 1
+  fi
+}
+
+default_render="$tmpdir/default.yaml"
+render > "$default_render"
+assert_contains "$default_render" "secrets.json.tmpl: |-"
+assert_not_contains "$default_render" "nvcfApiToken"
+assert_not_contains "$default_render" "NVCF_API_TOKEN_BLOCK"
+assert_not_contains "$default_render" "services/nvcf-api/jwt/sign/nvcf-autoscaler-service"
+
+static_token_render="$tmpdir/static-token.yaml"
+render --set functionautoscaler.vault.staticNvcfApiToken.enabled=true > "$static_token_render"
+assert_contains "$static_token_render" "services/nvcf-api/jwt/sign/nvcf-autoscaler-service"
+assert_contains "$static_token_render" '"nvcfApiToken": "{{ .Data.token }}",'
+assert_not_contains "$static_token_render" "NVCF_API_TOKEN_BLOCK"
+
+custom_path_render="$tmpdir/custom-path.yaml"
+render \
+  --set functionautoscaler.vault.staticNvcfApiToken.enabled=true \
+  --set functionautoscaler.vault.staticNvcfApiToken.path=services/custom/jwt/sign/autoscaler \
+  > "$custom_path_render"
+assert_contains "$custom_path_render" "services/custom/jwt/sign/autoscaler"
+assert_not_contains "$custom_path_render" "services/nvcf-api/jwt/sign/nvcf-autoscaler-service"
diff --git a/deploy/helm/function-autoscaler/templates/configmap-vault-agent-template.yaml b/deploy/helm/function-autoscaler/templates/configmap-vault-agent-template.yaml
index dce42cfbd..115968718 100644
--- a/deploy/helm/function-autoscaler/templates/configmap-vault-agent-template.yaml
+++ b/deploy/helm/function-autoscaler/templates/configmap-vault-agent-template.yaml
@@ -14,6 +14,14 @@
 # limitations under the License.
 
 {{ if .Values.functionautoscaler.vault.enabled }}
+{{- $open := "{{" -}}
+{{- $close := "}}" -}}
+{{- $staticTokenMarker := printf "%s/* NVCF_API_TOKEN_BLOCK */%s" $open $close -}}
+{{- $staticTokenBlock := "" -}}
+{{- if .Values.functionautoscaler.vault.staticNvcfApiToken.enabled -}}
+{{- $staticTokenPath := required "A valid static NVCF API token path (.Values.functionautoscaler.vault.staticNvcfApiToken.path) is required when staticNvcfApiToken.enabled=true!" .Values.functionautoscaler.vault.staticNvcfApiToken.path -}}
+{{- $staticTokenBlock = printf "%s- with secret %q %s\n  \"nvcfApiToken\": \"%s .Data.token %s\",\n  %s- end %s" $open $staticTokenPath $close $open $close $open $close -}}
+{{- end }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -23,5 +31,5 @@ metadata:
     {{- include "function-autoscaler.labels" . | nindent 4 }}
 data:
   secrets.json.tmpl: |-
-{{ .Files.Get "vault-agent-templates/secrets.json.tmpl" | indent 4 }}
+{{ replace $staticTokenMarker $staticTokenBlock (.Files.Get "vault-agent-templates/secrets.json.tmpl") | indent 4 }}
 {{- end }}
diff --git a/deploy/helm/function-autoscaler/values.yaml b/deploy/helm/function-autoscaler/values.yaml
index d15a2b63f..15822f27c 100644
--- a/deploy/helm/function-autoscaler/values.yaml
+++ b/deploy/helm/function-autoscaler/values.yaml
@@ -51,6 +51,13 @@ functionautoscaler:
     role: nvcf-autoscaler-service
     authPath: auth/jwt
 
+    # Enable only for deployments that provision the OpenBao JWT signing path.
+    # When disabled, the app falls back to OAuth2 credentials or auth-disabled
+    # mode without the Vault Agent dereferencing this optional secret.
+    staticNvcfApiToken:
+      enabled: false
+      path: services/nvcf-api/jwt/sign/nvcf-autoscaler-service
+
   podAnnotations: {}
   podLabels: {}
   podSecurityContext: {}
diff --git a/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl b/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl
index 47e8735d3..b4de8acd2 100644
--- a/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl
+++ b/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl
@@ -7,9 +7,7 @@
   corresponding app integration is enabled and wired to a secret path.
 */}}
 {
-  {{- with secret `services/nvcf-api/jwt/sign/nvcf-autoscaler-service` }}
-  "nvcfApiToken": "{{ .Data.token }}",
-  {{- end }}
+  {{/* NVCF_API_TOKEN_BLOCK */}}
   "kv": {
     "cassandra": {
       {{- with secret "services/nvcf-autoscaler-service/kv/data/cassandra/creds" }}