diff --git a/deploy/helm/function-autoscaler/scripts/test-render-static-token.sh b/deploy/helm/function-autoscaler/scripts/test-render-static-token.sh new file mode 100755 index 000000000..909635663 --- /dev/null +++ b/deploy/helm/function-autoscaler/scripts/test-render-static-token.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +chart_dir=${CHART_DIR:-.} +release_name=${RELEASE_NAME:-function-autoscaler} +namespace=${NAMESPACE:-nvcf} +tmpdir=$(mktemp -d) + +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +export HELM_CONFIG_HOME=${HELM_CONFIG_HOME:-"$tmpdir/helm-config"} +export HELM_CACHE_HOME=${HELM_CACHE_HOME:-"$tmpdir/helm-cache"} +export HELM_DATA_HOME=${HELM_DATA_HOME:-"$tmpdir/helm-data"} + +render() { + helm template "$release_name" "$chart_dir" \ + --namespace "$namespace" \ + --values "$chart_dir/values.yaml" \ + --set functionautoscaler.image.repository=rs-autoscaler \ + "$@" +} + +assert_contains() { + file=$1 + pattern=$2 + + if ! grep -Fq -- "$pattern" "$file"; then + echo "expected rendered output to contain: $pattern" >&2 + return 1 + fi +} + +assert_not_contains() { + file=$1 + pattern=$2 + + if grep -Fq -- "$pattern" "$file"; then + echo "expected rendered output not to contain: $pattern" >&2 + return 1 + fi +} + +default_render="$tmpdir/default.yaml" +render > "$default_render" +assert_contains "$default_render" "secrets.json.tmpl: |-" +assert_not_contains "$default_render" "nvcfApiToken" +assert_not_contains "$default_render" "NVCF_API_TOKEN_BLOCK" +assert_not_contains "$default_render" "services/nvcf-api/jwt/sign/nvcf-autoscaler-service" + +static_token_render="$tmpdir/static-token.yaml" +render --set functionautoscaler.vault.staticNvcfApiToken.enabled=true > "$static_token_render" +assert_contains "$static_token_render" "services/nvcf-api/jwt/sign/nvcf-autoscaler-service" +assert_contains "$static_token_render" '"nvcfApiToken": "{{ .Data.token }}",' +assert_not_contains "$static_token_render" "NVCF_API_TOKEN_BLOCK" + +custom_path_render="$tmpdir/custom-path.yaml" +render \ + --set functionautoscaler.vault.staticNvcfApiToken.enabled=true \ + --set functionautoscaler.vault.staticNvcfApiToken.path=services/custom/jwt/sign/autoscaler \ + > "$custom_path_render" +assert_contains "$custom_path_render" "services/custom/jwt/sign/autoscaler" +assert_not_contains "$custom_path_render" "services/nvcf-api/jwt/sign/nvcf-autoscaler-service" diff --git a/deploy/helm/function-autoscaler/templates/configmap-vault-agent-template.yaml b/deploy/helm/function-autoscaler/templates/configmap-vault-agent-template.yaml index dce42cfbd..115968718 100644 --- a/deploy/helm/function-autoscaler/templates/configmap-vault-agent-template.yaml +++ b/deploy/helm/function-autoscaler/templates/configmap-vault-agent-template.yaml @@ -14,6 +14,14 @@ # limitations under the License. {{ if .Values.functionautoscaler.vault.enabled }} +{{- $open := "{{" -}} +{{- $close := "}}" -}} +{{- $staticTokenMarker := printf "%s/* NVCF_API_TOKEN_BLOCK */%s" $open $close -}} +{{- $staticTokenBlock := "" -}} +{{- if .Values.functionautoscaler.vault.staticNvcfApiToken.enabled -}} +{{- $staticTokenPath := required "A valid static NVCF API token path (.Values.functionautoscaler.vault.staticNvcfApiToken.path) is required when staticNvcfApiToken.enabled=true!" .Values.functionautoscaler.vault.staticNvcfApiToken.path -}} +{{- $staticTokenBlock = printf "%s- with secret %q %s\n \"nvcfApiToken\": \"%s .Data.token %s\",\n %s- end %s" $open $staticTokenPath $close $open $close $open $close -}} +{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -23,5 +31,5 @@ metadata: {{- include "function-autoscaler.labels" . | nindent 4 }} data: secrets.json.tmpl: |- -{{ .Files.Get "vault-agent-templates/secrets.json.tmpl" | indent 4 }} +{{ replace $staticTokenMarker $staticTokenBlock (.Files.Get "vault-agent-templates/secrets.json.tmpl") | indent 4 }} {{- end }} diff --git a/deploy/helm/function-autoscaler/values.yaml b/deploy/helm/function-autoscaler/values.yaml index db8eec8f7..15822f27c 100644 --- a/deploy/helm/function-autoscaler/values.yaml +++ b/deploy/helm/function-autoscaler/values.yaml @@ -51,6 +51,13 @@ functionautoscaler: role: nvcf-autoscaler-service authPath: auth/jwt + # Enable only for deployments that provision the OpenBao JWT signing path. + # When disabled, the app falls back to OAuth2 credentials or auth-disabled + # mode without the Vault Agent dereferencing this optional secret. + staticNvcfApiToken: + enabled: false + path: services/nvcf-api/jwt/sign/nvcf-autoscaler-service + podAnnotations: {} podLabels: {} podSecurityContext: {} @@ -97,6 +104,9 @@ functionautoscaler: - name: probe containerPort: 8181 protocol: TCP + - name: metrics + containerPort: 41338 + protocol: TCP # Stable service DNS. Override for multiple releases in one namespace. service: @@ -110,6 +120,10 @@ functionautoscaler: port: 8181 targetPort: probe protocol: TCP + - name: metrics + port: 41338 + targetPort: metrics + protocol: TCP # Additional volumes on the output Deployment definition. # The projected service account token lets the Vault Agent authenticate to @@ -140,3 +154,16 @@ functionautoscaler: # Read the secrets file rendered by the Vault Agent. Override this value # when mounting secrets manually. SECRETS_PATH: "/vault/secrets/secrets.json" + + # Self-managed in-cluster defaults. Consuming stacks should override these + # only when service names, namespaces, or auth mode differ. + CASSANDRA__CONTACT_POINTS: "cassandra.cassandra-system.svc.cluster.local" + CASSANDRA__PORT: "9042" + CASSANDRA__KEYSPACE: "nvcf_autoscaler" + CASSANDRA__SSL__ENABLED: "false" + TIMESERIES_DB__TIMESERIES_DB_URL: "http://vmsingle.monitoring.svc.cluster.local:8428" + TIMESERIES_DB__DISABLE_AUTH: "true" + TIMESERIES_DB__ENV: "stg" + TIMESERIES_DB__IGNORE_ENV: "true" + NVCF_API__NVCF_API_GRPC_ADDRESS: "http://api.nvcf.svc.cluster.local:9090" + NVCF_API__DISABLE_AUTH: "true" diff --git a/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl b/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl index 11a032a73..b4de8acd2 100644 --- a/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl +++ b/deploy/helm/function-autoscaler/vault-agent-templates/secrets.json.tmpl @@ -7,6 +7,7 @@ corresponding app integration is enabled and wired to a secret path. */}} { + {{/* NVCF_API_TOKEN_BLOCK */}} "kv": { "cassandra": { {{- with secret "services/nvcf-autoscaler-service/kv/data/cassandra/creds" }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml index 1605cd7ee..13ad72b7d 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml @@ -45,26 +45,35 @@ controlPlane: selector: app.kubernetes.io/instance: state-metrics app.kubernetes.io/name: helm-nvcf-state-metrics + - name: function-autoscaler + enabled: true + selector: + app.kubernetes.io/instance: function-autoscaler + app.kubernetes.io/name: helm-nvcf-function-autoscaler - name: grpc-proxy enabled: true selector: app.kubernetes.io/instance: grpc-proxy - name: api enabled: true + path: /actuator/prometheus + port: management-port selector: app.kubernetes.io/name: helm-nvcf-api - name: sis enabled: true namespaces: - sis + path: /actuator/prometheus + port: management-port selector: app.kubernetes.io/name: helm-nvcf-sis - name: llm-api-gateway enabled: true selector: app.kubernetes.io/instance: llm-api-gateway - # Off until the Invocation Service exporter is enabled. - name: invocation-service - enabled: false + enabled: true selector: app.kubernetes.io/instance: invocation-service + app.kubernetes.io/name: helm-nvcf-invocation-service diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index a73d69869..608da42cc 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -125,6 +125,31 @@ global: # global.observability.metrics.enabled enabled: false +# Optional observability runtime (stage 03). `install` deploys the complete +# managed metrics path; `existing` assumes a customer-owned backend. +observability: + mode: disabled + namespace: monitoring + +prometheusOperatorCrds: + version: "30.0.0" + +opentelemetryOperator: + version: "0.114.1" + +victoriaMetrics: + version: "0.38.0" + +controlPlaneCollector: + version: "0.1.0" + +defaultMonitors: + version: "0.1.0" + # External observability mode may set enabled: true to install the NVCF-owned + # default monitors against customer-managed scrape infrastructure. + controlPlane: + enabled: true + accounts: limits: maxFunctions: 10 @@ -260,6 +285,19 @@ addons: tolerations: [] affinity: {} +# Function Autoscaler workload (stage 03). Disabled by default; requires the +# stage-01 Cassandra/OpenBao migrations and a Prometheus-compatible TSDB. +functionAutoscaler: + enabled: false + chartVersion: "0.1.0" + image: + tag: "1.18.6" + pullPolicy: IfNotPresent + # Optional overrides for chart-owned defaults. + timeseriesDb: {} + nvcfApi: {} + cassandra: {} + stateMetrics: enabled: false # Disable ServiceMonitor for environments without Prometheus Operator diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index df17ad4c7..4744aeb75 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -336,6 +336,8 @@ api: NVCF_REGISTRIES_RECOGNIZED_RESOURCE_NGC_OAUTH2_GROUP_SCOPE: "ngc" # Observability MANAGEMENT_TRACING_ENABLED: {{ .Values.global.observability.tracing.enabled | quote }} + MANAGEMENT_PROMETHEUS_METRICS_EXPORT_ENABLED: {{ .Values.global.observability.metrics.enabled | quote }} + MANAGEMENT_METRICS_ENABLE_ALL: {{ .Values.global.observability.metrics.enabled | quote }} {{- if .Values.global.observability.tracing.enabled }} MANAGEMENT_OTLP_TRACING_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}/v1/traces" {{- end }} @@ -565,7 +567,7 @@ adminIssuerProxy: path: "/v1/admin/keys" stateMetrics: - enabled: false + enabled: {{ dig "stateMetrics" "enabled" false .Values }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -583,6 +585,106 @@ stateMetrics: serviceMonitor: enabled: false +{{- $observabilityNamespace := dig "observability" "namespace" "monitoring" .Values }} +{{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" $observabilityNamespace .Values }} +{{- $functionAutoscalerValues := (index .Values "functionAutoscaler") | default (index .Values "functionautoscaler") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerValues) }} +{{- fail "functionAutoscaler must be a map" }} +{{- end }} +{{- $functionAutoscalerImageValues := (index $functionAutoscalerValues "image") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerImageValues) }} +{{- fail "functionAutoscaler.image must be a map" }} +{{- end }} +{{- $functionAutoscalerTimeseriesDbValues := (index $functionAutoscalerValues "timeseriesDb") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerTimeseriesDbValues) }} +{{- fail "functionAutoscaler.timeseriesDb must be a map" }} +{{- end }} +{{- $functionAutoscalerNvcfApiValues := (index $functionAutoscalerValues "nvcfApi") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerNvcfApiValues) }} +{{- fail "functionAutoscaler.nvcfApi must be a map" }} +{{- end }} +{{- $functionAutoscalerVaultValues := (index $functionAutoscalerValues "vault") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerVaultValues) }} +{{- fail "functionAutoscaler.vault must be a map" }} +{{- end }} +{{- $functionAutoscalerCassandraValues := (index $functionAutoscalerValues "cassandra") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerCassandraValues) }} +{{- fail "functionAutoscaler.cassandra must be a map" }} +{{- end }} +{{- $functionAutoscalerCassandraSslValues := (index $functionAutoscalerCassandraValues "ssl") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerCassandraSslValues) }} +{{- fail "functionAutoscaler.cassandra.ssl must be a map" }} +{{- end }} +{{- $functionAutoscalerTsdbURL := (index $functionAutoscalerValues "timeseriesDbUrl") | default (index $functionAutoscalerTimeseriesDbValues "url") | default "" }} +{{- if and (not $functionAutoscalerTsdbURL) (ne $victoriaMetricsNamespace "monitoring") }} +{{- $functionAutoscalerTsdbURL = printf "http://vmsingle.%s.svc.cluster.local:8428" $victoriaMetricsNamespace }} +{{- end }} +{{- $functionAutoscalerCassandraContactPoints := (index $functionAutoscalerValues "cassandraContactPoints") | default (index $functionAutoscalerCassandraValues "contactPoints") | default "" }} +{{- $functionAutoscalerGrpcAddress := (index $functionAutoscalerValues "nvcfApiGrpcAddress") | default (index $functionAutoscalerNvcfApiValues "grpcAddress") | default "" }} +{{- $functionAutoscalerOauthURL := (index $functionAutoscalerValues "oauth2TokenApiAddress") | default (index $functionAutoscalerNvcfApiValues "oauth2TokenApiAddress") | default "" }} +{{- $functionAutoscalerImageRepository := (index $functionAutoscalerImageValues "repository") | default (printf "%s/nvcf-autoscaler-service" .Values.global.image.repository) }} +{{- $functionAutoscalerHasEnvOverrides := or $functionAutoscalerTsdbURL $functionAutoscalerCassandraContactPoints $functionAutoscalerGrpcAddress $functionAutoscalerOauthURL (hasKey $functionAutoscalerTimeseriesDbValues "disableAuth") (hasKey $functionAutoscalerTimeseriesDbValues "env") (hasKey $functionAutoscalerTimeseriesDbValues "ignoreEnv") (hasKey $functionAutoscalerCassandraValues "port") (hasKey $functionAutoscalerCassandraValues "keyspace") (hasKey $functionAutoscalerCassandraSslValues "enabled") (hasKey $functionAutoscalerNvcfApiValues "disableAuth") }} +functionautoscaler: + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} + {{- end }} + image: + registry: {{ .Values.global.image.registry }} + repository: {{ $functionAutoscalerImageRepository }} + {{- with (index $functionAutoscalerImageValues "tag") }} + tag: {{ . | quote }} + {{- end }} + {{- with (index $functionAutoscalerImageValues "pullPolicy") }} + pullPolicy: {{ . | quote }} + {{- end }} + {{- with $functionAutoscalerVaultValues }} + vault: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) }} + {{- . | nindent 2 }} + {{- end }} + {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} + {{- . | nindent 2 }} + {{- end }} + {{- if $functionAutoscalerHasEnvOverrides }} + env: + {{- with $functionAutoscalerTsdbURL }} + TIMESERIES_DB__TIMESERIES_DB_URL: {{ $functionAutoscalerTsdbURL | quote }} + {{- end }} + {{- if hasKey $functionAutoscalerTimeseriesDbValues "disableAuth" }} + TIMESERIES_DB__DISABLE_AUTH: {{ index $functionAutoscalerTimeseriesDbValues "disableAuth" | quote }} + {{- end }} + {{- with (index $functionAutoscalerTimeseriesDbValues "env") }} + TIMESERIES_DB__ENV: {{ . | quote }} + {{- end }} + {{- if hasKey $functionAutoscalerTimeseriesDbValues "ignoreEnv" }} + TIMESERIES_DB__IGNORE_ENV: {{ index $functionAutoscalerTimeseriesDbValues "ignoreEnv" | quote }} + {{- end }} + {{- with $functionAutoscalerCassandraContactPoints }} + CASSANDRA__CONTACT_POINTS: {{ $functionAutoscalerCassandraContactPoints | quote }} + {{- end }} + {{- with (index $functionAutoscalerCassandraValues "port") }} + CASSANDRA__PORT: {{ . | quote }} + {{- end }} + {{- with (index $functionAutoscalerCassandraValues "keyspace") }} + CASSANDRA__KEYSPACE: {{ . | quote }} + {{- end }} + {{- if hasKey $functionAutoscalerCassandraSslValues "enabled" }} + CASSANDRA__SSL__ENABLED: {{ index $functionAutoscalerCassandraSslValues "enabled" | quote }} + {{- end }} + {{- with $functionAutoscalerGrpcAddress }} + NVCF_API__NVCF_API_GRPC_ADDRESS: {{ $functionAutoscalerGrpcAddress | quote }} + {{- end }} + {{- with $functionAutoscalerOauthURL }} + NVCF_API__OAUTH2_TOKEN_API_ADDRESS: {{ $functionAutoscalerOauthURL | quote }} + {{- end }} + {{- if hasKey $functionAutoscalerNvcfApiValues "disableAuth" }} + NVCF_API__DISABLE_AUTH: {{ index $functionAutoscalerNvcfApiValues "disableAuth" | quote }} + {{- end }} + {{- end }} + reval: fullnameOverride: reval {{- if .Values.global.imagePullSecrets }} diff --git a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl index 0370fb678..86c310f82 100644 --- a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl @@ -6,6 +6,90 @@ environments: --- +{{- $mode := dig "observability" "mode" "disabled" .Values }} +{{- if not (has $mode (list "install" "existing" "disabled")) }} +{{- fail (printf "observability.mode must be install, existing, or disabled, got %q" $mode) }} +{{- end }} +{{- $installMode := eq $mode "install" }} +{{- $existingMode := eq $mode "existing" }} +{{- $observabilityNamespace := dig "observability" "namespace" "monitoring" .Values }} +{{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" $observabilityNamespace .Values }} +{{- $prometheusOperatorCrdsValues := dig "prometheusOperatorCrds" "values" dict .Values }} +{{- $opentelemetryOperatorValues := dig "opentelemetryOperator" "values" dict .Values }} +{{- $victoriaMetricsValues := dig "victoriaMetrics" "values" dict .Values }} +{{- $controlPlaneCollectorValues := dig "controlPlaneCollector" "values" dict .Values }} +{{- $controlPlaneCollectorRoot := dig "controlPlaneCollector" dict .Values }} +{{- $defaultMonitorsValues := dig "defaultMonitors" dict .Values }} +{{- $controlPlaneDefaultMonitorsEnabled := dig "controlPlane" "enabled" true $defaultMonitorsValues }} +{{- $defaultMonitorsEnabled := or $installMode (and $existingMode (dig "enabled" false $defaultMonitorsValues)) }} +{{- $victoriaMetricsEnabled := $installMode }} +{{- $controlPlaneCollectorEnabled := $installMode }} +{{- $prometheusOperatorCrdsEnabled := $installMode }} +{{- $opentelemetryOperatorEnabled := $installMode }} +{{- $bundledRemoteWriteEndpoint := printf "http://vmsingle.%s.svc.cluster.local:8428/api/v1/write" $victoriaMetricsNamespace }} +{{- $configuredRemoteWriteEndpoint := dig "config" "exporters" "prometheusremotewrite" "endpoint" "" $controlPlaneCollectorRoot }} +{{- if and $controlPlaneCollectorEnabled $victoriaMetricsEnabled $configuredRemoteWriteEndpoint (ne $configuredRemoteWriteEndpoint $bundledRemoteWriteEndpoint) }} +{{- fail (printf "controlPlaneCollector.config.exporters.prometheusremotewrite.endpoint must be %q when victoriaMetrics.enabled=true, got %q" $bundledRemoteWriteEndpoint $configuredRemoteWriteEndpoint) }} +{{- end }} +{{- if and $installMode (hasKey (dig "victoriaMetrics" dict .Values) "enabled") (not (dig "victoriaMetrics" "enabled" true .Values)) }} +{{- fail "observability.mode=install owns the managed metrics backend; remove victoriaMetrics.enabled=false or use observability.mode=existing with an external TSDB" }} +{{- end }} +{{- if and $installMode (hasKey (dig "controlPlaneCollector" dict .Values) "enabled") (not (dig "controlPlaneCollector" "enabled" true .Values)) }} +{{- fail "observability.mode=install owns the control-plane collector; remove controlPlaneCollector.enabled=false or use observability.mode=existing" }} +{{- end }} +{{- if and $installMode (hasKey (dig "defaultMonitors" dict .Values) "enabled") (not (dig "defaultMonitors" "enabled" true .Values)) }} +{{- fail "observability.mode=install owns the default monitor set; remove defaultMonitors.enabled=false or use observability.mode=existing" }} +{{- end }} +{{- if and $installMode (not $controlPlaneDefaultMonitorsEnabled) }} +{{- fail "observability.mode=install requires defaultMonitors.controlPlane.enabled=true" }} +{{- end }} +{{- if and $installMode (hasKey (dig "prometheusOperatorCrds" dict .Values) "enabled") (not (dig "prometheusOperatorCrds" "enabled" true .Values)) }} +{{- fail "observability.mode=install owns Prometheus Operator CRDs; remove prometheusOperatorCrds.enabled=false or use observability.mode=existing" }} +{{- end }} +{{- if and $installMode (hasKey (dig "opentelemetryOperator" dict .Values) "enabled") (not (dig "opentelemetryOperator" "enabled" true .Values)) }} +{{- fail "observability.mode=install owns the OpenTelemetry Operator; remove opentelemetryOperator.enabled=false or use observability.mode=existing" }} +{{- end }} +{{- $functionAutoscalerValues := (index .Values "functionAutoscaler") | default (index .Values "functionautoscaler") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerValues) }} +{{- fail "functionAutoscaler must be a map" }} +{{- end }} +{{- $functionAutoscalerTimeseriesDbValues := (index $functionAutoscalerValues "timeseriesDb") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerTimeseriesDbValues) }} +{{- fail "functionAutoscaler.timeseriesDb must be a map" }} +{{- end }} +{{- $functionAutoscalerNvcfApiValues := (index $functionAutoscalerValues "nvcfApi") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerNvcfApiValues) }} +{{- fail "functionAutoscaler.nvcfApi must be a map" }} +{{- end }} +{{- $functionAutoscalerVaultValues := (index $functionAutoscalerValues "vault") | default dict }} +{{- if not (kindIs "map" $functionAutoscalerVaultValues) }} +{{- fail "functionAutoscaler.vault must be a map" }} +{{- end }} +{{- $functionAutoscalerEnabled := (index $functionAutoscalerValues "enabled") | default false }} +{{- $functionAutoscalerChartVersion := (index $functionAutoscalerValues "chartVersion") | default "" }} +{{- $functionAutoscalerTsdbURL := (index $functionAutoscalerValues "timeseriesDbUrl") | default (index $functionAutoscalerTimeseriesDbValues "url") | default "" }} +{{- $functionAutoscalerOauthURL := (index $functionAutoscalerValues "oauth2TokenApiAddress") | default (index $functionAutoscalerNvcfApiValues "oauth2TokenApiAddress") | default "" }} +{{- $functionAutoscalerAuthEnabled := and (hasKey $functionAutoscalerNvcfApiValues "disableAuth") (not (index $functionAutoscalerNvcfApiValues "disableAuth")) }} +{{- $functionAutoscalerVaultEnabled := true }} +{{- if hasKey $functionAutoscalerVaultValues "enabled" }} +{{- $functionAutoscalerVaultEnabled = index $functionAutoscalerVaultValues "enabled" }} +{{- end }} +{{- if and $functionAutoscalerEnabled (eq $mode "disabled") }} +{{- fail "functionAutoscaler.enabled requires observability.mode to be install or existing" }} +{{- end }} +{{- if and $functionAutoscalerEnabled (not $functionAutoscalerChartVersion) }} +{{- fail "functionAutoscaler.chartVersion must be set to the published helm-nvcf-function-autoscaler chart version when functionAutoscaler.enabled=true" }} +{{- end }} +{{- if and $functionAutoscalerEnabled (not (dig "stateMetrics" "enabled" false .Values)) }} +{{- fail "functionAutoscaler.enabled requires stateMetrics.enabled=true" }} +{{- end }} +{{- if and $functionAutoscalerEnabled $existingMode (not $functionAutoscalerTsdbURL) }} +{{- fail "functionAutoscaler.timeseriesDbUrl or functionAutoscaler.timeseriesDb.url is required when observability.mode=existing" }} +{{- end }} +{{- if and $functionAutoscalerEnabled $functionAutoscalerAuthEnabled (not $functionAutoscalerOauthURL) (not $functionAutoscalerVaultEnabled) }} +{{- fail "functionAutoscaler.nvcfApi.oauth2TokenApiAddress is required when functionAutoscaler.enabled=true, nvcfApi.disableAuth=false, and functionAutoscaler.vault.enabled=false" }} +{{- end }} + repositories: - name: nvcf url: {{ .Values.global.helm.sources.registry }}/{{ .Values.global.helm.sources.repository }} @@ -31,6 +115,114 @@ templates: releases: + {{- if $prometheusOperatorCrdsEnabled }} + - name: prometheus-operator-crds + chart: nvcf/prometheus-operator-crds + version: {{ dig "prometheusOperatorCrds" "version" "30.0.0" .Values | quote }} + namespace: {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }} + {{- with $prometheusOperatorCrdsValues }} + values: + - {{- toYaml . | nindent 8 }} + {{- end }} + labels: + release-group: observability + {{- end }} + + {{- if $opentelemetryOperatorEnabled }} + - name: opentelemetry-operator + chart: nvcf/opentelemetry-operator + version: {{ dig "opentelemetryOperator" "version" "0.114.1" .Values | quote }} + namespace: {{ dig "opentelemetryOperator" "namespace" $observabilityNamespace .Values }} + {{- if $prometheusOperatorCrdsEnabled }} + needs: + - {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}/prometheus-operator-crds + {{- end }} + {{- if or $opentelemetryOperatorValues .Values.global.imagePullSecrets }} + values: + {{- with $opentelemetryOperatorValues }} + - {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + - imagePullSecrets: + {{- toYaml .Values.global.imagePullSecrets | nindent 10 }} + {{- end }} + {{- end }} + labels: + release-group: observability + {{- end }} + + {{- if $victoriaMetricsEnabled }} + - name: victoria-metrics + chart: nvcf/victoria-metrics-single + version: {{ dig "victoriaMetrics" "version" "0.38.0" .Values | quote }} + namespace: {{ $victoriaMetricsNamespace }} + values: + - server: + fullnameOverride: vmsingle + {{- with $victoriaMetricsValues }} + - {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + - global: + imagePullSecrets: + {{- toYaml .Values.global.imagePullSecrets | nindent 12 }} + {{- end }} + labels: + release-group: observability + {{- end }} + + {{- if $controlPlaneCollectorEnabled }} + - name: control-plane-collector + chart: nvcf/nvcf-otel-collector + version: {{ dig "controlPlaneCollector" "version" "0.1.0" .Values | quote }} + namespace: {{ dig "controlPlaneCollector" "namespace" $observabilityNamespace .Values }} + {{- if or $opentelemetryOperatorEnabled $prometheusOperatorCrdsEnabled $victoriaMetricsEnabled }} + needs: + {{- if $opentelemetryOperatorEnabled }} + - {{ dig "opentelemetryOperator" "namespace" $observabilityNamespace .Values }}/opentelemetry-operator + {{- end }} + {{- if $prometheusOperatorCrdsEnabled }} + - {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}/prometheus-operator-crds + {{- end }} + {{- if $victoriaMetricsEnabled }} + - {{ $victoriaMetricsNamespace }}/victoria-metrics + {{- end }} + {{- end }} + {{- if or $controlPlaneCollectorValues (ne $victoriaMetricsNamespace "monitoring") .Values.global.imagePullSecrets }} + values: + {{- if ne $victoriaMetricsNamespace "monitoring" }} + - config: + exporters: + prometheusremotewrite: + endpoint: {{ $bundledRemoteWriteEndpoint | quote }} + {{- end }} + {{- with $controlPlaneCollectorValues }} + - {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + - imagePullSecrets: + {{- toYaml .Values.global.imagePullSecrets | nindent 10 }} + {{- end }} + {{- end }} + labels: + release-group: observability + {{- end }} + + {{- if $defaultMonitorsEnabled }} + - name: default-monitors + chart: nvcf/nvcf-default-monitors + version: {{ dig "defaultMonitors" "version" "0.1.0" .Values | quote }} + namespace: {{ dig "defaultMonitors" "namespace" $observabilityNamespace .Values }} + {{- if $prometheusOperatorCrdsEnabled }} + needs: + - {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}/prometheus-operator-crds + {{- end }} + values: + - {{- toYaml $defaultMonitorsValues | nindent 8 }} + labels: + release-group: observability + {{- end }} + # --- Observability Services --- - name: state-metrics version: 1.0.1 @@ -40,3 +232,24 @@ releases: - template: service # When enabled, this service will export realtime function information that can be # scraped by a Prometheus ServiceMonitor + + {{- if $functionAutoscalerEnabled }} + # --- Function Autoscaler workload --- + - name: function-autoscaler + chart: nvcf/helm-nvcf-function-autoscaler + version: {{ $functionAutoscalerChartVersion | quote }} + namespace: nvcf + values: + - ../global.yaml.gotmpl + - ../secrets/{{ requiredEnv "HELMFILE_ENV" }}-secrets.yaml + needs: + - nvcf/state-metrics + {{- if $victoriaMetricsEnabled }} + - {{ $victoriaMetricsNamespace }}/victoria-metrics + {{- end }} + {{- if $controlPlaneCollectorEnabled }} + - {{ dig "controlPlaneCollector" "namespace" $observabilityNamespace .Values }}/control-plane-collector + {{- end }} + labels: + release-group: observability + {{- end }} diff --git a/src/control-plane-services/function-autoscaler/crates/server/src/nvcf_api/oauth2_client.rs b/src/control-plane-services/function-autoscaler/crates/server/src/nvcf_api/oauth2_client.rs index e6756f037..50b2abd65 100644 --- a/src/control-plane-services/function-autoscaler/crates/server/src/nvcf_api/oauth2_client.rs +++ b/src/control-plane-services/function-autoscaler/crates/server/src/nvcf_api/oauth2_client.rs @@ -136,6 +136,10 @@ impl OAuth2Client { } pub async fn get_jwt_token(&self) -> Result> { + if let Some(token) = self.secrets_watcher.get_nvcf_api_token() { + return Ok(token); + } + // Check if we have a cached valid token { let cache = self.token_cache.read().await; @@ -164,6 +168,10 @@ impl OAuth2Client { secrets_watcher: &Arc, http_client: &reqwest_middleware::ClientWithMiddleware, ) -> Result> { + if let Some(token) = secrets_watcher.get_nvcf_api_token() { + return Ok(token); + } + // Get current secrets let secrets = secrets_watcher.get_config(); @@ -305,6 +313,26 @@ mod tests { (temp_dir, secrets_file) } + async fn create_test_secrets_file_with_static_token() -> (TempDir, std::path::PathBuf) { + let temp_dir = TempDir::new().unwrap(); + let secrets_file = temp_dir.path().join("secrets.json"); + + let secrets_content = serde_json::json!({ + "nvcfApiToken": "openbao-rendered-token", + "kv": { + "nvcf_api": { + "client_id": "test_client_id", + "client_secret": "test_client_secret" + } + } + }); + + fs::write(&secrets_file, secrets_content.to_string()) + .await + .unwrap(); + (temp_dir, secrets_file) + } + #[tokio::test] async fn test_cached_token() { let (_temp_dir, secrets_file) = create_test_secrets_file().await; @@ -325,6 +353,23 @@ mod tests { assert_eq!(creds.client_secret, "test_client_secret"); } + #[tokio::test] + async fn test_static_nvcf_api_token_takes_precedence() { + let (_temp_dir, secrets_file) = create_test_secrets_file_with_static_token().await; + let secrets_watcher = SecretFileWatcher::new(&secrets_file).await.unwrap(); + let oauth2_client = OAuth2Client::new( + "http://127.0.0.1:1".to_string(), + TEST_SCOPE.to_string(), + Arc::new(secrets_watcher), + ) + .unwrap(); + + let token = oauth2_client.get_jwt_token().await.unwrap(); + + assert_eq!(token, "openbao-rendered-token"); + assert!(oauth2_client.token_cache.read().await.is_none()); + } + #[tokio::test] async fn test_token_expiration() { let (_temp_dir, secrets_file) = create_test_secrets_file().await; diff --git a/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_config.rs b/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_config.rs index a72db49fb..cf5603561 100644 --- a/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_config.rs +++ b/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_config.rs @@ -19,6 +19,8 @@ use serde::Deserialize; #[derive(Debug, Deserialize, Clone)] pub struct Secrets { + #[serde(rename = "nvcfApiToken")] + pub nvcf_api_token: Option, pub kv: CredentialsData, } diff --git a/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_file_watcher.rs b/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_file_watcher.rs index 4cf47a6a4..9e79b0502 100644 --- a/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_file_watcher.rs +++ b/src/control-plane-services/function-autoscaler/crates/server/src/secrets/secrets_file_watcher.rs @@ -157,6 +157,16 @@ impl SecretFileWatcher { let guard = self.config.read().unwrap(); guard.kv.clone() } + + pub fn get_nvcf_api_token(&self) -> Option { + let guard = self.config.read().unwrap(); + guard + .nvcf_api_token + .as_deref() + .map(str::trim) + .filter(|token| !token.is_empty()) + .map(str::to_owned) + } } #[cfg(test)] @@ -235,6 +245,29 @@ mod tests { test_file.cleanup(); } + #[tokio::test] + async fn test_secrets_file_watcher_reads_nvcf_api_token() { + let test_file = TestSecretFile::new(); + let creds_with_token = r#"{ + "nvcfApiToken": " bearer-token ", + "kv": { + "cassandra": { + "username": "test_username", + "password": "test_password" + } + } + }"#; + std::fs::create_dir_all(test_file.path.parent().unwrap()).unwrap(); + fs::write(&test_file.path, creds_with_token).unwrap(); + let secrets_file_watcher = SecretFileWatcher::new(&test_file.path).await.unwrap(); + + assert_eq!( + secrets_file_watcher.get_nvcf_api_token().as_deref(), + Some("bearer-token") + ); + test_file.cleanup(); + } + #[tokio::test] async fn test_secrets_file_watcher_with_invalid_file() { let test_file = TestSecretFile::new();