diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml index 6c4264a5f..361f21942 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml @@ -46,6 +46,17 @@ spec: runAsUser: 1000 runAsGroup: 1010 fsGroup: 1010 + # Defense in depth: enforce non-root + seccomp at the pod + # level. runAsUser=1000 already guarantees non-root; setting + # runAsNonRoot=true makes the constraint explicit (kubelet + # will reject the pod if the image's USER somehow goes back + # to root). seccompProfile=RuntimeDefault blocks the + # ~40 syscalls Docker's default seccomp profile blocks + # (clock_settime, modify_ldt, …) — safe for any normal Go + # operator. + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end}} diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/operator-networkpolicy.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/operator-networkpolicy.yaml new file mode 100644 index 000000000..00e159c94 --- /dev/null +++ b/deploy/helm/nvca-operator/nvca-operator/templates/operator-networkpolicy.yaml @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# NetworkPolicy hardening for the nvca-operator pod itself. +# +# By default, in clusters with a default-deny CNI (Calico, Cilium with +# enforcement, etc.) operator egress is wide open inside the cluster. +# This template adds explicit allow rules for the three legitimate +# egress targets: +# +# 1. kube-apiserver — for CRD watches, leader-election leases, +# child-resource create/update, status writes. CIDR-based rule +# because the apiserver IP is cluster-specific; default +# 0.0.0.0/0 - cluster-internal-CIDRs covers the public apiserver +# endpoint without leaking to other pods. +# +# 2. nvsnap-server (default nvsnap-system/nvsnap-server, port 8080) — +# for L2 promote-state polls and audit calls. Namespace + label +# selector keeps it tight. +# +# 3. kube-dns — for service name resolution. Without this rule the +# operator can't resolve nvsnap-server.nvsnap-system.svc anyway. +# +# Ingress: container port 8000 (health/metrics) and 8002 (auxiliary). +# Allowed from any namespace by default (Prometheus scraping + +# kubelet probes can come from anywhere). Operators can tighten by +# overriding ingressFrom. +# +# Opt-in via .Values.networkPolicy.operator.enabled (default false) +# so existing deployments keep working unchanged. Turn on after +# verifying the rule set fits your cluster. +# +# Note: K8s NetworkPolicy semantics are "deny by default IF any +# NetworkPolicy selects the pod." Creating this policy WITHOUT a +# matching cluster CNI may have no effect (e.g. some kindnet +# configs); test in your environment before relying on it. +{{- if and (hasKey .Values "networkPolicy") (hasKey .Values.networkPolicy "operator") .Values.networkPolicy.operator.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "nvcaop.fullname" . }}-operator-egress + namespace: {{ .Release.Namespace }} + labels: + {{- include "nvcaop.labels" . | nindent 4 }} + annotations: + nvca.io/purpose: | + Default-deny operator egress + explicit allows for + kube-apiserver, nvsnap-server, and kube-dns. Tighten further by + overriding .Values.networkPolicy.operator.* — see the template + header for the rule shape. +spec: + podSelector: + matchLabels: + {{- include "nvcaop.baseSelectorLabels" . | nindent 6 }} + policyTypes: + - Egress + - Ingress + egress: + # 1. Kube-DNS — needed for nvsnap-server.nvsnap-system.svc resolution. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # 2. nvsnap-server (L2 promote-state, audit log writes). + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.networkPolicy.operator.nvsnapServerNamespace | default "nvsnap-system" }} + podSelector: + matchLabels: + app: {{ .Values.networkPolicy.operator.nvsnapServerPodLabel | default "nvsnap-server" }} + ports: + - protocol: TCP + port: {{ .Values.networkPolicy.operator.nvsnapServerPort | default 8080 }} + # 3. Kube-apiserver. K8s doesn't expose the apiserver via a + # Service selector inside this NetworkPolicy syntax (kubernetes.default + # isn't a "podSelector" target — it's a virtual endpoint). The cleanest + # portable shape is an IP-based egress to the apiserver port. + # Operators with a known apiserver CIDR can pin it; default 0.0.0.0/0:443 + # allows the apiserver but ALSO any other 443 target, which is + # accepted because this rule list is additive (deny-by-default + # elsewhere blocks anything not explicitly allowed). + {{- with .Values.networkPolicy.operator.apiServerCIDRs }} + - to: + {{- range . }} + - ipBlock: + cidr: {{ . | quote }} + {{- end }} + ports: + - protocol: TCP + port: {{ $.Values.networkPolicy.operator.apiServerPort | default 443 }} + {{- end }} + # 4. Operator-defined extra egress rules (private registries, + # OTLP collectors, anything cluster-specific). Render verbatim. + {{- with .Values.networkPolicy.operator.extraEgress }} + {{- toYaml . | nindent 4 }} + {{- end }} + ingress: + # Allow scraping + probes from anywhere by default. Tighten via + # .Values.networkPolicy.operator.ingressFrom (list of NetworkPolicyPeer). + {{- with .Values.networkPolicy.operator.ingressFrom }} + - from: + {{- toYaml . | nindent 8 }} + ports: + - protocol: TCP + port: 8000 + - protocol: TCP + port: 8002 + {{- else }} + - ports: + - protocol: TCP + port: 8000 + - protocol: TCP + port: 8002 + {{- end }} +{{- end }} diff --git a/deploy/helm/nvca-operator/nvca-operator/values.yaml b/deploy/helm/nvca-operator/nvca-operator/values.yaml index 6ffe6c9d3..9c7899b7a 100644 --- a/deploy/helm/nvca-operator/nvca-operator/values.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/values.yaml @@ -340,6 +340,23 @@ gracefulShutdown: networkPolicy: clusterNetworkCIDRs: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/12"] customPolicies: [] + ## @param networkPolicy.operator.enabled Apply a default-deny NetworkPolicy to the operator pod itself, with explicit allows for kube-dns, nvsnap-server, and the kube-apiserver. Opt-in: false by default so existing deployments are unchanged. + ## @param networkPolicy.operator.nvsnapServerNamespace Namespace where nvsnap-server runs (egress target for L2 promote-state polls + audit writes) + ## @param networkPolicy.operator.nvsnapServerPodLabel app= label value matched against pods in the nvsnap-server namespace + ## @param networkPolicy.operator.nvsnapServerPort nvsnap-server REST port + ## @param networkPolicy.operator.apiServerCIDRs CIDR list containing the kube-apiserver. Empty = no apiserver allow (operator loses K8s API access). Pin to your cluster's apiserver range. + ## @param networkPolicy.operator.apiServerPort kube-apiserver port + ## @param networkPolicy.operator.ingressFrom Optional NetworkPolicyPeer list for ingress (Prometheus, kubelet probes). Empty = allow from anywhere. + ## @param networkPolicy.operator.extraEgress Additional egress rule entries appended verbatim (private registries, OTLP collector, …) + operator: + enabled: false + nvsnapServerNamespace: "nvsnap-system" + nvsnapServerPodLabel: "nvsnap-server" + nvsnapServerPort: 8080 + apiServerCIDRs: [] + apiServerPort: 443 + ingressFrom: [] + extraEgress: [] ## @section Cluster Validator Configuration ## @param clusterValidator.enabled Enable the cluster-validator CronJob and init container diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml index 6c4264a5f..361f21942 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml @@ -46,6 +46,17 @@ spec: runAsUser: 1000 runAsGroup: 1010 fsGroup: 1010 + # Defense in depth: enforce non-root + seccomp at the pod + # level. runAsUser=1000 already guarantees non-root; setting + # runAsNonRoot=true makes the constraint explicit (kubelet + # will reject the pod if the image's USER somehow goes back + # to root). seccompProfile=RuntimeDefault blocks the + # ~40 syscalls Docker's default seccomp profile blocks + # (clock_settime, modify_ldt, …) — safe for any normal Go + # operator. + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end}} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/operator-networkpolicy.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/operator-networkpolicy.yaml new file mode 100644 index 000000000..bdb63091c --- /dev/null +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/operator-networkpolicy.yaml @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# NetworkPolicy hardening for the nvca-operator pod itself. +# +# By default, in clusters with a default-deny CNI (Calico, Cilium with +# enforcement, etc.) operator egress is wide open inside the cluster. +# This template adds explicit allow rules for the three legitimate +# egress targets: +# +# 1. kube-apiserver — for CRD watches, leader-election leases, +# child-resource create/update, status writes. CIDR-based rule +# because the apiserver IP is cluster-specific; default +# 0.0.0.0/0 - cluster-internal-CIDRs covers the public apiserver +# endpoint without leaking to other pods. +# +# 2. nvsnap-server (default nvsnap-system/nvsnap-server, port 8080) — +# for L2 promote-state polls and audit calls. Namespace + label +# selector keeps it tight. +# +# 3. kube-dns — for service name resolution. Without this rule the +# operator can't resolve nvsnap-server.nvsnap-system.svc anyway. +# +# Ingress: container port 8000 (health/metrics) and 8002 (auxiliary). +# Allowed from any namespace by default (Prometheus scraping + +# kubelet probes can come from anywhere). Operators can tighten by +# overriding ingressFrom. +# +# Opt-in via .Values.networkPolicy.operator.enabled (default false) +# so existing deployments keep working unchanged. Turn on after +# verifying the rule set fits your cluster. +# +# Note: K8s NetworkPolicy semantics are "deny by default IF any +# NetworkPolicy selects the pod." Creating this policy WITHOUT a +# matching cluster CNI may have no effect (e.g. some kindnet +# configs); test in your environment before relying on it. +{{- if and (hasKey .Values "networkPolicy") (hasKey .Values.networkPolicy "operator") .Values.networkPolicy.operator.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "nvcaop.fullname" . }}-operator-egress + namespace: {{ .Release.Namespace }} + labels: + {{- include "nvcaop.labels" . | nindent 4 }} + annotations: + nvca.io/purpose: | + Default-deny operator egress + explicit allows for + kube-apiserver, nvsnap-server, and kube-dns. Tighten further by + overriding .Values.networkPolicy.operator.* — see the template + header for the rule shape. +spec: + podSelector: + matchLabels: + {{- include "nvcaop.baseSelectorLabels" . | nindent 6 }} + policyTypes: + - Egress + - Ingress + egress: + # 1. Kube-DNS — needed for nvsnap-server.nvsnap-system.svc resolution. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # 2. nvsnap-server (L2 promote-state, audit log writes). + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.networkPolicy.operator.nvsnapServerNamespace | default "nvsnap-system" }} + podSelector: + matchLabels: + app: {{ .Values.networkPolicy.operator.nvsnapServerPodLabel | default "nvsnap-server" }} + ports: + - protocol: TCP + port: {{ .Values.networkPolicy.operator.nvsnapServerPort | default 8080 }} + # 3. Kube-apiserver. K8s doesn't expose the apiserver via a + # Service selector inside this NetworkPolicy syntax (kubernetes.default + # isn't a "podSelector" target — it's a virtual endpoint). The cleanest + # portable shape is an IP-based egress to the apiserver port. + # Operators with a known apiserver CIDR can pin it; default 0.0.0.0/0:443 + # allows the apiserver but ALSO any other 443 target, which is + # accepted because this rule list is additive (deny-by-default + # elsewhere blocks anything not explicitly allowed). + {{- with .Values.networkPolicy.operator.apiServerCIDRs }} + - to: + {{- range . }} + - ipBlock: + cidr: {{ . | quote }} + {{- end }} + ports: + - protocol: TCP + port: {{ $.Values.networkPolicy.operator.apiServerPort | default 443 }} + {{- end }} + # 4. Operator-defined extra egress rules (private registries, + # OTLP collectors, anything cluster-specific). Render verbatim. + {{- with .Values.networkPolicy.operator.extraEgress }} + {{- toYaml . | nindent 4 }} + {{- end }} + ingress: + # Allow scraping + probes from anywhere by default. Tighten via + # .Values.networkPolicy.operator.ingressFrom (list of NetworkPolicyPeer). + {{- with .Values.networkPolicy.operator.ingressFrom }} + - from: + {{- toYaml . | nindent 8 }} + ports: + - protocol: TCP + port: 8000 + - protocol: TCP + port: 8002 + {{- else }} + - ports: + - protocol: TCP + port: 8000 + - protocol: TCP + port: 8002 + {{- end }} +{{- end }} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml index b797c279e..514bbe38c 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml @@ -364,6 +364,24 @@ networkPolicy: clusterNetworkCIDRs: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/12"] customPolicies: [] + ## @param networkPolicy.operator.enabled Apply a default-deny NetworkPolicy to the operator pod itself, with explicit allows for kube-dns, nvsnap-server, and the kube-apiserver. Opt-in: false by default so existing deployments are unchanged. + ## @param networkPolicy.operator.nvsnapServerNamespace Namespace where nvsnap-server runs (egress target for L2 promote-state polls + audit writes) + ## @param networkPolicy.operator.nvsnapServerPodLabel app= label value matched against pods in the nvsnap-server namespace + ## @param networkPolicy.operator.nvsnapServerPort nvsnap-server REST port + ## @param networkPolicy.operator.apiServerCIDRs CIDR list containing the kube-apiserver. Empty = no apiserver allow (operator loses K8s API access). Pin to your cluster's apiserver range. + ## @param networkPolicy.operator.apiServerPort kube-apiserver port + ## @param networkPolicy.operator.ingressFrom Optional NetworkPolicyPeer list for ingress (Prometheus, kubelet probes). Empty = allow from anywhere. + ## @param networkPolicy.operator.extraEgress Additional egress rule entries appended verbatim (private registries, OTLP collector, …) + operator: + enabled: false + nvsnapServerNamespace: "nvsnap-system" + nvsnapServerPodLabel: "nvsnap-server" + nvsnapServerPort: 8080 + apiServerCIDRs: [] + apiServerPort: 443 + ingressFrom: [] + extraEgress: [] + ## @section Cluster Validator Configuration ## @param clusterValidator.enabled Enable the cluster-validator CronJob and init container diff --git a/src/compute-plane-services/nvca/docs/users/nvsnap/DURABLE-WARM-SWEEP.md b/src/compute-plane-services/nvca/docs/users/nvsnap/DURABLE-WARM-SWEEP.md new file mode 100644 index 000000000..e8c6da0c9 --- /dev/null +++ b/src/compute-plane-services/nvca/docs/users/nvsnap/DURABLE-WARM-SWEEP.md @@ -0,0 +1,82 @@ +# Durable CFS Warm flip — controller sweep (nvca#104 follow-up) + +## Problem + +The `NvSnapFunctionState.status.localCacheState = Warm` flip is produced by +exactly one in-process reconcile goroutine that must survive the entire +Hook B flow: + +``` +pod-ready event → warmup buffer → POST /checkpoints + → pollCheckpointTerminal (≤30 min) + → pollPVCPromoteTerminal (≤15 min, Hyperdisk-ML snap+clone) + → writeStatus(Warm) +``` + +If that goroutine is interrupted *after* the capture succeeds but *before* +`writeStatus` — controller restart, leader-election change, a poll that +silently never reaches terminal, or the source pod being scaled away +mid-flight — CFS stays not-Warm. + +The only existing recovery is the `recoverExistingCapture` short-circuit +(step 0b in `Reconcile`). It runs at the **top of a reconcile**, so it only +fires when a *new pod stamped `nvsnap.io/checkpoint-on-warm`* appears for the +same function-version. A **restore** pod is not stamped that annotation, and +NVCF does not re-deploy a capture candidate once the function is warm — so +once the source pod is gone, **nothing ever re-evaluates** and CFS is stuck +not-Warm forever, silently cold-starting every restore until an operator +hand-patches it. + +Observed on nvsnap-h100-a 2026-06-18 (DeepSeek, fvID `b96ac357`): checkpoint +`Completed 00:23:32`, L2 promote `ready 00:28:14`, source pod alive until +`00:29:30` — yet the reconcile emitted no terminal log and CFS never went +Warm. The next pod (`00:31:09`) was a restore attempt, so step 0b never ran. + +## Fix + +Add a **periodic CFS reconcile sweep** to the controller, independent of pod +events. The catalog (nvsnap-server) is the source of truth for "does a usable +capture exist"; the sweep reconciles every `NvSnapFunctionState` against it. + +``` +every SweepInterval (default 60s): + list NvSnapFunctionState + for each cfs where state != Warm && !optOut && spec.workloadLookup.imageRef != "": + hash = findUsableCapture(imageRef, modelId) # LookupCheckpoints + pvc-state usable + if hash != "": + writeStatus(cfs, Warm, hash, capturedHere=true) +``` + +`findUsableCapture` is the existing `recoverExistingCapture` body, refactored +to take `(imageRef, modelId)` instead of a `*Pod`, so the sweep and the +pod-triggered step-0b share one code path and one definition of "usable" +(L2 promote `ready`, or no-L2 → L1/peer-cascade). + +### Persisting the lookup inputs + +The sweep has no live pod, so it can't derive `imageRef`/`modelId` from a pod +spec. Hook B persists them onto `NvSnapFunctionState.spec.workloadLookup` right +after `getOrCreateCFS`, *before* the risky poll — so any capture that Hook B +starts is recoverable by the sweep even if the reconcile dies. The CRD uses +`x-kubernetes-preserve-unknown-fields: true`, so no CRD-schema change is +needed. + +## Properties + +- **Eventually consistent**: CFS goes Warm within one `SweepInterval` of the + capture completing, regardless of pod/reconcile/controller lifecycle. +- **Idempotent**: a Warm CFS is skipped; re-running the sweep is a no-op. +- **Safe**: only flips Warm when nvsnap-server confirms a `Completed` capture + with a terminal-usable L2 promote — same gate as the live reconcile. +- **No new RBAC**: list/get/update on `nvsnapfunctionstates` already granted. + +## Files + +- `pkg/apis/nvsnap/v1alpha1/nvsnapfunctionstate_types.go` — `WorkloadLookupSpec` + on `Spec`. +- `pkg/nvca/nvsnap/reconciler/state.go` — read/write `spec.workloadLookup`. +- `pkg/nvca/nvsnap/reconciler/recover.go` — extract `findUsableCapture`. +- `pkg/nvca/nvsnap/reconciler/sweep.go` — `SweepOnce`. +- `pkg/nvca/nvsnap/reconciler/reconciler.go` — persist lookup inputs pre-poll. +- `pkg/nvca/nvsnap/controller/controller.go` — sweep ticker in `Run`. +- `pkg/nvca/nvsnap/reconciler/metrics.go` — `nvca_nvsnap_cfs_sweep_recovered_total`. diff --git a/src/compute-plane-services/nvca/docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN-DELTA.md b/src/compute-plane-services/nvca/docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN-DELTA.md new file mode 100644 index 000000000..3e2ba3ead --- /dev/null +++ b/src/compute-plane-services/nvca/docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN-DELTA.md @@ -0,0 +1,295 @@ +# NVCA × NvSnap — design delta (post-implementation review) + +**Status:** Proposed, 2026-05-29 +**Branch:** `feat/nvsnap-integration` +**Builds on:** [`NVSNAP-INTEGRATION-DESIGN.md`](./NVSNAP-INTEGRATION-DESIGN.md) + +This delta layers on top of the original design after reviewing the +landed implementation against a production QA scenario. + +## Motivation — the QA scenario the original design under-served + +QA deploys **10 different function-versions** of the same NIM/vLLM +image, same model, against the same H100 SKU. Each function-version has +**10 replica pods**. Total: 100 pods, all functionally identical from a +checkpoint standpoint. + +Original design + landed code: each function-version is treated +independently (NvSnapFunctionState keyed by `functionVersionID`). All 10 +function-versions cold-boot in parallel, each captures its own checkpoint, +each writes its own hash to NGC. Storage dedups via content-addressing +*after* the fact, but the 10× cold-boot work has already been spent — 10 +model downloads, 10 engine compiles, 10 CRIU dumps. + +QA's complaint is real: **the whole point of checkpoint/restore is that +identical workloads should pay the cold cost once.** This delta closes +that gap. + +## Three gaps to close + +| # | Gap | Fix | +|---|---|---| +| 1 | Impl drifted from original Q3 — landed code keys by `functionVersionID`, design said "use nvsnap's canonical pod-spec hash" | Rekey the dedup index by `canonicalHash` (see §1) | +| 2 | No protection against concurrent identical workloads ("thundering herd"). 10 function-versions with the same canonical hash all cold-boot simultaneously | Add `Capturing` state + init-container waiter (see §2) | +| 3 | Observability surface (original §H) is three counters. Insufficient for a production system that affects pod startup time | Full metrics/logging/alerts/tracing/audit surface (see §3) | + +--- + +## §1 — Canonical-hash key (closes impl drift from original Q3) + +### Current state in code + +`pkg/nvca/nvsnap_hook.go` reads `NvSnapFunctionVersionIDAnnotation` and +looks up `NvSnapFunctionState` by that fvID. One NvSnapFunctionState +per fvID. Hash dedup happens at the nvsnap-blobstore CAS layer, not +at the NVCA decision layer. + +### Proposed + +Compute a **canonical identity hash** at pod-create time, using the +same algorithm nvsnap's `internal/rootfsonly/composer.go` uses (or a +thin Go wrapper that produces an identical hash). Inputs that +affect checkpoint validity: + +``` +canonicalHash = sha256( + imageDigest // immutable digest, not :tag + + literalEnvVars // only env[].value entries; skip valueFrom (per-pod metadata) + + sortedArgs + + acceleratorType // nvidia-h100-80gb + + gpuCount // TP=1 vs TP=4 produce incompatible checkpoints + + driverVersionMajor // 555.x vs 560.x — GPU plugin state is driver-coupled +) +``` + +**Crucially, NOT in the hash:** anything from `valueFrom` env (those are +per-pod / per-function-version fieldRef injections like +`NVCF_FUNCTION_ID`, `INSTANCE_ID`, `NVCF_FUNCTION_VERSION_ID`). Including +them would re-fragment the index back to per-pod or per-fvID, defeating +the whole point. Verified against the live function pod on nvsnap-h100-a +(2026-05-29) — NVCA stamps 9 `valueFrom`-style env vars that would +otherwise poison the hash. + +**Override**: pod annotation `nvsnap.io/identity-key: ` wins over +the derived hash. Escape hatch for cases where the auto-derivation is +wrong (e.g. function depends on a configMap whose change should invalidate +the checkpoint). + +### NvSnapFunctionState becomes identity-keyed + +`NvSnapFunctionState` rekeys from `name: ` to +`name: nvsnap-id-`. One NvSnapFunctionState per +identity, not per fvID. **N fvIDs that produce the same canonical hash +share one NvSnapFunctionState** — that's the dedup. + +Migration: existing CRs (per-fvID) are left in place but ignored by +the new code paths. A one-time backfill job (or just letting them +expire via TTL) cleans them up. + +### Failure mode + +If two `valueFrom` env vars *should* have been included (e.g. a config +that legitimately differs between functions) and the user didn't catch +it, restores will misbehave at runtime. Mitigations: + +- Auto-disable restore for any pod whose annotated `nvsnap.io/identity-key` + doesn't match the runtime-computed hash → cold-boot fallback, alert + fires. +- Document the override clearly. Use the `nvsnap.io/identity-key` + annotation for explicit equivalence. + +--- + +## §2 — Thundering-herd protection (3-state model + waiter) + +### State machine on NvSnapFunctionState + +| State | Meaning | Hook A behavior | Hook B behavior | +|---|---|---|---| +| `NoCheckpoint` | first pod with this identity | normal admit (no restore-from) | first pod to acquire lease becomes checkpointer | +| `Capturing` | someone holds the lease, is checkpointing | inject `nvsnap-wait` init container | other pods skip (lease denied) | +| `Warm` | checkpoint exists, ready to restore | inject `restore-from` annotation | no-op (already exists) | +| `Failed` | last capture failed | inject `nvsnap-wait` init container that bypasses to cold after backoff | retry from any pod after backoff window | + +Transitions: +- `NoCheckpoint → Capturing` — first pod calls `POST /identities//lease`, gets `acquired` +- `Capturing → Warm` — nvsnap-server records hash via existing flow, NVCA reconciler updates status +- `Capturing → Failed` — lease TTL expired without success, or checkpointer reported failure +- `Warm → Capturing` — re-checkpoint window (existing `nvsnap_refresh_interval`) +- `Failed → Capturing` — backoff window elapsed, next pod retries + +Lease TTL: `WarmupTimeoutSeconds + CheckpointBudgetSeconds` (default ~20 min). + +### `nvsnap-wait` init container + +Injected by Hook A when state is `Capturing` or `Failed`. Long-polls +nvsnap-server until terminal: + +```bash +# pseudocode +identity= +deadline=$(date +%s) + WaitTimeoutSeconds # default 600s +while [ $(date +%s) -lt $deadline ]; do + state=$(curl -fsSL nvsnap-server/identities/$identity) + case $state in + Warm) echo "[nvsnap-wait] checkpoint ready, restore will fire"; exit 0 ;; + Capturing) sleep 5 ;; # poll every 5s + NoCheckpoint|Failed) + echo "[nvsnap-wait] no checkpoint available, cold-booting" + exit 0 # let inference cold-boot; this pod becomes a candidate checkpointer + ;; + esac +done +# timeout — fall back to cold +echo "[nvsnap-wait] timed out after ${WaitTimeoutSeconds}s, cold-booting" +exit 0 +``` + +Notes: +- Always `exit 0` — never block the pod permanently. Worst case, fall + back to cold-boot and emit metrics. +- Image: bundled with NVCA's existing utils image (curl + bash) so no + new container pull. +- Resource footprint: 50m CPU, 64Mi RAM — same shape as other init + containers in the function pod. +- The inference container's restore-entrypoint then either picks up the + `restore-from` annotation (if state went Warm during wait) or + cold-boots normally. + +### Failure-mode reasoning + +- **Checkpointer dies mid-capture** — lease TTL expires, next pod + acquires lease, state goes `Failed → Capturing`. Waiters time out and + cold-boot. Eventually catches up. +- **Network partition between waiter and nvsnap-server** — `curl -fsSL` + returns non-zero, `nvsnap-wait` exits 0 (cold-boot fallback). Don't fail + the pod over an observability dependency. +- **Capture succeeds but blob upload to L3 (S3/GCS) fails** — + NvSnapFunctionState stays at `Capturing` until upload completes or + TTL expires. Tradeoff: we wait for upload before flipping to Warm so + cross-cluster restore actually works. Acceptable. + +### Numbers — QA scenario after this delta + +100 pods, same canonical hash: +- 1 pod cold-boots (~2-3 min warmup), captures (~1-2 min) +- 99 pods init-container-wait (~3-5 min idle), restore (~30-60s each, parallel) +- **Total wall time ~5 min** vs. ~30 min today +- **Total GPU-minutes wasted on warmup: 1 vs. 100** + +--- + +## §3 — Observability surface + +### Metrics (Prometheus, NVCA-side) + +| Metric | Type | Labels | Purpose | +|---|---|---|---| +| `nvca_nvsnap_checkpoint_attempts_total` | counter | `result`, `reason` | success / failure / skipped (lease denied) / aborted | +| `nvca_nvsnap_checkpoint_duration_seconds` | histogram | — | from lease-acquire to Warm | +| `nvca_nvsnap_checkpoint_in_progress` | gauge | — | identities currently in `Capturing` | +| `nvca_nvsnap_restore_attempts_total` | counter | `result`, `reason` | success / failure / fallback-cold | +| `nvca_nvsnap_restore_duration_seconds` | histogram | — | pod-create → container Ready (via restore path) | +| `nvca_nvsnap_restore_silent_fallback_total` | counter | `reason` | **critical**: annotation stamped, restore failed silently, cold-booted | +| `nvca_nvsnap_warmup_duration_seconds` | histogram | — | PodReady → first `/health` 200 | +| `nvca_nvsnap_lease_wait_seconds` | histogram | `outcome` | init-container `nvsnap-wait` time, outcome=warm/cold/timeout | +| `nvca_nvsnap_reconciler_queue_depth` | gauge | — | back-pressure indicator | +| `nvca_nvsnap_identities_tracked` | gauge | — | cardinality of distinct identities | +| `nvca_nvsnap_server_request_errors_total` | counter | `endpoint` | network/4xx/5xx talking to nvsnap-server | + +**Cardinality discipline**: do NOT label by `fvID`, `podName`, or full +`canonicalHash`. Use `reason` enums and `outcome` enums only. If +per-identity debugging is needed, drop to logs/traces/audit, not metrics. + +### Logging (structured JSON via existing logrus) + +Every line carries: `identity` (16-char hash prefix), `fvID`, `pod`, +`namespace`, `clusterID`, `reason`. + +Levels: +- `ERROR` — any checkpoint/restore failure, with nvsnap-server response body +- `WARN` — lease-timeout fallbacks, silent fallbacks, reconciler retries +- `INFO` — state transitions (`NoCheckpoint → Capturing → Warm`), lease acquire/release +- `DEBUG` — poll-loop ticks, network round-trips + +### Alerts (PrometheusRule, ships with operator chart) + +| Alert | Expression | Severity | +|---|---|---| +| `NvSnapCheckpointFailureRate` | `rate(nvca_nvsnap_checkpoint_attempts_total{result="failure"}[5m]) > 0.1 * rate(nvca_nvsnap_checkpoint_attempts_total[5m])` | page | +| `NvSnapRestoreSilentFallback` | `rate(nvca_nvsnap_restore_silent_fallback_total[5m]) > 0` | page (this is the "customer thinks it's working but it isn't" case) | +| `NvSnapRestoreFallbackRate` | `rate(nvca_nvsnap_restore_attempts_total{result="fallback-cold"}[5m]) > 0.5 * rate(nvca_nvsnap_restore_attempts_total[5m])` | page | +| `NvSnapServerUnreachable` | `rate(nvca_nvsnap_server_request_errors_total[2m]) > 0.5` | page | +| `NvSnapLeaseStuck` | `nvca_nvsnap_checkpoint_in_progress > 0 unless on() rate(...) for 2 * WarmupTimeout` | warn | +| `NvSnapReconcilerBackpressure` | `nvca_nvsnap_reconciler_queue_depth > 100` | warn | +| `NvSnapIdentityCardinalitySpike` | `delta(nvca_nvsnap_identities_tracked[1h]) > 1000` | warn (catches identity-hash bugs) | + +### Tracing (OTel — propagate existing pattern) + +NvSnap-agent and restore-entrypoint already emit OTel spans (issue #50). +Extend the trace from NVCA's side: + +| Span | When | Parent | +|---|---|---| +| `nvca.nvsnap.hook_a.admit` | pod create webhook | root | +| `nvca.nvsnap.hook_a.identity_compute` | derive canonical hash | hook_a.admit | +| `nvca.nvsnap.hook_a.lease_check` | query NvSnapFunctionState | hook_a.admit | +| `nvca.nvsnap.hook_b.warmup_poll` | Ready → /health 200 | new root | +| `nvca.nvsnap.hook_b.checkpoint_request` | POST to nvsnap-server | hook_b.warmup_poll | +| `nvca.nvsnap.lease.wait` | inside `nvsnap-wait` init container | new root | + +Propagation: NVCA pod-mutator stamps `OTEL_TRACE_PARENT` on the +inference container's env, same mechanism the restore-entrypoint +already uses. NvSnap-agent picks it up and emits checkpoint-side spans +under the same trace. **Result: one trace from pod-admission through +checkpoint completion**, viewable end-to-end in Jaeger. + +### Audit (nvsnap-server side, existing `audit_log` table from #45) + +nvsnap-server already has an audit table. Add identity-keyed entries: + +| Action | Resource | Details | +|---|---|---| +| `lease.acquire` | `identity:` | who, when, TTL | +| `lease.release` | `identity:` | success/failure | +| `state.transition` | `identity:` | from → to | +| `restore.fallback` | `identity:`, `pod:` | reason | + +Queryable via existing `/api/v1/audit?resource=identity:` — +gives a per-identity timeline across all pods/clusters that used it. + +--- + +## Sequencing — what lands first + +1. **§3 metrics counters** (mechanically safe — observability for the + current broken behavior, helps quantify pain before fixing it) +2. **§1 canonical hash key** (the dedup fix, decoupled from waiter mechanics) +3. **§2 lease + `nvsnap-wait` init container** (depends on §1 being live) +4. **§3 alerts** (depends on metrics being live and gathering baseline data) + +Each is its own MR. Per gpucr CLAUDE.md rule #5 (>3 files = break into +smaller tasks), this is the natural split. + +## Open questions + +1. **Where does the canonical-hash compute live?** Either NVCA imports + nvsnap's `composer.go` directly (creates a build-time dep on nvsnap), + OR NVCA POSTs the pod spec to nvsnap-server `POST /identities/compute` + and gets the hash back (adds a network hop per pod admission, but + keeps nvsnap as a service boundary). Original design (Q2) decided + "NVCA pushes, doesn't depend on nvsnap internals" — implies the + network-hop variant. Cost: ~10ms per admission. Tolerable? + +2. **Init-container injection — webhook order**: nvsnap's own mutating + webhook also injects volumes for the restore path. NVCA's + `nvsnap-wait` init container must run BEFORE the inference container, + which is the default ordering, but if nvsnap's webhook reorders + things this could break. Verify ordering in nvsnap's `nvsnap-rootfs-restore` + webhook before landing §2. + +3. **WaitTimeoutSeconds default**: how long should a waiter sit? 10 + minutes feels right for 70B models (whose cold-boot is ~13min). For + small models 10 min is far too generous. Could derive from + warm-baseline metric, but for v1 just take a config knob with default + 600s. diff --git a/src/compute-plane-services/nvca/docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md b/src/compute-plane-services/nvca/docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md new file mode 100644 index 000000000..115c25aae --- /dev/null +++ b/src/compute-plane-services/nvca/docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md @@ -0,0 +1,347 @@ +# NVCA × NvSnap — checkpoint/restore integration design + +**Status:** Proposed, 2026-05-12 +**Branch:** `feat/nvsnap-integration` +**NvSnap project:** `github.com/balaji-g/nvsnap` (currently @ `main` 1cc2410) + +## What this delivers + +Today: NVCA creates an inference pod for a function-version. Cold start +takes 1–13 min depending on the framework (HF model download, engine +compile, NCCL handshake). Every pod for the same function-version pays +the same cost. Scale-out of N replicas = N cold starts. + +After this integration: + +1. NVCA creates the first pod cold. Once it's warm (inference ready, + passed N seconds), NVCA calls nvsnap to **checkpoint** the `inference` + container's running state into a node-local cache + the cluster's + nvsnap-blobstore. +2. NVCA stores the resulting **checkpoint hash** on the function-version + metadata. +3. For every subsequent pod creation for that function-version, NVCA + stamps the pod with `nvsnap.io/restore-from: `. NvSnap's + admission webhook injects bind-mounts of the warmed cache onto the + inference container. Pod reaches Ready in tens of seconds instead + of minutes. + +Measured speedups on nvsnap's own test cluster (see `docs/ARCHITECTURE.md` +in the nvsnap repo for the bench table): + +- vLLM Llama-3.1-8B: **2.1×** (cold 2m → restore 56s) +- NIM Llama-3.1-8B: **1.5×** (cold 1m40s → restore 1m05s) +- NIM Qwen3-32B TP=2: **6×** (cold 3m16s → restore 33s) — flagship +- vLLM Llama-3.1-70B TP=4: **~4×** (cold ~13m → restore 3m29s) + +## Non-goals + +- NvSnap's internals — agent, CRIU patches, sitecustomize injection. + This integration treats nvsnap as a service. +- Cross-cluster blob replication mechanics — covered by the + nvsnap-blobstore design (S3/GCS-backed content-addressed store). + This doc assumes any cluster's nvsnap-agent can resolve a hash to + bytes via the blobstore. +- Replacing utils sidecar / init containers. NvSnap only checkpoints the + `inference` container. + +## Architecture + +``` + ┌───────────────────────────────────────────┐ + │ Function-version object (NGC catalog) │ + │ + nvsnap_checkpoint_hash (new field) │ + └──────────────┬────────────────────────────┘ + │ + │ NVCA reads/writes per version + ▼ + ┌─────────────────────────────────────────────────────────┐ + │ NVCA agent │ + │ pkg/nvca/k8scomputebackend.go : CreatePodArtifactInstances + │ 1. for function-version V → fetch hash from NGC │ + │ 2. if hash exists & feature flag on → annotate pod │ + │ metadata.annotations.nvsnap.io/restore-from = hash │ + │ 3. apply pod │ + │ 4. when pod is Ready + warm window elapses → call │ + │ nvsnap.Checkpoint(pod, container="inference") │ + │ 5. on success, write hash back to NGC metadata │ + └─────────────────────────────────────────────────────────┘ + │ + │ HTTP + ▼ + ┌─────────────────────────────────────────────────────────┐ + │ nvsnap-server (nvsnap-system namespace) │ + │ POST /checkpoint/pod │ + │ { pod, namespace, container } → { hash, size, ... } │ + │ GET /api/v1/checkpoints/ status / metadata │ + │ ← admission webhook handles nvsnap.io/restore-from │ + └─────────────────────────────────────────────────────────┘ +``` + +## Integration point inside NVCA + +`pkg/nvca/k8scomputebackend.go` is the only file that needs deep +wiring. `CreatePodArtifactInstances` (line 956) is where workload +pods are constructed and applied. Two hooks: + +### Hook A — restore-on-create (pre-apply) + +Just before `podClient.Create(ctx, pod, ...)`, look up the nvsnap hash +for this function-version. If found and the feature flag is on, add +the annotation: + +```go +pod.Annotations["nvsnap.io/restore-from"] = checkpointHash +``` + +That's the entirety of the restore path NVCA needs. NvSnap's +`MutatingWebhookConfiguration nvsnap-rootfs-restore` runs at admission, +reads the annotation, resolves the manifest from +`nvsnap-capture-` ConfigMap, injects nodeAffinity + +hostPath volume mounts on the inference container. Customer image ++ args unchanged. + +### Hook B — checkpoint-after-warm (post-health-OK) + +A separate reconcile loop watches for newly-Ready pods bearing the +`nvsnap.io/checkpoint-on-warm: "true"` annotation that NVCA stamped at +creation. Once `PodReady` fires, the loop polls the inference +container's health endpoint directly. When `/health` returns HTTP +200, the engine is provably warm (model loaded, lazy compile done, +ready to serve real requests). Then the loop: + +1. POSTs to `nvsnap-server.nvsnap-system.svc.cluster.local:8080/api/v1/checkpoint/pod` + with `{namespace, pod, container: "inference"}`. +2. Polls status until terminal (success / fail). +3. On success, writes the hash to the function-version object via the + NGC API. +4. Removes the `checkpoint-on-warm` annotation so the loop is idempotent. + +Health-check details: +- NVCA reads the health endpoint URL from the function-version's + inference spec (already there for traffic routing). Default + contract: `GET :/v1/health/ready` returns 200 when the + engine is ready to serve. +- Poll interval: 5s. Total budget: `NvSnapWarmupTimeoutSeconds` (default + 900, 15 min) — TRT-LLM 70B engine compile can take 10+ min. +- After the first 200, wait `NvSnapWarmupBufferSeconds` (default 10s) + to let any post-first-response setup land (CUDA graph capture, JIT + warmup), then trigger checkpoint. +- If the health endpoint never returns 200 within the budget, log + + metric, give up on this pod (no checkpoint, no retry — operator + investigates). + +The same reconcile loop can re-checkpoint if the function-version's +`nvsnap_refresh_interval` has elapsed (handles model updates without +operator intervention). + +Failure modes (fail-open): + +- nvsnap-server unreachable → log + retry with exponential backoff; + pod stays running, just no future restore speedup. +- Checkpoint fails (CRIU error, GPU state issue) → log + report metric; + function-version's hash isn't updated; subsequent pods cold-start. + +## NvSnap APIs NVCA consumes + +``` +POST /api/v1/checkpoint/pod + body: {"namespace": "nvcf-backend", "pod": "0-sr-...", "container": "inference"} + → 202 Accepted {"checkpoint_id": "...", "status_url": "/api/v1/checkpoints/..."} + +GET /api/v1/checkpoints/ + → { "id": "...", "hash": "", "size_bytes": N, "status": "succeeded|failed", ... } + +DELETE /api/v1/checkpoints/ + → 204 (cleanup when a function-version is removed) +``` + +The `nvsnap.io/restore-from: ` annotation is the only restore-side +contract — no new API. Webhook does the rest. + +## What NVCA needs from nvsnap that isn't yet there + +1. **Container selector on `POST /checkpoint/pod`.** Today the nvsnap + handler in `internal/server/demo.go` keys off pod-name only. Need + to take `container` and pass it through to the agent's process + discovery. **Tracked: nvsnap issue (new).** +2. **Multi-cluster registration of nvsnap-server URL.** NVCA's agent + needs to know where nvsnap lives. Today it's + `nvsnap-server.nvsnap-system.svc.cluster.local:8080` in-cluster — fine + as a default, configurable via NVCA cluster config key + `NvSnapServerURL` (override for off-cluster deployments). +3. **Idempotent checkpoint calls.** If NVCA retries a checkpoint + request that succeeded but the response was lost, nvsnap should + return the existing checkpoint, not start a new one. NvSnap computes + the manifest hash from the pod spec deterministically (see + `internal/rootfsonly/composer.go`), so this is mostly already true + for the rootfs path — needs verification. + +## Configuration surface (new) + +### Cluster-level config keys (set via NGC during cluster registration) + +| Key | Default | Meaning | +|---|---|---| +| `NvSnapIntegrationEnabled` | `false` | Master switch for the entire integration. Pod creation behaves identically to today when off. | +| `NvSnapServerURL` | `http://nvsnap-server.nvsnap-system.svc.cluster.local:8080` | Where nvsnap-server lives. | +| `NvSnapWarmupTimeoutSeconds` | `900` (15 min) | Max time to wait for `/health` 200 after `PodReady` before giving up. | +| `NvSnapWarmupBufferSeconds` | `10` | After first `/health` 200, wait this long before triggering checkpoint (lets post-response setup land). | +| `NvSnapRestoreEnabled` | `false` | Independent toggle for the restore side. Lets us roll out checkpoint-only first. | + +### Per-function-version (NGC API) + +| Field | Default | Meaning | +|---|---|---| +| `nvsnap_checkpoint_hash` | empty | Set by NVCA after a successful checkpoint. Used as the value of `nvsnap.io/restore-from`. | +| `nvsnap_opt_out` | `false` | Function-version opt-out (e.g., for workloads with state that can't be safely restored). | + +## Feature flag + +`pkg/featureflag/featureflag.go` gets one new flag: + +```go +NvSnapCheckpointRestore = newFeatureFlag("NvSnapCheckpointRestore", newBool(false)) +``` + +Wraps both `NvSnapIntegrationEnabled` and `NvSnapRestoreEnabled` for +emergency cluster-wide disable. Per-cluster + per-function-version +overrides via the config keys above. + +## Rollout plan + +1. **Stage 1 — checkpoint-only.** Flag ON, restore OFF. Every pod + gets checkpointed after warmup; no pod uses a restore. NVCA verifies + the checkpoint flow works end-to-end and gathers timing data on + nvsnap's checkpoint duration without affecting customer pod boot time. +2. **Stage 2 — restore for new functions.** Restore ON for + function-versions deployed after a cut-over date. Existing + function-versions stay cold-start until they're re-deployed. +3. **Stage 3 — backfill.** A maintenance job walks existing + function-versions, kicks one cold-start pod each, checkpoints + after warmup, then enables restore for that version. + +Each stage rolls per-cluster via the config keys. + +## Pieces of work + +Filed as nvca tasks (this design doc + the items below): + +- A. NvSnap HTTP client package `pkg/nvca/nvsnap/`. Thin Go bindings for + the four endpoints. Mocked in unit tests. +- B. Feature flag definition. One-line addition. +- C. Cluster config keys (read at NVCA startup, similar pattern to + existing `AgentNodeSelectorLabelKey` etc.). +- D. Restore-on-create hook in `CreatePodArtifactInstances`. ~30 LOC. +- E. Checkpoint-after-warm reconcile loop. New file + `pkg/nvca/nvsnap/checkpoint_reconciler.go`. ~200 LOC. +- F. Function-version metadata field `nvsnap_checkpoint_hash`. NGC API + schema update + NVCA read/write paths. +- G. Cluster-wide disable: respect `NvSnapCheckpointRestore` feature flag + in both A and D. +- H. Metrics — `nvca_nvsnap_checkpoint_duration_seconds`, + `nvca_nvsnap_checkpoint_total`, `nvca_nvsnap_restore_attempted_total` — + for ops visibility. +- I. E2E test — apply a function-version, wait for warmup, verify + checkpoint hash recorded, scale up to N replicas, verify N-1 of + them came up via restore (timing assertion). + +NvSnap-side dependencies (tracked in the nvsnap repo): + +- Container selector on `/api/v1/checkpoint/pod` (nvsnap issue TBD). +- Idempotent checkpoint replay verified. + +## Open questions + +1. **Where exactly does the hash live?** **Decided 2026-05-12: split + across two stores.** + + - **NGC function-version object** holds `nvsnap_checkpoint_hash` + (global, authoritative). One cluster checkpoints a + function-version; NGC records the hash; *every* cluster can then + stamp the `nvsnap.io/restore-from: ` annotation on its pods + without re-checkpointing. This is what makes checkpoints scale + across clusters — the hash is content-addressed, the blobs live + in the nvsnap-blobstore (S3/GCS), and any cluster's nvsnap-agent + pulls bytes via the standard tier-3 cascade. + - **Per-cluster CRD `NvSnapFunctionState`** (cluster-scoped) tracks + local state that NGC shouldn't carry: which cluster *first* + captured this hash, local cache freshness, per-cluster opt-outs, + last-attempted timestamps, retry backoff, error history. This is + the working state of the checkpoint-after-warm reconciler. + + ```yaml + apiVersion: nvsnap.nvidia.com/v1alpha1 + kind: NvSnapFunctionState + metadata: + name: # 1:1 with NGC function-version + spec: + functionVersionID: + optOut: false # nvsnap_opt_out + status: + # Mirror of NGC for fast local reads (canonical source = NGC): + checkpointHash: + # Per-cluster operational state: + capturedHere: false # did THIS cluster do the capture + capturedAt: + localCacheState: warm|cold|fetching|failed + lastAttemptAt: + lastError: "" + attemptCount: 0 + ``` + + Reasons this split wins for production: + - **Cross-cluster scaling**: hash is global, so cluster #2..N skip + the cold-start checkpoint entirely. Capturing once in any + cluster benefits all of them. + - **Per-cluster operational state stays per-cluster**: a fetch + failure on AWS-us-east-2 doesn't pollute NGC; an opt-out on + a single cluster doesn't need NGC permissions. + - **CRD gives typed schema, RBAC, watch semantics, status + subresource, audit** — required for production-grade operator. + - **Authoritative read path**: NVCA pre-apply hook reads NGC (one + network hop, cached); falls back to local CRD if NGC is + transiently unreachable. + +2. **Does NVCA call nvsnap, or does nvsnap watch NVCA?** **Decided + 2026-05-12: NVCA calls nvsnap (push).** NvSnap stays NVCA-agnostic; any + compute backend or scheduler that wants C/R can drive nvsnap's HTTP + API the same way. NVCA owns the lifecycle decisions; nvsnap executes + the C/R primitives. + +3. **Per-pod-template hash, or per-pod hash?** NvSnap's + `internal/rootfsonly/composer.go` already canonicalizes the pod + spec (TP size, image, CUDA driver major, etc.) into a deterministic + hash. NVCA can use that hash as the function-version key + automatically — if two function-versions happen to produce the same + canonical pod spec, they SHARE a checkpoint, which is exactly the + semantics we want. + +4. **What's the warm-window heuristic?** **Decided 2026-05-12: poll + the inference container's health endpoint directly.** K8s + `PodReady` is too lenient (readiness probes often just check that + the HTTP server answers, not that the engine is loaded). All + supported inference frameworks expose a `/health` (or + `/v1/health/ready`) endpoint that returns 200 only when the model + is fully loaded and ready to serve. NVCA polls this endpoint at 5s + intervals starting from PodReady, waits `NvSnapWarmupBufferSeconds` + (default 10s) after the first 200 to absorb post-first-response + setup, then triggers checkpoint. Total polling budget is + `NvSnapWarmupTimeoutSeconds` (default 15 min) — long enough for + TRT-LLM 70B engine compile. + + Why this beats a fixed timeout: + - Workload-aware: vLLM warm in 30s, TRT-LLM 70B in ~10min — both + handled by the same logic. + - Deterministic: checkpoint fires when the engine *says* it's + ready, not when a clock says it should be. + - Zero per-workload tuning: no `nvsnap_warmup_seconds` per + function-version to maintain. + + Health endpoint URL comes from the function-version's inference + spec (already there for traffic routing). + +--- + +*Next:* skeleton `pkg/nvca/nvsnap/client.go` + the feature flag entry, +without wiring into `CreatePodArtifactInstances` yet. Get the client + +flag merged as a no-op baseline, then layer hooks on top. diff --git a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/BUILD.bazel b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/BUILD.bazel index f3569fe25..0229484ec 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/BUILD.bazel @@ -35,8 +35,8 @@ alias( go_test( name = "nvcf_test", srcs = [ - "cryo_config_test.go", "nvcfbackend_types_test.go", + "nvsnap_config_test.go", ], embed = [":nvcf"], deps = [ diff --git a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/cryo_config_test.go b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/cryo_config_test.go deleted file mode 100644 index a19b48424..000000000 --- a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/cryo_config_test.go +++ /dev/null @@ -1,141 +0,0 @@ -/* -SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1 - -import "testing" - -func TestCryoConfigCompleteNil(t *testing.T) { - var c *CryoConfig - got := c.Complete(EnvTypeStage) - if got == nil { - t.Fatal("Complete(nil) returned nil; should return defaulted CryoConfig") - } - if got.IntegrationEnabled || got.RestoreEnabled { - t.Errorf("nil → defaults should leave both disabled, got %+v", got) - } - if got.ServerURL != DefaultCryoServerURL { - t.Errorf("ServerURL = %q, want %q", got.ServerURL, DefaultCryoServerURL) - } - if got.WarmupTimeoutSeconds != DefaultCryoWarmupTimeoutSeconds { - t.Errorf("WarmupTimeoutSeconds = %d, want %d", got.WarmupTimeoutSeconds, DefaultCryoWarmupTimeoutSeconds) - } - if got.WarmupBufferSeconds != DefaultCryoWarmupBufferSeconds { - t.Errorf("WarmupBufferSeconds = %d, want %d", got.WarmupBufferSeconds, DefaultCryoWarmupBufferSeconds) - } -} - -func TestCryoConfigCompletePreservesOverrides(t *testing.T) { - in := &CryoConfig{ - IntegrationEnabled: true, - RestoreEnabled: true, - ServerURL: "http://my-cryo.example.com:8080", - WarmupTimeoutSeconds: 600, - WarmupBufferSeconds: 30, - } - got := in.Complete(EnvTypeProd) - if !got.IntegrationEnabled || !got.RestoreEnabled { - t.Errorf("Complete should preserve enable bits; got %+v", got) - } - if got.ServerURL != "http://my-cryo.example.com:8080" { - t.Errorf("ServerURL override not preserved: %q", got.ServerURL) - } - if got.WarmupTimeoutSeconds != 600 || got.WarmupBufferSeconds != 30 { - t.Errorf("warmup overrides not preserved: %+v", got) - } -} - -func TestCryoConfigCompletePartialOverride(t *testing.T) { - // Enabled but no URL set — defaults should fill the URL. - in := &CryoConfig{IntegrationEnabled: true} - got := in.Complete(EnvTypeStage) - if !got.IntegrationEnabled { - t.Errorf("IntegrationEnabled lost across Complete") - } - if got.ServerURL != DefaultCryoServerURL { - t.Errorf("ServerURL should default when empty, got %q", got.ServerURL) - } -} - -func TestCryoConfigCompleteReturnsCopy(t *testing.T) { - in := &CryoConfig{IntegrationEnabled: true} - got := in.Complete(EnvTypeStage) - got.IntegrationEnabled = false - if !in.IntegrationEnabled { - t.Error("Complete returned a reference to input — should be a deep copy") - } -} - -func TestCryoConfigIsEnabled(t *testing.T) { - cases := []struct { - name string - c *CryoConfig - want bool - }{ - {"nil", nil, false}, - {"zero", &CryoConfig{}, false}, - {"checkpoint only", &CryoConfig{IntegrationEnabled: true}, true}, - {"restore only", &CryoConfig{RestoreEnabled: true}, true}, - {"both", &CryoConfig{IntegrationEnabled: true, RestoreEnabled: true}, true}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := tc.c.IsEnabled(); got != tc.want { - t.Errorf("IsEnabled() = %v, want %v", got, tc.want) - } - }) - } -} - -func TestCryoConfigDeepCopy(t *testing.T) { - in := &CryoConfig{ - IntegrationEnabled: true, - RestoreEnabled: false, - ServerURL: "http://x:1", - WarmupTimeoutSeconds: 900, - WarmupBufferSeconds: 10, - } - out := in.DeepCopy() - if out == in { - t.Fatal("DeepCopy returned the same pointer") - } - if *out != *in { - t.Errorf("DeepCopy contents differ: in=%+v out=%+v", in, out) - } - out.ServerURL = "mutated" - if in.ServerURL == "mutated" { - t.Error("mutation on DeepCopy leaked into input") - } -} - -func TestClusterConfigDeepCopyIncludesCryo(t *testing.T) { - in := &ClusterConfig{ - ClusterName: "c1", - Cryo: &CryoConfig{ - IntegrationEnabled: true, - ServerURL: "http://x:1", - }, - } - out := in.DeepCopy() - if out.Cryo == in.Cryo { - t.Fatal("ClusterConfig.DeepCopy shared the Cryo pointer with input") - } - out.Cryo.ServerURL = "mutated" - if in.Cryo.ServerURL == "mutated" { - t.Error("mutation on copied Cryo leaked into input") - } -} diff --git a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/generated.openapi.go b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/generated.openapi.go index a02b5f2a7..cc01a7338 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/generated.openapi.go +++ b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/generated.openapi.go @@ -34,7 +34,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA return map[string]common.OpenAPIDefinition{ "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.AccountConfig": schema_pkg_apis_nvcf_v1_AccountConfig(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.ClusterConfig": schema_pkg_apis_nvcf_v1_ClusterConfig(ref), - "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.CryoConfig": schema_pkg_apis_nvcf_v1_CryoConfig(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.DeploymentConfig": schema_pkg_apis_nvcf_v1_DeploymentConfig(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.FNDServiceConfig": schema_pkg_apis_nvcf_v1_FNDServiceConfig(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.FeatureGate": schema_pkg_apis_nvcf_v1_FeatureGate(ref), @@ -48,6 +47,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.NVCFBackend": schema_pkg_apis_nvcf_v1_NVCFBackend(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.NVCFBackendList": schema_pkg_apis_nvcf_v1_NVCFBackendList(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.NVCFBackendSpec": schema_pkg_apis_nvcf_v1_NVCFBackendSpec(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.NvSnapConfig": schema_pkg_apis_nvcf_v1_NvSnapConfig(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.OAuthConfig": schema_pkg_apis_nvcf_v1_OAuthConfig(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.OTELConfig": schema_pkg_apis_nvcf_v1_OTELConfig(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.OTelCollectorConfig": schema_pkg_apis_nvcf_v1_OTelCollectorConfig(ref), @@ -55,55 +55,55 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.SharedStorageSpec": schema_pkg_apis_nvcf_v1_SharedStorageSpec(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.VaultConfig": schema_pkg_apis_nvcf_v1_VaultConfig(ref), "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.WebhookConfig": schema_pkg_apis_nvcf_v1_WebhookConfig(ref), - metav1.APIGroup{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroup(ref), - metav1.APIGroupList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroupList(ref), - metav1.APIResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResource(ref), - metav1.APIResourceList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResourceList(ref), - metav1.APIVersions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIVersions(ref), - metav1.ApplyOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ApplyOptions(ref), - metav1.Condition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Condition(ref), - metav1.CreateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_CreateOptions(ref), - metav1.DeleteOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_DeleteOptions(ref), - metav1.Duration{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Duration(ref), - metav1.FieldSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldSelectorRequirement(ref), - metav1.FieldsV1{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldsV1(ref), - metav1.GetOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GetOptions(ref), - metav1.GroupKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupKind(ref), - metav1.GroupResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupResource(ref), - metav1.GroupVersion{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersion(ref), - metav1.GroupVersionForDiscovery{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref), - metav1.GroupVersionKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionKind(ref), - metav1.GroupVersionResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionResource(ref), - metav1.InternalEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_InternalEvent(ref), - metav1.LabelSelector{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelector(ref), - metav1.LabelSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref), - metav1.List{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_List(ref), - metav1.ListMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListMeta(ref), - metav1.ListOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListOptions(ref), - metav1.ManagedFieldsEntry{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref), - metav1.MicroTime{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_MicroTime(ref), - metav1.ObjectMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ObjectMeta(ref), - metav1.OwnerReference{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_OwnerReference(ref), - metav1.PartialObjectMetadata{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadata(ref), - metav1.PartialObjectMetadataList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref), - metav1.Patch{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Patch(ref), - metav1.PatchOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PatchOptions(ref), - metav1.Preconditions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Preconditions(ref), - metav1.RootPaths{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_RootPaths(ref), - metav1.ServerAddressByClientCIDR{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref), - metav1.Status{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Status(ref), - metav1.StatusCause{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusCause(ref), - metav1.StatusDetails{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusDetails(ref), - metav1.Table{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Table(ref), - metav1.TableColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableColumnDefinition(ref), - metav1.TableOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableOptions(ref), - metav1.TableRow{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRow(ref), - metav1.TableRowCondition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRowCondition(ref), - metav1.Time{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Time(ref), - metav1.Timestamp{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Timestamp(ref), - metav1.TypeMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TypeMeta(ref), - metav1.UpdateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_UpdateOptions(ref), - metav1.WatchEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_WatchEvent(ref), + metav1.APIGroup{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroup(ref), + metav1.APIGroupList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroupList(ref), + metav1.APIResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResource(ref), + metav1.APIResourceList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResourceList(ref), + metav1.APIVersions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIVersions(ref), + metav1.ApplyOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ApplyOptions(ref), + metav1.Condition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Condition(ref), + metav1.CreateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_CreateOptions(ref), + metav1.DeleteOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_DeleteOptions(ref), + metav1.Duration{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Duration(ref), + metav1.FieldSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldSelectorRequirement(ref), + metav1.FieldsV1{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldsV1(ref), + metav1.GetOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GetOptions(ref), + metav1.GroupKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupKind(ref), + metav1.GroupResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupResource(ref), + metav1.GroupVersion{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersion(ref), + metav1.GroupVersionForDiscovery{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref), + metav1.GroupVersionKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionKind(ref), + metav1.GroupVersionResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionResource(ref), + metav1.InternalEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_InternalEvent(ref), + metav1.LabelSelector{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelector(ref), + metav1.LabelSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref), + metav1.List{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_List(ref), + metav1.ListMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListMeta(ref), + metav1.ListOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListOptions(ref), + metav1.ManagedFieldsEntry{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref), + metav1.MicroTime{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_MicroTime(ref), + metav1.ObjectMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ObjectMeta(ref), + metav1.OwnerReference{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_OwnerReference(ref), + metav1.PartialObjectMetadata{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadata(ref), + metav1.PartialObjectMetadataList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref), + metav1.Patch{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Patch(ref), + metav1.PatchOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PatchOptions(ref), + metav1.Preconditions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Preconditions(ref), + metav1.RootPaths{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_RootPaths(ref), + metav1.ServerAddressByClientCIDR{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref), + metav1.Status{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Status(ref), + metav1.StatusCause{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusCause(ref), + metav1.StatusDetails{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusDetails(ref), + metav1.Table{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Table(ref), + metav1.TableColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableColumnDefinition(ref), + metav1.TableOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableOptions(ref), + metav1.TableRow{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRow(ref), + metav1.TableRowCondition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRowCondition(ref), + metav1.Time{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Time(ref), + metav1.Timestamp{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Timestamp(ref), + metav1.TypeMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TypeMeta(ref), + metav1.UpdateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_UpdateOptions(ref), + metav1.WatchEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_WatchEvent(ref), } } @@ -267,10 +267,10 @@ func schema_pkg_apis_nvcf_v1_ClusterConfig(ref common.ReferenceCallback) common. }, }, }, - "cryo": { + "nvsnap": { SchemaProps: spec.SchemaProps{ - Description: "Cryo configures the per-cluster behavior of the Cryo checkpoint/restore integration. Empty (or unset Enabled flag) → integration disabled on this cluster; pod creation behaves identically to today. The global featureflag.CryoCheckpointRestore short-circuits this field — both must be true for either hook to fire. See docs/users/cryo/CRYO-INTEGRATION-DESIGN.md.", - Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.CryoConfig"), + Description: "NvSnap configures the per-cluster behavior of the NvSnap checkpoint/restore integration. Empty (or unset Enabled flag) → integration disabled on this cluster; pod creation behaves identically to today. The global featureflag.NvSnapCheckpointRestore short-circuits this field — both must be true for either hook to fire. See docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md.", + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.NvSnapConfig"), }, }, }, @@ -278,55 +278,7 @@ func schema_pkg_apis_nvcf_v1_ClusterConfig(ref common.ReferenceCallback) common. }, }, Dependencies: []string{ - "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.CryoConfig", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.FNDServiceConfig", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.GPUDiscoveryConfig", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.MiniServiceConfig"}, - } -} - -func schema_pkg_apis_nvcf_v1_CryoConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "CryoConfig is the per-cluster configuration for the Cryo checkpoint/restore integration. All fields default to \"off\" or to the in-cluster Service URL — a default-constructed CryoConfig is a no-op.\n\nLayered disable order (any false short-circuits):\n\n\tfeatureflag.CryoCheckpointRestore (global kill switch)\n\t→ CryoConfig.IntegrationEnabled (per-cluster)\n\t→ CryoFunctionState.spec.optOut (per-function-version)\n\nIntegrationEnabled controls the checkpoint side (Hook B). RestoreEnabled controls the restore side (Hook A) — split so we can roll out checkpoint-only first, then restore once we've gathered timing data.", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "integrationEnabled": { - SchemaProps: spec.SchemaProps{ - Description: "IntegrationEnabled turns on the post-Ready checkpoint reconciler (Hook B). When false, no pod is checkpointed on this cluster.", - Type: []string{"boolean"}, - Format: "", - }, - }, - "restoreEnabled": { - SchemaProps: spec.SchemaProps{ - Description: "RestoreEnabled turns on the restore-on-create hook (Hook A). When false, NVCA does not stamp cryo.io/restore-from on new pods even if a checkpoint hash exists in NGC. Lets us validate checkpoint capture independently of restore.", - Type: []string{"boolean"}, - Format: "", - }, - }, - "serverURL": { - SchemaProps: spec.SchemaProps{ - Description: "ServerURL is the cryo-server endpoint NVCA's checkpoint reconciler POSTs to. Empty → DefaultCryoServerURL (in-cluster Service).", - Type: []string{"string"}, - Format: "", - }, - }, - "warmupTimeoutSeconds": { - SchemaProps: spec.SchemaProps{ - Description: "WarmupTimeoutSeconds caps the per-pod health-check polling window before NVCA gives up on triggering a checkpoint. 0 → DefaultCryoWarmupTimeoutSeconds (15 min).", - Type: []string{"integer"}, - Format: "int32", - }, - }, - "warmupBufferSeconds": { - SchemaProps: spec.SchemaProps{ - Description: "WarmupBufferSeconds is the dwell time between first /health 200 and the checkpoint POST. 0 → DefaultCryoWarmupBufferSeconds (10s).", - Type: []string{"integer"}, - Format: "int32", - }, - }, - }, - }, - }, + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.FNDServiceConfig", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.GPUDiscoveryConfig", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.MiniServiceConfig", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1.NvSnapConfig"}, } } @@ -846,6 +798,54 @@ func schema_pkg_apis_nvcf_v1_NVCFBackendSpec(ref common.ReferenceCallback) commo } } +func schema_pkg_apis_nvcf_v1_NvSnapConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "NvSnapConfig is the per-cluster configuration for the NvSnap checkpoint/restore integration. All fields default to \"off\" or to the in-cluster Service URL — a default-constructed NvSnapConfig is a no-op.\n\nLayered disable order (any false short-circuits):\n\n\tfeatureflag.NvSnapCheckpointRestore (global kill switch)\n\t→ NvSnapConfig.IntegrationEnabled (per-cluster)\n\t→ NvSnapFunctionState.spec.optOut (per-function-version)\n\nIntegrationEnabled controls the checkpoint side (Hook B). RestoreEnabled controls the restore side (Hook A) — split so we can roll out checkpoint-only first, then restore once we've gathered timing data.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "integrationEnabled": { + SchemaProps: spec.SchemaProps{ + Description: "IntegrationEnabled turns on the post-Ready checkpoint reconciler (Hook B). When false, no pod is checkpointed on this cluster.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "restoreEnabled": { + SchemaProps: spec.SchemaProps{ + Description: "RestoreEnabled turns on the restore-on-create hook (Hook A). When false, NVCA does not stamp nvsnap.io/restore-from on new pods even if a checkpoint hash exists in NGC. Lets us validate checkpoint capture independently of restore.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "serverURL": { + SchemaProps: spec.SchemaProps{ + Description: "ServerURL is the nvsnap-server endpoint NVCA's checkpoint reconciler POSTs to. Empty → DefaultNvSnapServerURL (in-cluster Service).", + Type: []string{"string"}, + Format: "", + }, + }, + "warmupTimeoutSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "WarmupTimeoutSeconds caps the per-pod health-check polling window before NVCA gives up on triggering a checkpoint. 0 → DefaultNvSnapWarmupTimeoutSeconds (15 min).", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "warmupBufferSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "WarmupBufferSeconds is the dwell time between first /health 200 and the checkpoint POST. Default 0 (no extra dwell — the readiness probe is the contract). Set explicitly to override per-NVCFBackend for workloads that need post-Ready dwell time (e.g., lazy-JIT engines that finish compiling shortly after the first 200). The previous semantics (\"0 means use 10s default\") conflated \"I want zero\" with \"I didn't set this\", so this field no longer treats 0 as a sentinel; the value you set is the value used.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + } +} + func schema_pkg_apis_nvcf_v1_OAuthConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvcfbackend_types.go b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvcfbackend_types.go index 039f98818..afed44f30 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvcfbackend_types.go +++ b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvcfbackend_types.go @@ -97,13 +97,13 @@ type ClusterConfig struct { Attributes []string `json:"attributes,omitempty"` K8sClusterNetworkCIDRs []string `json:"k8sClusterNetworkCIDRs,omitempty"` - // Cryo configures the per-cluster behavior of the Cryo + // NvSnap configures the per-cluster behavior of the NvSnap // checkpoint/restore integration. Empty (or unset Enabled flag) // → integration disabled on this cluster; pod creation behaves - // identically to today. The global featureflag.CryoCheckpointRestore + // identically to today. The global featureflag.NvSnapCheckpointRestore // short-circuits this field — both must be true for either hook - // to fire. See docs/users/cryo/CRYO-INTEGRATION-DESIGN.md. - Cryo *CryoConfig `json:"cryo,omitempty"` + // to fire. See docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md. + NvSnap *NvSnapConfig `json:"nvsnap,omitempty"` } // +k8s:openapi-gen=true @@ -217,32 +217,37 @@ type MiniServiceConfig struct { } const ( - // DefaultCryoServerURL is the in-cluster cryo-server Service URL. + // DefaultNvSnapServerURL is the in-cluster nvsnap-server Service URL. // Operators can override per cluster for off-cluster deployments - // (e.g., a hosted cryo-server fronting a fleet of NVCA clusters). - DefaultCryoServerURL = "http://cryo-server.cryo-system.svc.cluster.local:8080" + // (e.g., a hosted nvsnap-server fronting a fleet of NVCA clusters). + DefaultNvSnapServerURL = "http://nvsnap-server.nvsnap-system.svc.cluster.local:8080" - // DefaultCryoWarmupTimeoutSeconds caps how long NVCA polls + // DefaultNvSnapWarmupTimeoutSeconds caps how long NVCA polls // /v1/health/ready on the inference container before giving up on // a checkpoint attempt. 15 min covers TRT-LLM 70B engine compile. - DefaultCryoWarmupTimeoutSeconds = 15 * 60 - - // DefaultCryoWarmupBufferSeconds is the gap between the first - // /health 200 and the actual checkpoint POST — lets post-response - // setup (CUDA graph capture, lazy JIT) land before the snapshot. - DefaultCryoWarmupBufferSeconds = 10 + DefaultNvSnapWarmupTimeoutSeconds = 15 * 60 + + // DefaultNvSnapWarmupBufferSeconds is the gap between the first + // /health 200 and the actual checkpoint POST. Flipped from 10 → 0 + // in the v0.0.49 / rootfs-everywhere design: an honest readiness + // probe is the contract; padding every capture by 10s adds latency + // for no benefit and can capture mid-flight CUDA-graph builds that + // destabilize the snapshot. Operators that need a workload-specific + // dwell can still set spec.nvsnap.warmupBufferSeconds explicitly + // per-NVCFBackend. + DefaultNvSnapWarmupBufferSeconds = 0 ) -// CryoConfig is the per-cluster configuration for the Cryo +// NvSnapConfig is the per-cluster configuration for the NvSnap // checkpoint/restore integration. All fields default to "off" or to -// the in-cluster Service URL — a default-constructed CryoConfig is +// the in-cluster Service URL — a default-constructed NvSnapConfig is // a no-op. // // Layered disable order (any false short-circuits): // -// featureflag.CryoCheckpointRestore (global kill switch) -// → CryoConfig.IntegrationEnabled (per-cluster) -// → CryoFunctionState.spec.optOut (per-function-version) +// featureflag.NvSnapCheckpointRestore (global kill switch) +// → NvSnapConfig.IntegrationEnabled (per-cluster) +// → NvSnapFunctionState.spec.optOut (per-function-version) // // IntegrationEnabled controls the checkpoint side (Hook B). // RestoreEnabled controls the restore side (Hook A) — split so we @@ -250,60 +255,67 @@ const ( // gathered timing data. // // +k8s:openapi-gen=true -type CryoConfig struct { +type NvSnapConfig struct { // IntegrationEnabled turns on the post-Ready checkpoint reconciler // (Hook B). When false, no pod is checkpointed on this cluster. IntegrationEnabled bool `json:"integrationEnabled,omitempty"` // RestoreEnabled turns on the restore-on-create hook (Hook A). - // When false, NVCA does not stamp cryo.io/restore-from on new + // When false, NVCA does not stamp nvsnap.io/restore-from on new // pods even if a checkpoint hash exists in NGC. Lets us validate // checkpoint capture independently of restore. RestoreEnabled bool `json:"restoreEnabled,omitempty"` - // ServerURL is the cryo-server endpoint NVCA's checkpoint reconciler - // POSTs to. Empty → DefaultCryoServerURL (in-cluster Service). + // ServerURL is the nvsnap-server endpoint NVCA's checkpoint reconciler + // POSTs to. Empty → DefaultNvSnapServerURL (in-cluster Service). ServerURL string `json:"serverURL,omitempty"` // WarmupTimeoutSeconds caps the per-pod health-check polling // window before NVCA gives up on triggering a checkpoint. 0 → - // DefaultCryoWarmupTimeoutSeconds (15 min). + // DefaultNvSnapWarmupTimeoutSeconds (15 min). WarmupTimeoutSeconds int32 `json:"warmupTimeoutSeconds,omitempty"` // WarmupBufferSeconds is the dwell time between first /health 200 - // and the checkpoint POST. 0 → DefaultCryoWarmupBufferSeconds (10s). + // and the checkpoint POST. Default 0 (no extra dwell — the + // readiness probe is the contract). Set explicitly to override + // per-NVCFBackend for workloads that need post-Ready dwell time + // (e.g., lazy-JIT engines that finish compiling shortly after the + // first 200). The previous semantics ("0 means use 10s default") + // conflated "I want zero" with "I didn't set this", so this field + // no longer treats 0 as a sentinel; the value you set is the + // value used. WarmupBufferSeconds int32 `json:"warmupBufferSeconds,omitempty"` } -// Complete returns a copy of CryoConfig with empty fields filled from +// Complete returns a copy of NvSnapConfig with empty fields filled from // the defaults. Returns a non-nil pointer even when called on nil, // so callers can rely on the result for field reads without nil // checks. The returned value has the same enable bits as the input // (defaults do not opt anything in). -func (c *CryoConfig) Complete(_ EnvType) *CryoConfig { - var tmp *CryoConfig +func (c *NvSnapConfig) Complete(_ EnvType) *NvSnapConfig { + var tmp *NvSnapConfig if c == nil { - tmp = &CryoConfig{} + tmp = &NvSnapConfig{} } else { tmp = c.DeepCopy() } if tmp.ServerURL == "" { - tmp.ServerURL = DefaultCryoServerURL + tmp.ServerURL = DefaultNvSnapServerURL } if tmp.WarmupTimeoutSeconds == 0 { - tmp.WarmupTimeoutSeconds = DefaultCryoWarmupTimeoutSeconds - } - if tmp.WarmupBufferSeconds == 0 { - tmp.WarmupBufferSeconds = DefaultCryoWarmupBufferSeconds + tmp.WarmupTimeoutSeconds = DefaultNvSnapWarmupTimeoutSeconds } + // WarmupBufferSeconds: NO fallback. The value you set is the + // value used. Default 0 (DefaultNvSnapWarmupBufferSeconds) is a + // real choice, not a sentinel — see the field's doc comment. return tmp } // IsEnabled reports whether the per-cluster integration is on AT ALL // — either checkpoint-side, restore-side, or both. The global feature -// flag (featureflag.CryoCheckpointRestore) is a separate AND-gate +// flag (featureflag.NvSnapCheckpointRestore) is a separate AND-gate // checked by callers. -func (c *CryoConfig) IsEnabled() bool { +func (c *NvSnapConfig) IsEnabled() bool { return c != nil && (c.IntegrationEnabled || c.RestoreEnabled) } diff --git a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvsnap_config_test.go b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvsnap_config_test.go new file mode 100644 index 000000000..f4eb413cc --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvsnap_config_test.go @@ -0,0 +1,163 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package v1 + +import "testing" + +func TestNvSnapConfigCompleteNil(t *testing.T) { + var c *NvSnapConfig + got := c.Complete(EnvTypeStage) + if got == nil { + t.Fatal("Complete(nil) returned nil; should return defaulted NvSnapConfig") + } + if got.IntegrationEnabled || got.RestoreEnabled { + t.Errorf("nil → defaults should leave both disabled, got %+v", got) + } + if got.ServerURL != DefaultNvSnapServerURL { + t.Errorf("ServerURL = %q, want %q", got.ServerURL, DefaultNvSnapServerURL) + } + if got.WarmupTimeoutSeconds != DefaultNvSnapWarmupTimeoutSeconds { + t.Errorf("WarmupTimeoutSeconds = %d, want %d", got.WarmupTimeoutSeconds, DefaultNvSnapWarmupTimeoutSeconds) + } + if got.WarmupBufferSeconds != DefaultNvSnapWarmupBufferSeconds { + t.Errorf("WarmupBufferSeconds = %d, want %d", got.WarmupBufferSeconds, DefaultNvSnapWarmupBufferSeconds) + } +} + +func TestNvSnapConfigCompletePreservesOverrides(t *testing.T) { + in := &NvSnapConfig{ + IntegrationEnabled: true, + RestoreEnabled: true, + ServerURL: "http://my-nvsnap.example.com:8080", + WarmupTimeoutSeconds: 600, + WarmupBufferSeconds: 30, + } + got := in.Complete(EnvTypeProd) + if !got.IntegrationEnabled || !got.RestoreEnabled { + t.Errorf("Complete should preserve enable bits; got %+v", got) + } + if got.ServerURL != "http://my-nvsnap.example.com:8080" { + t.Errorf("ServerURL override not preserved: %q", got.ServerURL) + } + if got.WarmupTimeoutSeconds != 600 || got.WarmupBufferSeconds != 30 { + t.Errorf("warmup overrides not preserved: %+v", got) + } +} + +func TestNvSnapConfigCompletePartialOverride(t *testing.T) { + // Enabled but no URL set — defaults should fill the URL. + in := &NvSnapConfig{IntegrationEnabled: true} + got := in.Complete(EnvTypeStage) + if !got.IntegrationEnabled { + t.Errorf("IntegrationEnabled lost across Complete") + } + if got.ServerURL != DefaultNvSnapServerURL { + t.Errorf("ServerURL should default when empty, got %q", got.ServerURL) + } +} + +func TestNvSnapConfigCompleteReturnsCopy(t *testing.T) { + in := &NvSnapConfig{IntegrationEnabled: true} + got := in.Complete(EnvTypeStage) + got.IntegrationEnabled = false + if !in.IntegrationEnabled { + t.Error("Complete returned a reference to input — should be a deep copy") + } +} + +// v0.0.49 / rootfs-everywhere: DefaultNvSnapWarmupBufferSeconds flipped +// from 10 → 0, AND Complete() no longer treats 0 as a sentinel — the +// value the operator sets is the value used. This test pins both +// behaviors so a future reviewer can't silently re-introduce a +// default dwell time. +func TestNvSnapConfigCompleteWarmupBufferIsLiteralZero(t *testing.T) { + if DefaultNvSnapWarmupBufferSeconds != 0 { + t.Fatalf("DefaultNvSnapWarmupBufferSeconds = %d; v0.0.49 design pins it to 0", DefaultNvSnapWarmupBufferSeconds) + } + // Explicit zero must survive Complete() unchanged. + in := &NvSnapConfig{IntegrationEnabled: true, WarmupBufferSeconds: 0} + got := in.Complete(EnvTypeStage) + if got.WarmupBufferSeconds != 0 { + t.Errorf("Complete() must preserve explicit 0; got %d", got.WarmupBufferSeconds) + } + // And an explicit non-zero override still works. + in2 := &NvSnapConfig{IntegrationEnabled: true, WarmupBufferSeconds: 5} + got2 := in2.Complete(EnvTypeStage) + if got2.WarmupBufferSeconds != 5 { + t.Errorf("Complete() must preserve explicit non-zero; got %d", got2.WarmupBufferSeconds) + } +} + +func TestNvSnapConfigIsEnabled(t *testing.T) { + cases := []struct { + name string + c *NvSnapConfig + want bool + }{ + {"nil", nil, false}, + {"zero", &NvSnapConfig{}, false}, + {"checkpoint only", &NvSnapConfig{IntegrationEnabled: true}, true}, + {"restore only", &NvSnapConfig{RestoreEnabled: true}, true}, + {"both", &NvSnapConfig{IntegrationEnabled: true, RestoreEnabled: true}, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.c.IsEnabled(); got != tc.want { + t.Errorf("IsEnabled() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestNvSnapConfigDeepCopy(t *testing.T) { + in := &NvSnapConfig{ + IntegrationEnabled: true, + RestoreEnabled: false, + ServerURL: "http://x:1", + WarmupTimeoutSeconds: 900, + WarmupBufferSeconds: 10, + } + out := in.DeepCopy() + if out == in { + t.Fatal("DeepCopy returned the same pointer") + } + if *out != *in { + t.Errorf("DeepCopy contents differ: in=%+v out=%+v", in, out) + } + out.ServerURL = "mutated" + if in.ServerURL == "mutated" { + t.Error("mutation on DeepCopy leaked into input") + } +} + +func TestClusterConfigDeepCopyIncludesNvSnap(t *testing.T) { + in := &ClusterConfig{ + ClusterName: "c1", + NvSnap: &NvSnapConfig{ + IntegrationEnabled: true, + ServerURL: "http://x:1", + }, + } + out := in.DeepCopy() + if out.NvSnap == in.NvSnap { + t.Fatal("ClusterConfig.DeepCopy shared the NvSnap pointer with input") + } + out.NvSnap.ServerURL = "mutated" + if in.NvSnap.ServerURL == "mutated" { + t.Error("mutation on copied NvSnap leaked into input") + } +} diff --git a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/zz_generated.deepcopy.go b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/zz_generated.deepcopy.go index d1e9be434..2695ed191 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/zz_generated.deepcopy.go +++ b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/zz_generated.deepcopy.go @@ -122,9 +122,9 @@ func (in *ClusterConfig) DeepCopyInto(out *ClusterConfig) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.Cryo != nil { - in, out := &in.Cryo, &out.Cryo - *out = new(CryoConfig) + if in.NvSnap != nil { + in, out := &in.NvSnap, &out.NvSnap + *out = new(NvSnapConfig) **out = **in } return @@ -140,22 +140,6 @@ func (in *ClusterConfig) DeepCopy() *ClusterConfig { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CryoConfig) DeepCopyInto(out *CryoConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CryoConfig. -func (in *CryoConfig) DeepCopy() *CryoConfig { - if in == nil { - return nil - } - out := new(CryoConfig) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeploymentConfig) DeepCopyInto(out *DeploymentConfig) { *out = *in @@ -598,6 +582,22 @@ func (in *NVCFWorkerConfig) DeepCopy() *NVCFWorkerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NvSnapConfig) DeepCopyInto(out *NvSnapConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NvSnapConfig. +func (in *NvSnapConfig) DeepCopy() *NvSnapConfig { + if in == nil { + return nil + } + out := new(NvSnapConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OAuthConfig) DeepCopyInto(out *OAuthConfig) { *out = *in diff --git a/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/BUILD.bazel b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/BUILD.bazel new file mode 100644 index 000000000..5b3493e30 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/BUILD.bazel @@ -0,0 +1,31 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "v1alpha1", + srcs = [ + "doc.go", + "nvsnapfunctionstate_types.go", + "register.go", + "zz_generated.deepcopy.go", + ], + importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1", + visibility = ["//visibility:public"], + deps = [ + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", + "//vendor/k8s.io/apimachinery/pkg/runtime", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema", + ], +) + +alias( + name = "go_default_library", + actual = ":v1alpha1", + visibility = ["//visibility:public"], +) + +go_test( + name = "v1alpha1_test", + srcs = ["nvsnapfunctionstate_types_test.go"], + embed = [":v1alpha1"], + deps = ["//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta"], +) diff --git a/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/doc.go b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/doc.go new file mode 100644 index 000000000..e1f3a0739 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/doc.go @@ -0,0 +1,26 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// Package v1alpha1 contains the per-cluster NvSnap state CRDs read and +// written by NVCA's checkpoint/restore reconciler (Hook B) and stamped +// onto restoring pods by NVCA's pod-creation hook (Hook A). +// +// The canonical authority for "which function-version has a checkpoint +// hash" is NGC (the global function-version object). This CRD tracks +// per-cluster operational state — local cache status, retry counters, +// last-error, opt-outs — that NGC shouldn't carry. See +// docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md §"Where exactly does the +// hash live" for the split-storage rationale. +// +// +k8s:deepcopy-gen=package +// +k8s:defaulter-gen=TypeMeta +// +groupName=nvsnap.nvcf.nvidia.io +package v1alpha1 diff --git a/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/nvsnapfunctionstate_types.go b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/nvsnapfunctionstate_types.go new file mode 100644 index 000000000..c5fa8b0c0 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/nvsnapfunctionstate_types.go @@ -0,0 +1,231 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NvSnapFunctionStateLocalCacheState enumerates the per-cluster cache +// states for a checkpoint hash. Reflects whether the bytes are +// reachable from this cluster's local nvsnap-blobstore / per-node +// hostPath caches, independent of whether the global hash exists. +type NvSnapFunctionStateLocalCacheState string + +const ( + // LocalCacheStateCold = no copy on this cluster. Restore will + // cold-start until the cascade fetches from the source cluster + // (cross-cluster replication via S3, when that lands). + LocalCacheStateCold NvSnapFunctionStateLocalCacheState = "Cold" + + // LocalCacheStateFetching = replication is in progress. NVCA + // won't stamp nvsnap.io/restore-from while in this state because + // the bytes might not be local yet by pod-create time. + LocalCacheStateFetching NvSnapFunctionStateLocalCacheState = "Fetching" + + // LocalCacheStateCapturing = a checkpoint capture is in flight for + // this function-version on this cluster. Set via an optimistic- + // concurrency compare-and-swap (Cold→Capturing) before Hook B + // issues the checkpoint POST, so that when N pods of the same + // function-version admit cold simultaneously, exactly one reconcile + // wins the transition and runs the capture; the others observe + // Capturing and back off rather than firing N parallel captures + // (nvca#189 capture-once / thundering-herd guard). Bounded by + // status.CaptureOwner + status.CaptureLeaseExpiry so a crashed + // capturer's claim expires and a fresh pod can retry rather than + // the function being pinned Capturing forever. + LocalCacheStateCapturing NvSnapFunctionStateLocalCacheState = "Capturing" + + // LocalCacheStateWarm = bytes are local (either captured here or + // fetched from another cluster). Restore-on-create can proceed. + LocalCacheStateWarm NvSnapFunctionStateLocalCacheState = "Warm" + + // LocalCacheStateFailed = last fetch attempt failed; see + // status.LastError. Reconciler will retry with backoff. + LocalCacheStateFailed NvSnapFunctionStateLocalCacheState = "Failed" +) + +// +genclient +// +genclient:nonNamespaced +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// NvSnapFunctionState records per-cluster NvSnap state for one NGC +// function-version. Cluster-scoped: the global hash + checkpoint +// existence live in NGC; this CR holds operational state that's +// per-cluster (which cluster captured it, local cache state, retry +// history, opt-outs). +// +// One NvSnapFunctionState per function-version, named after the +// function-version UUID — keeps NGC's canonical id the lookup key. +// +// Lifecycle: +// +// NVCA pod-creation sees a deploy for function-version V. +// → Reads NvSnapFunctionState/V from this cluster's API server. +// → If !optOut && status.LocalCacheState == Warm && status.CheckpointHash != "", +// stamps nvsnap.io/restore-from = checkpointHash on the pod. +// → Otherwise pod cold-starts (existing behavior). +// +// First pod for V reaches /health 200 + buffer. +// → Hook B reconciler POSTs nvsnap-server /api/v1/checkpoints. +// → On terminal Completed: updates status.CheckpointHash, +// status.CapturedHere=true, status.LocalCacheState=Warm, +// writes hash to NGC. +type NvSnapFunctionState struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NvSnapFunctionStateSpec `json:"spec,omitempty"` + Status NvSnapFunctionStateStatus `json:"status,omitempty"` +} + +// +k8s:openapi-gen=true +type NvSnapFunctionStateSpec struct { + // FunctionVersionID is the NGC function-version UUID this CR + // tracks. Matches metadata.name; required for callers that read + // the spec without parsing the name. + FunctionVersionID string `json:"functionVersionID"` + + // OptOut excludes this function-version from NvSnap checkpoint AND + // restore on this cluster. Useful for workloads with state that + // can't be safely restored (e.g., engines that bake host IDs into + // memory). Operators set this on a per-cluster basis; NGC may + // expose a global opt-out separately. + OptOut bool `json:"optOut,omitempty"` + + // WarmupTimeoutSecondsOverride is an optional per-function-version + // override of NvSnapConfig.WarmupTimeoutSeconds. Useful when a + // specific workload takes much longer to warm than the cluster + // default (e.g., a custom TRT-LLM build with long engine compile). + // Zero = use cluster default. + WarmupTimeoutSecondsOverride int32 `json:"warmupTimeoutSecondsOverride,omitempty"` + + // WorkloadLookup carries the content-addressed lookup inputs for + // this function-version, persisted by Hook B at capture time. It + // lets the controller's periodic CFS recovery sweep find an + // existing capture in nvsnap-server and flip LocalCacheState=Warm + // WITHOUT a live source pod — closing the gap where an interrupted + // reconcile (controller restart, lost goroutine, source pod scaled + // away) leaves CFS stuck not-Warm forever (nvca#104 durable-warm, + // see docs/users/nvsnap/DURABLE-WARM-SWEEP.md). Empty ImageRef = not + // yet persisted; the sweep skips such CFS. + WorkloadLookup WorkloadLookupSpec `json:"workloadLookup,omitempty"` +} + +// WorkloadLookupSpec is the subset of a pod's workload identity the +// controller needs to query nvsnap-server's content-addressed lookup +// (POST /api/v1/checkpoints/lookup) without holding the pod. Mirrors +// the inputs nvsnap_hook.go and recover.go derive from the inference +// container. +// +// +k8s:openapi-gen=true +type WorkloadLookupSpec struct { + // ImageRef is the inference container's image string. + ImageRef string `json:"imageRef,omitempty"` + // ModelID is the model identifier pulled from the container env or + // --model args (empty = match any model for the image). + ModelID string `json:"modelId,omitempty"` +} + +// +k8s:openapi-gen=true +type NvSnapFunctionStateStatus struct { + // CheckpointHash is the content-addressed sha256 of the captured + // state, mirrored from NGC for fast local reads. Empty until + // the first successful checkpoint anywhere in the fleet + // (the canonical source is NGC; this is the local cache). + CheckpointHash string `json:"checkpointHash,omitempty"` + + // CapturedHere is true iff THIS cluster ran the checkpoint that + // produced CheckpointHash. False = the hash was captured on + // another cluster and replicated here (or the bytes aren't here + // yet). + CapturedHere bool `json:"capturedHere,omitempty"` + + // CapturedAt is the time the checkpoint was committed. RFC3339. + CapturedAt *metav1.Time `json:"capturedAt,omitempty"` + + // LocalCacheState is the readiness of the checkpoint bytes on + // this cluster. See LocalCacheState* constants. + LocalCacheState NvSnapFunctionStateLocalCacheState `json:"localCacheState,omitempty"` + + // LastAttemptAt is the time of the most recent checkpoint + // attempt (regardless of outcome). + LastAttemptAt *metav1.Time `json:"lastAttemptAt,omitempty"` + + // AttemptCount is the cumulative number of checkpoint attempts + // this CR has triggered (resets on successful Completed). + AttemptCount int32 `json:"attemptCount,omitempty"` + + // LastError is the error message from the most recent failed + // checkpoint attempt. Cleared on success. + LastError string `json:"lastError,omitempty"` + + // CaptureOwner is the namespace/name of the pod whose reconcile won + // the Cold→Capturing compare-and-swap and is running the in-flight + // capture (nvca#189 capture-once). Set alongside + // LocalCacheState=Capturing; cleared on any terminal write (Warm on + // success, or back to the prior state on failure). Lets a + // re-reconcile of the SAME owning pod proceed re-entrantly and lets + // observers attribute the in-flight claim. + CaptureOwner string `json:"captureOwner,omitempty"` + + // CaptureLeaseExpiry bounds how long a Capturing claim is honored. + // If the owning reconcile dies mid-capture (controller restart, + // lost leader election, ctx cancel) the claim would otherwise pin + // the function Capturing forever; once now > CaptureLeaseExpiry any + // pod may steal the claim and retry. Set to now + a TTL that + // provably exceeds the maximum legitimate capture duration + // (warmup buffer + checkpoint timeout + L2 promote timeout + + // margin) so a valid in-flight capture is never stolen. RFC3339. + CaptureLeaseExpiry *metav1.Time `json:"captureLeaseExpiry,omitempty"` + + // ColdStartPioneer is the namespace/name of the ICMSRequest whose + // MiniService-creation reconcile won the cold-start pioneer election + // (serialized-herd cold start). It backs the same compare-and-swap + // idea as CaptureOwner but applied one step earlier — at MiniService + // creation rather than at capture: when N replicas of one + // function-version deploy cold (no Warm checkpoint yet), exactly one + // "pioneer" is allowed to create its MiniService and cold-start; the + // others defer (requeue) until that pioneer's capture flips + // LocalCacheState=Warm, then they proceed and warm-restore via Hook A. + // Empty when no pioneer is elected. See TryClaimColdStartPioneer in + // pkg/nvca/nvsnap/reconciler/state.go. + // +optional + ColdStartPioneer string `json:"coldStartPioneer,omitempty"` + + // ColdStartPioneerExpiry bounds how long a ColdStartPioneer claim is + // honored (serialized-herd cold start). If the pioneer's reconcile + // dies before its capture warms the function, the claim would + // otherwise defer every other replica forever; once + // now > ColdStartPioneerExpiry any replica may steal the pioneer slot + // and cold-start itself. Set to now + the same lease TTL as + // CaptureLeaseExpiry (it must cover cold-start + warmup + capture). + // RFC3339. + // +optional + ColdStartPioneerExpiry *metav1.Time `json:"coldStartPioneerExpiry,omitempty"` + + // Conditions follow the K8s standard format. Recommended: + // - type: Ready, status: True iff checkpoint is usable on this cluster + // - type: CaptureInProgress, status: True while Hook B is mid-flight + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type NvSnapFunctionStateList struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + + Items []NvSnapFunctionState `json:"items"` +} diff --git a/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/nvsnapfunctionstate_types_test.go b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/nvsnapfunctionstate_types_test.go new file mode 100644 index 000000000..e12cb4afe --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/nvsnapfunctionstate_types_test.go @@ -0,0 +1,132 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package v1alpha1 + +import ( + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestNvSnapFunctionStateDeepCopy(t *testing.T) { + now := metav1.NewTime(time.Now()) + in := &NvSnapFunctionState{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fv-uuid-1", + }, + Spec: NvSnapFunctionStateSpec{ + FunctionVersionID: "fv-uuid-1", + OptOut: false, + WarmupTimeoutSecondsOverride: 900, + }, + Status: NvSnapFunctionStateStatus{ + CheckpointHash: "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + CapturedHere: true, + CapturedAt: &now, + LocalCacheState: LocalCacheStateWarm, + AttemptCount: 1, + Conditions: []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "CaptureCommitted", Message: "ok"}, + }, + }, + } + + out := in.DeepCopy() + if out == in { + t.Fatal("DeepCopy returned the same pointer") + } + if out.Spec != in.Spec { + t.Errorf("Spec contents differ: in=%+v out=%+v", in.Spec, out.Spec) + } + if out.Status.CheckpointHash != in.Status.CheckpointHash { + t.Errorf("Status.CheckpointHash differ") + } + + // Mutate the copy; original must not change. + out.Status.CheckpointHash = "mutated" + if in.Status.CheckpointHash == "mutated" { + t.Error("mutation on DeepCopy leaked into input") + } + + // Conditions slice must be independent. + out.Status.Conditions[0].Message = "changed" + if in.Status.Conditions[0].Message == "changed" { + t.Error("Conditions slice was shared between in and out") + } + + // Time pointer must be independent. + if out.Status.CapturedAt == in.Status.CapturedAt { + t.Error("CapturedAt pointer was shared") + } +} + +func TestNvSnapFunctionStateListDeepCopy(t *testing.T) { + in := &NvSnapFunctionStateList{ + Items: []NvSnapFunctionState{ + {ObjectMeta: metav1.ObjectMeta{Name: "fv-1"}, Spec: NvSnapFunctionStateSpec{FunctionVersionID: "fv-1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "fv-2"}, Spec: NvSnapFunctionStateSpec{FunctionVersionID: "fv-2", OptOut: true}}, + }, + } + out := in.DeepCopy() + if len(out.Items) != 2 { + t.Fatalf("Items len = %d, want 2", len(out.Items)) + } + out.Items[0].Spec.OptOut = true + if in.Items[0].Spec.OptOut { + t.Error("Items slice was shared between in and out") + } +} + +func TestSchemeGroupVersionStable(t *testing.T) { + // Pinned: NVCA convention is *.nvcf.nvidia.io, and v1alpha1 means + // breaking changes are allowed. Tightening this test forces a + // conscious decision when bumping versions. + if SchemeGroupVersion.Group != "nvsnap.nvcf.nvidia.io" { + t.Errorf("Group = %q, want nvsnap.nvcf.nvidia.io", SchemeGroupVersion.Group) + } + if SchemeGroupVersion.Version != "v1alpha1" { + t.Errorf("Version = %q, want v1alpha1", SchemeGroupVersion.Version) + } +} + +func TestResourceGroupQualified(t *testing.T) { + gr := Resource("nvsnapfunctionstates") + if gr.Group != SchemeGroupVersion.Group { + t.Errorf("Group = %q, want %q", gr.Group, SchemeGroupVersion.Group) + } + if gr.Resource != "nvsnapfunctionstates" { + t.Errorf("Resource = %q", gr.Resource) + } +} + +func TestLocalCacheStateValues(t *testing.T) { + // Stable string values — NVCA agents in different versions may + // read the same status object, so changing these is a wire- + // protocol break. Guard with a test that fails loudly on rename. + want := map[NvSnapFunctionStateLocalCacheState]string{ + LocalCacheStateCold: "Cold", + LocalCacheStateFetching: "Fetching", + LocalCacheStateWarm: "Warm", + LocalCacheStateFailed: "Failed", + } + for state, expected := range want { + if string(state) != expected { + t.Errorf("%v = %q, want %q", state, string(state), expected) + } + } +} diff --git a/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/register.go b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/register.go new file mode 100644 index 000000000..2bf794870 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/register.go @@ -0,0 +1,62 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// SchemeGroupVersion for the NvSnap CRDs. +// +// Group "nvsnap.nvcf.nvidia.io" follows the NVCA convention +// (matching nvca.nvcf.nvidia.io etc.) — the CRDs are NVCA-managed +// state, not part of the NvSnap cluster service's API. The design doc +// originally proposed "nvsnap.nvidia.com" but consistency with sibling +// groups in this repo wins for clarity ("everything under +// .nvcf.nvidia.io is NVCA-related"). +var SchemeGroupVersion = schema.GroupVersion{ + Group: "nvsnap.nvcf.nvidia.io", + Version: "v1alpha1", +} + +var ( + SchemeBuilder runtime.SchemeBuilder + localSchemeBuilder = &SchemeBuilder + AddToScheme = localSchemeBuilder.AddToScheme +) + +func init() { + // Manual registrations only; generated registrations live in + // zz_generated.deepcopy.go-adjacent files. + localSchemeBuilder.Register(addKnownTypes) +} + +// Resource takes an unqualified resource and returns a Group qualified GroupResource. +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes( + SchemeGroupVersion, + &NvSnapFunctionState{}, + &NvSnapFunctionStateList{}, + ) + scheme.AddKnownTypes( + SchemeGroupVersion, + &metav1.Status{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/zz_generated.deepcopy.go b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..1623ddac5 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,160 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NvSnapFunctionState) DeepCopyInto(out *NvSnapFunctionState) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NvSnapFunctionState. +func (in *NvSnapFunctionState) DeepCopy() *NvSnapFunctionState { + if in == nil { + return nil + } + out := new(NvSnapFunctionState) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NvSnapFunctionState) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NvSnapFunctionStateList) DeepCopyInto(out *NvSnapFunctionStateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NvSnapFunctionState, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NvSnapFunctionStateList. +func (in *NvSnapFunctionStateList) DeepCopy() *NvSnapFunctionStateList { + if in == nil { + return nil + } + out := new(NvSnapFunctionStateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NvSnapFunctionStateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NvSnapFunctionStateSpec) DeepCopyInto(out *NvSnapFunctionStateSpec) { + *out = *in + out.WorkloadLookup = in.WorkloadLookup + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NvSnapFunctionStateSpec. +func (in *NvSnapFunctionStateSpec) DeepCopy() *NvSnapFunctionStateSpec { + if in == nil { + return nil + } + out := new(NvSnapFunctionStateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NvSnapFunctionStateStatus) DeepCopyInto(out *NvSnapFunctionStateStatus) { + *out = *in + if in.CapturedAt != nil { + in, out := &in.CapturedAt, &out.CapturedAt + *out = (*in).DeepCopy() + } + if in.LastAttemptAt != nil { + in, out := &in.LastAttemptAt, &out.LastAttemptAt + *out = (*in).DeepCopy() + } + if in.CaptureLeaseExpiry != nil { + in, out := &in.CaptureLeaseExpiry, &out.CaptureLeaseExpiry + *out = (*in).DeepCopy() + } + if in.ColdStartPioneerExpiry != nil { + in, out := &in.ColdStartPioneerExpiry, &out.ColdStartPioneerExpiry + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NvSnapFunctionStateStatus. +func (in *NvSnapFunctionStateStatus) DeepCopy() *NvSnapFunctionStateStatus { + if in == nil { + return nil + } + out := new(NvSnapFunctionStateStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadLookupSpec) DeepCopyInto(out *WorkloadLookupSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadLookupSpec. +func (in *WorkloadLookupSpec) DeepCopy() *WorkloadLookupSpec { + if in == nil { + return nil + } + out := new(WorkloadLookupSpec) + in.DeepCopyInto(out) + return out +} diff --git a/src/compute-plane-services/nvca/pkg/featureflag/featureflag.go b/src/compute-plane-services/nvca/pkg/featureflag/featureflag.go index c60b496cd..f7275accc 100644 --- a/src/compute-plane-services/nvca/pkg/featureflag/featureflag.go +++ b/src/compute-plane-services/nvca/pkg/featureflag/featureflag.go @@ -80,6 +80,14 @@ var ( // FCO operator-specific feature flags. To be removed once Phase 1 of the FCO SDD is in progress. DynamoOperatorSupport = newFeatureFlag("DynamoOperatorSupport", newBool(false)) + + // NvSnapCheckpointRestore is the global kill switch for the NvSnap + // integration (see docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md). + // When false, NVCA pod creation behaves exactly as before — no + // nvsnap.io/restore-from stamping on apply, no post-Ready checkpoint + // kick-off. Disabled by default. Per-cluster + per-function-version + // overrides land in a follow-up CRD update. + NvSnapCheckpointRestore = newFeatureFlag("NvSnapCheckpointRestore", newBool(false)) ) // Feature flags for migrating resource limits. diff --git a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel index 9e03ecfe8..aa39c1534 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel @@ -22,6 +22,10 @@ go_library( "k8scomputebackend_miniservice.go", "k8scomputebackend_modelcache.go", "k8scomputebackend_task_container.go", + "nvsnap_coldstart_gate.go", + "nvsnap_coldstart_metrics.go", + "nvsnap_controller_start.go", + "nvsnap_hook.go", "queue_manager.go", "transport_tls.go", "validator_summary_reconciler.go", @@ -50,6 +54,7 @@ go_library( "//pkg/apis/nvca/v1:nvca", "//pkg/apis/nvca/v1alpha1", "//pkg/apis/nvca/v2beta1", + "//pkg/apis/nvsnap/v1alpha1", "//pkg/client/clientset/versioned/scheme", "//pkg/client/informers/externalversions", "//pkg/client/listers/nvca/v2beta1", @@ -65,6 +70,9 @@ go_library( "//pkg/nvca/errors", "//pkg/nvca/fnds", "//pkg/nvca/health", + "//pkg/nvca/nvsnap", + "//pkg/nvca/nvsnap/controller", + "//pkg/nvca/nvsnap/reconciler", "//pkg/profiling", "//pkg/queue", "//pkg/queue/nats", @@ -91,6 +99,7 @@ go_library( "//vendor/github.com/google/go-cmp/cmp/cmpopts", "//vendor/github.com/google/uuid", "//vendor/github.com/prometheus/client_golang/prometheus", + "//vendor/github.com/prometheus/client_golang/prometheus/promauto", "//vendor/github.com/sirupsen/logrus", "//vendor/github.com/sourcegraph/conc/pool", "//vendor/github.com/spf13/cobra", @@ -171,6 +180,8 @@ go_test( "k8scomputebackend_modelcache_test.go", "k8scomputebackend_task_container_test.go", "k8scomputebackend_test.go", + "nvsnap_hook_lookup_test.go", + "nvsnap_hook_test.go", "queue_manager_test.go", "transport_tls_test.go", "validator_summary_reconciler_test.go", @@ -221,6 +232,7 @@ go_test( "//internal/util/translate", "//pkg/apis/nvca/v1alpha1", "//pkg/apis/nvca/v2beta1", + "//pkg/apis/nvsnap/v1alpha1", "//pkg/client/clientset/versioned/fake", "//pkg/client/informers/externalversions", "//pkg/client/listers/nvca/v2beta1", @@ -232,6 +244,7 @@ go_test( "//pkg/nvca/enforce", "//pkg/nvca/errors", "//pkg/nvca/health", + "//pkg/nvca/nvsnap", "//pkg/queue", "//pkg/queue/mock", "//pkg/queue/nats", @@ -273,6 +286,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured", "//vendor/k8s.io/apimachinery/pkg/labels", "//vendor/k8s.io/apimachinery/pkg/runtime", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema", "//vendor/k8s.io/apimachinery/pkg/types", "//vendor/k8s.io/apimachinery/pkg/util/runtime", "//vendor/k8s.io/apimachinery/pkg/util/sets", diff --git a/src/compute-plane-services/nvca/pkg/nvca/agent.go b/src/compute-plane-services/nvca/pkg/nvca/agent.go index 733b50e7d..0efb3af2b 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/agent.go +++ b/src/compute-plane-services/nvca/pkg/nvca/agent.go @@ -1331,6 +1331,14 @@ func (a *Agent) Start(ctx context.Context) error { log.Info("Starting event dispatchers") a.startEventProcessDispatchers(ctx, a.getTickerEvents(ctx)) + // Start the NvSnap Hook B controller (post-Ready checkpoint + // reconciler). No-op when NvSnapCheckpointRestore feature flag is + // off — see pkg/nvca/nvsnap_controller_start.go. Fire-and-forget; + // failure logs but doesn't block agent startup. + if err := a.startNvSnapController(ctx, k8sclients, log); err != nil { + log.WithError(err).Warn("nvsnap controller start failed (non-fatal)") + } + // Set readiness check only after all critical initialization (including ICMS registration) // has succeeded. This ensures the readiness probe stays not-ready on error returns, // preventing a rolling update from proceeding when registration fails (e.g. 409 Conflict diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go index 8b08a5e07..184b36394 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go @@ -67,6 +67,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/enforce" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/enforce/kaischeduler" nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap" nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" @@ -90,6 +91,16 @@ type K8sComputeBackend struct { discRestMapper meta.RESTMapper scheme *runtime.Scheme enabledAttrs featureflag.Attributes + + // nvsnapClient is the HTTP client to nvsnap-server. Used by Hook A + // (stampNvSnapAnnotations) to query the content-addressed lookup + // endpoint, so two pods with identical canonical workload + // identity but different function-version-IDs share one + // checkpoint instead of each cold-starting and re-capturing. + // May be nil if construction failed at NewK8sComputeBackend + // time — Hook A then falls through to the fvID-keyed path + // (fail-open). See nvnvsnap#59. + nvsnapClient *nvsnap.Client } // These are mocked in tests. @@ -127,6 +138,14 @@ func NewK8sComputeBackend(clients *kubeclients.KubeClients, bk8s *BackendK8sCach discRestMapper: restmapper.NewDiscoveryRESTMapper(grs), scheme: scheme, enabledAttrs: featureflag.GetEnabledAttributes(), + // nvsnap.NewClient with no options points at the in-cluster + // Service URL (nvsnap-server.nvsnap-system.svc.cluster.local:8080). + // Same defaulting as nvsnap_controller_start.go — the two + // callers will diverge when per-cluster ServerURL override + // lands. Hook A uses this in stampNvSnapAnnotations; nil-safe + // downstream so a NewClient panic (won't happen with no opts, + // but defense in depth) leaves Hook A on the fvID-only path. + nvsnapClient: nvsnap.NewClient(), } return newK8sBE, newK8sBE } @@ -1019,6 +1038,13 @@ func (c K8sComputeBackend) CreatePodArtifactInstances(ctx context.Context, pod * setTerminationGracePeriodIfNotSet(pod) k8sutil.ApplyCustomAnnotations(pod, c.bk8s.customAnnotations) + // Hook A: stamp nvsnap.io/restore-from + nvsnap.io/checkpoint-on-warm + // when the NvSnap integration is enabled and this function-version + // has a usable cached checkpoint. Fail-open: any error inside + // short-circuits to "no stamping" (pod cold-starts as before). + // See pkg/nvca/nvsnap_hook.go. + c.stampNvSnapAnnotations(ctx, pod, req, plog) + if _, err := c.clients.K8s.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}); err != nil { if !apierrors.IsAlreadyExists(err) { return nil, fmt.Errorf("failed to create instance for Request %v/%v, err: %v", req.Namespace, req.Name, err) diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_miniservice.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_miniservice.go index 1084e9fec..396c44dbb 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_miniservice.go +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_miniservice.go @@ -114,6 +114,17 @@ func (c K8sComputeBackend) applyMiniServiceCreationMessage(ctx context.Context, return c.bk8s.ApplyICMSRequestStatusChange(ctx, req) } + // Serialized-herd cold-start gate (pioneer election). Before + // creating the MiniService for a cold function-version, defer all + // but one "pioneer" replica until that pioneer's capture warms the + // function — the others then warm-restore via Hook A instead of all + // cold-starting. Returns a non-terminal error (requeue) for a + // deferred replica; nil (proceed) in every fail-open case. No-op + // when NvSnap is disabled. See shouldDeferColdStart. + if err := c.shouldDeferColdStart(ctx, req); err != nil { + return err + } + c.bk8s.eventRecorder.Eventf(req, corev1.EventTypeNormal, string(nvcatypes.EventCategoryInstanceCreation), "Creating %v requested instances", instCount) diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/BUILD.bazel new file mode 100644 index 000000000..68689d6a3 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "nvsnap", + srcs = ["client.go"], + importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap", + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":nvsnap", + visibility = ["//visibility:public"], +) + +go_test( + name = "nvsnap_test", + srcs = ["client_test.go"], + embed = [":nvsnap"], +) diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/client.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/client.go new file mode 100644 index 000000000..f40341776 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/client.go @@ -0,0 +1,382 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// Package nvsnap is the NVCA-side client for the NvSnap checkpoint/restore +// HTTP API exposed by nvsnap-server. This package is a thin transport +// layer only — no business logic, no policy decisions, no +// CreatePodArtifactInstances integration. Those land in subsequent PRs +// once the no-op baseline (this client + feature flag) is merged. +// +// Design reference: docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md +// +// Endpoints consumed: +// +// POST /api/v1/checkpoints CreateCheckpoint +// GET /api/v1/checkpoints/ GetCheckpoint +// DELETE /api/v1/checkpoints/ DeleteCheckpoint +// +// Checkpoint and Restore lifecycle on the nvsnap side is asynchronous +// — POST returns 202 with an id, and the caller polls GET until +// Phase reaches a terminal value (Completed | Failed). NVCA's +// post-Ready reconciler is what drives that polling; the client just +// exposes the raw operations. +package nvsnap + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// Default base URL for the in-cluster nvsnap-server Service. Callers +// override with WithBaseURL — operators may run nvsnap out-of-cluster +// or behind a custom ingress. +const DefaultBaseURL = "http://nvsnap-server.nvsnap-system.svc.cluster.local:8080" + +// Default per-request timeout. Checkpoint create returns 202 quickly; +// long-running operations are polled via GetCheckpoint. 30s is +// generous for the synchronous POST and avoids head-of-line blocking +// if nvsnap-server is briefly slow. +const DefaultTimeout = 30 * time.Second + +// Terminal phase values for Checkpoint.Phase. Non-terminal values +// (InProgress) mean the caller should keep polling. +const ( + PhaseInProgress = "InProgress" + PhaseCompleted = "Completed" + PhaseFailed = "Failed" +) + +// Client is the nvsnap-server HTTP client. Construct via NewClient. +// +// Safe for concurrent use by multiple goroutines. +type Client struct { + baseURL string + httpClient *http.Client + userAgent string +} + +// Option configures the Client. Pattern matches what's idiomatic +// across other NVCA HTTP clients. +type Option func(*Client) + +// WithBaseURL overrides DefaultBaseURL — used for out-of-cluster +// deployments or tests that point at httptest.Server. +func WithBaseURL(url string) Option { + return func(c *Client) { + c.baseURL = strings.TrimRight(url, "/") + } +} + +// WithHTTPClient overrides the underlying *http.Client. Useful for +// injecting timeouts, transport-level tracing, or test doubles. +// Default = &http.Client{Timeout: DefaultTimeout}. +func WithHTTPClient(hc *http.Client) Option { + return func(c *Client) { + c.httpClient = hc + } +} + +// WithUserAgent overrides the User-Agent header. Default +// "nvca-nvsnap-client/1". +func WithUserAgent(ua string) Option { + return func(c *Client) { + c.userAgent = ua + } +} + +// NewClient returns a Client. With no options, it talks to the +// in-cluster nvsnap-server Service with a 30s timeout. +func NewClient(opts ...Option) *Client { + c := &Client{ + baseURL: DefaultBaseURL, + httpClient: &http.Client{Timeout: DefaultTimeout}, + userAgent: "nvca-nvsnap-client/1", + } + for _, o := range opts { + o(c) + } + return c +} + +// CheckpointRequest is the body of POST /api/v1/checkpoints. Namespace +// and PodName are required; ContainerName defaults to single-container +// pods on the nvsnap side. LeaveRunning is true by default for NVCA's +// use case (don't kill the source pod after checkpoint). +type CheckpointRequest struct { + Namespace string `json:"namespace"` + PodName string `json:"podName"` + ContainerName string `json:"containerName,omitempty"` + LeaveRunning bool `json:"leaveRunning"` +} + +// Checkpoint is the response shape from POST /api/v1/checkpoints and +// GET /api/v1/checkpoints/. Fields are populated incrementally: +// POST returns ID + Phase=InProgress; GET adds Hash + Size + Duration +// when Phase reaches Completed (or Error when Phase=Failed). +type Checkpoint struct { + ID string `json:"id"` + Namespace string `json:"namespace,omitempty"` + Phase string `json:"phase"` + Message string `json:"message,omitempty"` + Hash string `json:"hash,omitempty"` + Path string `json:"path,omitempty"` + Size int64 `json:"size,omitempty"` + Duration float64 `json:"duration,omitempty"` + Error string `json:"error,omitempty"` +} + +// IsTerminal reports whether the checkpoint has reached a final phase. +// Callers poll GetCheckpoint while !IsTerminal. +func (c Checkpoint) IsTerminal() bool { + return c.Phase == PhaseCompleted || c.Phase == PhaseFailed +} + +// CreateCheckpoint kicks off a checkpoint for the named container in +// the named pod. Returns immediately with Phase=InProgress and the +// checkpoint ID; caller polls GetCheckpoint for the terminal status. +// +// Returns an error only for transport failures or non-2xx responses. +// A "succeeded but with errors" terminal state is communicated via +// the returned Checkpoint's Phase=Failed + Error. +func (c *Client) CreateCheckpoint(ctx context.Context, req CheckpointRequest) (*Checkpoint, error) { + if req.Namespace == "" || req.PodName == "" { + return nil, errors.New("nvsnap: CheckpointRequest.Namespace and PodName are required") + } + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + var out Checkpoint + if err := c.do(ctx, http.MethodPost, "/api/v1/checkpoints", bytes.NewReader(body), &out); err != nil { + return nil, err + } + return &out, nil +} + +// GetCheckpoint fetches the current state of a checkpoint by id. +// Returns *APIError with StatusCode=404 if the checkpoint doesn't +// exist (or is being deleted). +func (c *Client) GetCheckpoint(ctx context.Context, id string) (*Checkpoint, error) { + if id == "" { + return nil, errors.New("nvsnap: id required") + } + path := "/api/v1/checkpoints/" + url.PathEscape(id) + var out Checkpoint + if err := c.do(ctx, http.MethodGet, path, nil, &out); err != nil { + return nil, err + } + return &out, nil +} + +// LookupRequest asks nvsnap-server "do you have a checkpoint for this +// canonical workload identity?" The response carries any matching +// catalog rows, freshest-first. Used by Hook A to dedup checkpoints +// across function-version-IDs (nvnvsnap#59 / nvca cross-fvID restore): +// two functions with the same image + model + flags + driver should +// share one checkpoint instead of each cold-starting and re-capturing. +// +// ImageRef is required (the indexed lookup key — kubelet has it on +// every pod spec); the rest narrows the match. EngineFlags is +// canonicalized server-side (sort + strip --model*) before +// comparison. DriverMajor=0 means "match any driver". +type LookupRequest struct { + ImageRef string `json:"imageRef"` + ModelID string `json:"modelId,omitempty"` + EngineFlags []string `json:"engineFlags,omitempty"` + DriverMajor int `json:"driverMajor,omitempty"` + Limit int `json:"limit,omitempty"` +} + +// LookupMatch is one row of LookupResponse. Just enough for NVCA to +// decide "restore from this hash" — the rest of CatalogInfo (function +// name, GPU type, etc.) is accessible via GetCheckpoint(id). +type LookupMatch struct { + Hash string `json:"hash"` + CheckpointID string `json:"checkpointId"` + CapturedAt time.Time `json:"capturedAt"` + CapturedOnNode string `json:"capturedOnNode"` + ImageRef string `json:"imageRef"` + ImageDigest string `json:"imageDigest,omitempty"` + ModelID string `json:"modelId,omitempty"` + GPUType string `json:"gpuType,omitempty"` + DriverVersion string `json:"driverVersion,omitempty"` +} + +// LookupResponse is the body of POST /api/v1/checkpoints/lookup. +// Matches is always non-nil (empty array on no match), so callers +// can range without nil-checking. +type LookupResponse struct { + Matches []LookupMatch `json:"matches"` +} + +// LookupCheckpoints asks nvsnap-server for catalog rows matching the +// canonical workload identity. Returns the response struct (possibly +// with empty Matches) on 200; any non-2xx is an error. +// +// Used by Hook A. Empty result is NOT an error — it just means +// "this is a Cold start, fall through to the capture flow". +func (c *Client) LookupCheckpoints(ctx context.Context, req LookupRequest) (*LookupResponse, error) { + if req.ImageRef == "" { + return nil, errors.New("nvsnap: LookupRequest.ImageRef required") + } + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("nvsnap: marshal lookup: %w", err) + } + var out LookupResponse + if err := c.do(ctx, http.MethodPost, "/api/v1/checkpoints/lookup", bytes.NewReader(body), &out); err != nil { + return nil, err + } + return &out, nil +} + +// PVCPromoteState mirrors the catalog row for the L2 per-capture +// PVC pipeline. The reconciler (nvca#179) uses this to gate the +// CFS=Warm transition on "rox- is actually ready to serve" — +// without it, NVCA flips CFS=Warm the moment the CRIU checkpoint +// reaches terminal phase, even though the agent's async snap+clone +// promote is still in flight. Restored function pods then race the +// promote and either stall in ContainerCreating or get a partial +// dump. +// +// State values (mirroring nvsnap-server's pvcPromoteStateResponse): +// +// "" L2 disabled at agent startup, or agent died +// before the first state write. Treat as "no L2, +// proceed with whatever fallback the restore path +// uses" (peer cascade in production today). +// "pending" lease acquired, writer Job not yet started +// "writing" writer Job running (rwx- filling up) +// "snapshotting" VolumeSnapshot + rox- clone in progress +// "ready" rox- Bound and serving — safe to admit +// restore pods +// "failed" promote terminally failed; rox- won't +// appear. Restored pods must cold-start. +type PVCPromoteState struct { + Hash string `json:"hash"` + State string `json:"state"` + PVCName string `json:"pvc_name,omitempty"` +} + +// IsTerminal reports whether the state can no longer change. Used by +// the reconciler's poll loop to break out (and decide pass/fail). +func (s PVCPromoteState) IsTerminal() bool { + switch s.State { + case "ready", "failed", "": + return true + default: + return false + } +} + +// GetPVCPromoteState fetches the L2 promote-state for a checkpoint +// hash. Returns *APIError with StatusCode=404 if the catalog has no +// row for that hash (which the reconciler treats as "no L2" — same +// as the empty-state case). +// +// Symmetric to the nvsnap-server's +// +// GET /api/v1/checkpoints/by-hash/{hash}/pvc-state +// +// endpoint (internal/server/sources.go in the nvsnap repo). +func (c *Client) GetPVCPromoteState(ctx context.Context, hash string) (*PVCPromoteState, error) { + if hash == "" { + return nil, errors.New("nvsnap: hash required") + } + path := "/api/v1/checkpoints/by-hash/" + url.PathEscape(hash) + "/pvc-state" + var out PVCPromoteState + if err := c.do(ctx, http.MethodGet, path, nil, &out); err != nil { + return nil, err + } + return &out, nil +} + +// DeleteCheckpoint removes a checkpoint from the nvsnap-server catalog +// (and best-effort the blobs). Idempotent: 404 is treated as success +// so callers can call this from a cleanup pass without checking +// existence first. +func (c *Client) DeleteCheckpoint(ctx context.Context, id string) error { + if id == "" { + return errors.New("nvsnap: id required") + } + path := "/api/v1/checkpoints/" + url.PathEscape(id) + err := c.do(ctx, http.MethodDelete, path, nil, nil) + if err == nil { + return nil + } + var apiErr *APIError + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound { + return nil + } + return err +} + +// APIError is returned for non-2xx HTTP responses. Lets callers +// distinguish "transport failed" (a generic error) from "server said +// no" (*APIError with a status code). +type APIError struct { + StatusCode int + Body string +} + +func (e *APIError) Error() string { + if e.Body == "" { + return fmt.Sprintf("nvsnap: HTTP %d", e.StatusCode) + } + return fmt.Sprintf("nvsnap: HTTP %d: %s", e.StatusCode, e.Body) +} + +// do issues a request, decoding a 2xx body into `out` (if non-nil). +// Returns *APIError on non-2xx so callers can inspect StatusCode. +func (c *Client) do(ctx context.Context, method, path string, body io.Reader, out any) error { + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) + if err != nil { + return fmt.Errorf("build request: %w", err) + } + req.Header.Set("User-Agent", c.userAgent) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("http: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + // Cap body read so a misbehaving server can't OOM us. + const maxErrBody = 16 << 10 + b, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrBody)) + return &APIError{ + StatusCode: resp.StatusCode, + Body: strings.TrimSpace(string(b)), + } + } + if out == nil { + return nil + } + // 204 No Content has no body — return without decoding. + if resp.StatusCode == http.StatusNoContent { + return nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("decode response: %w", err) + } + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/client_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/client_test.go new file mode 100644 index 000000000..b5ca89a31 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/client_test.go @@ -0,0 +1,250 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package nvsnap + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestCreateCheckpoint(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("method = %q, want POST", r.Method) + } + if r.URL.Path != "/api/v1/checkpoints" { + t.Errorf("path = %q, want /api/v1/checkpoints", r.URL.Path) + } + if got := r.Header.Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q, want application/json", got) + } + var req CheckpointRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode body: %v", err) + } + if req.Namespace != "nvcf-backend" || req.PodName != "0-sr-abc" || req.ContainerName != "inference" { + t.Errorf("body = %+v, want {nvcf-backend, 0-sr-abc, inference}", req) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(Checkpoint{ + ID: "0-sr-abc-1234567890", + Namespace: "nvcf-backend", + Phase: PhaseInProgress, + Message: "Checkpoint started", + }) + })) + defer srv.Close() + + c := NewClient(WithBaseURL(srv.URL)) + got, err := c.CreateCheckpoint(context.Background(), CheckpointRequest{ + Namespace: "nvcf-backend", + PodName: "0-sr-abc", + ContainerName: "inference", + LeaveRunning: true, + }) + if err != nil { + t.Fatalf("CreateCheckpoint: %v", err) + } + if got.ID != "0-sr-abc-1234567890" { + t.Errorf("ID = %q, want 0-sr-abc-1234567890", got.ID) + } + if got.Phase != PhaseInProgress { + t.Errorf("Phase = %q, want %q", got.Phase, PhaseInProgress) + } + if got.IsTerminal() { + t.Errorf("IsTerminal() = true for InProgress, want false") + } +} + +func TestCreateCheckpointRequiresFields(t *testing.T) { + c := NewClient() + for _, tc := range []struct { + name string + req CheckpointRequest + }{ + {"missing namespace", CheckpointRequest{PodName: "p"}}, + {"missing pod", CheckpointRequest{Namespace: "ns"}}, + {"empty", CheckpointRequest{}}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := c.CreateCheckpoint(context.Background(), tc.req) + if err == nil { + t.Fatalf("expected error for %s", tc.name) + } + }) + } +} + +func TestGetCheckpointTerminal(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("method = %q, want GET", r.Method) + } + if r.URL.Path != "/api/v1/checkpoints/0-sr-abc-1234567890" { + t.Errorf("path = %q", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(Checkpoint{ + ID: "0-sr-abc-1234567890", + Phase: PhaseCompleted, + Hash: "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + Size: 81604378624, + Duration: 81.2, + Message: "Completed", + }) + })) + defer srv.Close() + + c := NewClient(WithBaseURL(srv.URL)) + got, err := c.GetCheckpoint(context.Background(), "0-sr-abc-1234567890") + if err != nil { + t.Fatalf("GetCheckpoint: %v", err) + } + if !got.IsTerminal() { + t.Errorf("IsTerminal() = false for Completed, want true") + } + if got.Hash == "" || got.Size == 0 { + t.Errorf("expected Hash + Size populated on terminal Completed, got %+v", got) + } +} + +func TestGetCheckpointNotFound(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "checkpoint not found", http.StatusNotFound) + })) + defer srv.Close() + + c := NewClient(WithBaseURL(srv.URL)) + _, err := c.GetCheckpoint(context.Background(), "missing") + if err == nil { + t.Fatal("expected error for 404, got nil") + } + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error type = %T, want *APIError", err) + } + if apiErr.StatusCode != http.StatusNotFound { + t.Errorf("StatusCode = %d, want 404", apiErr.StatusCode) + } +} + +func TestDeleteCheckpointIdempotent(t *testing.T) { + cases := []struct { + name string + statusCode int + wantErr bool + }{ + {"204 No Content", http.StatusNoContent, false}, + {"200 OK", http.StatusOK, false}, + {"404 treated as success (idempotent)", http.StatusNotFound, false}, + {"500 propagates", http.StatusInternalServerError, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Errorf("method = %q, want DELETE", r.Method) + } + w.WriteHeader(tc.statusCode) + })) + defer srv.Close() + + c := NewClient(WithBaseURL(srv.URL)) + err := c.DeleteCheckpoint(context.Background(), "any-id") + if tc.wantErr && err == nil { + t.Error("expected error, got nil") + } + if !tc.wantErr && err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func TestRequestContextCancellation(t *testing.T) { + // Server hangs forever. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + })) + defer srv.Close() + + c := NewClient(WithBaseURL(srv.URL)) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _, err := c.GetCheckpoint(ctx, "x") + if err == nil { + t.Fatal("expected context error, got nil") + } + if !strings.Contains(err.Error(), "context deadline exceeded") && + !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("error = %v, want context deadline exceeded", err) + } +} + +func TestUserAgentSent(t *testing.T) { + gotUA := "" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotUA = r.Header.Get("User-Agent") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"id":"x","phase":"InProgress"}`) + })) + defer srv.Close() + + c := NewClient(WithBaseURL(srv.URL), WithUserAgent("nvca-test/9")) + _, _ = c.GetCheckpoint(context.Background(), "x") + if gotUA != "nvca-test/9" { + t.Errorf("User-Agent = %q, want nvca-test/9", gotUA) + } +} + +func TestBaseURLTrailingSlashStripped(t *testing.T) { + c := NewClient(WithBaseURL("http://example.com/")) + if c.baseURL != "http://example.com" { + t.Errorf("baseURL = %q, want stripped trailing slash", c.baseURL) + } +} + +func TestAPIErrorContainsBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "namespace required", http.StatusBadRequest) + })) + defer srv.Close() + + c := NewClient(WithBaseURL(srv.URL)) + _, err := c.CreateCheckpoint(context.Background(), CheckpointRequest{Namespace: "ns", PodName: "p"}) + if err == nil { + t.Fatal("expected error") + } + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error type = %T, want *APIError", err) + } + if apiErr.StatusCode != http.StatusBadRequest { + t.Errorf("StatusCode = %d", apiErr.StatusCode) + } + if !strings.Contains(apiErr.Body, "namespace required") { + t.Errorf("Body = %q, want it to contain server message", apiErr.Body) + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/BUILD.bazel new file mode 100644 index 000000000..24457490b --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "controller", + srcs = [ + "controller.go", + "metrics.go", + ], + importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller", + visibility = ["//visibility:public"], + deps = [ + "//pkg/nvca/nvsnap/reconciler", + "//vendor/github.com/prometheus/client_golang/prometheus", + "//vendor/github.com/prometheus/client_golang/prometheus/promauto", + "//vendor/github.com/sirupsen/logrus", + "//vendor/k8s.io/api/core/v1:core", + "//vendor/k8s.io/client-go/informers", + "//vendor/k8s.io/client-go/kubernetes", + "//vendor/k8s.io/client-go/listers/core/v1:core", + "//vendor/k8s.io/client-go/tools/cache", + "//vendor/k8s.io/client-go/util/workqueue", + ], +) + +alias( + name = "go_default_library", + actual = ":controller", + visibility = ["//visibility:public"], +) + +go_test( + name = "controller_test", + srcs = [ + "controller_test.go", + "metrics_test.go", + ], + embed = [":controller"], + deps = [ + "//pkg/nvca/nvsnap/reconciler", + "//vendor/github.com/prometheus/client_model/go", + "//vendor/k8s.io/api/core/v1:core", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", + ], +) diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/controller.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/controller.go new file mode 100644 index 000000000..57f1316b1 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/controller.go @@ -0,0 +1,300 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// Package controller wires the post-Ready checkpoint reconciler (in +// pkg/nvca/nvsnap/reconciler) to a Pod informer + workqueue so the +// agent can drive it from a goroutine at startup. +// +// Responsibilities of this package: +// - Watch pods cluster-wide (or namespace-scoped). +// - Filter for the Hook A annotations (nvsnap.io/checkpoint-on-warm +// "true" AND nvsnap.io/function-version-id non-empty). +// - Enqueue a pod when it becomes PodReady. +// - Pop a key, look up the pod, call Reconciler.Reconcile. +// - On error, requeue with the workqueue's exponential backoff. +// +// What this package does NOT do (handled elsewhere or in follow-ups): +// - Read the global feature flag — caller decides whether to Start. +// - Bootstrap NvSnapFunctionState — done lazily inside Reconcile. +// - Wire into NVCA agent startup (PR-7). +package controller + +import ( + "context" + "fmt" + "time" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + nvsnapreconciler "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler" +) + +// Controller wraps the reconciler with a Pod informer + workqueue. +// One Controller per NVCA agent process. Stopped by canceling the +// context passed to Run. +type Controller struct { + // KubeClient is the shared K8s client. Run() constructs a + // SharedInformerFactory scoped to Namespace from this client + // each time it's invoked. (A future revision could accept an + // externally-managed factory so the agent's existing informer + // cache is reused; today Run owns its own.) + KubeClient kubernetes.Interface + + // Reconciler is the state machine that handles one pod. The + // controller calls Reconciler.Reconcile(ctx, pod) per workqueue + // dequeue. + Reconciler *nvsnapreconciler.Reconciler + + // Namespace scopes the Pod informer. Empty = cluster-wide. + // In production NVCA tenant pods live in nvcf-backend; empty + // works too if the agent has cluster-scoped pod-read RBAC. + Namespace string + + // Workers is the number of goroutines popping from the + // workqueue. Each runs one reconcile at a time. Default 2 — + // reconciles are long-lived (warmup + checkpoint can be 10+ min), + // so a small parallelism prevents the workqueue from starving + // under bursty deploys. + Workers int + + // SweepInterval is how often the pod-independent CFS recovery sweep + // runs (nvca#104 durable-warm). Each tick reconciles every + // NvSnapFunctionState against nvsnap-server and flips Warm for any whose + // capture has landed but whose live reconcile died before writeStatus. + // Default 60s. Set <0 to disable the sweep entirely. + SweepInterval time.Duration + + // Log is the structured logger. + Log logrus.FieldLogger + + // queue is built in Run. + queue workqueue.RateLimitingInterface +} + +// NewController constructs a Controller with sane defaults filled in. +// Caller must set KubeClient, Reconciler. Workers/Log/Namespace default +// if zero. +func NewController(kc kubernetes.Interface, r *nvsnapreconciler.Reconciler) *Controller { + return &Controller{ + KubeClient: kc, + Reconciler: r, + Workers: 2, + } +} + +// Run starts the informer + workers and blocks until ctx is canceled. +// Returns ctx.Err() on shutdown (no other error path — informer + queue +// failures are logged and retried internally). +func (c *Controller) Run(ctx context.Context) error { + if c.KubeClient == nil { + return fmt.Errorf("controller: KubeClient is nil") + } + if c.Reconciler == nil { + return fmt.Errorf("controller: Reconciler is nil") + } + if c.Log == nil { + c.Log = logrus.NewEntry(logrus.New()) + } + if c.Workers <= 0 { + c.Workers = 2 + } + c.queue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + defer c.queue.ShutDown() + + factory := informers.NewSharedInformerFactoryWithOptions(c.KubeClient, 0, + informers.WithNamespace(c.Namespace)) + podInf := factory.Core().V1().Pods() + informer := podInf.Informer() + if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + // Pods already Ready when the informer first sees them + // (controller restart, cache resync) still count — pass + // nil oldObj so observePodFirstReady treats it as a + // transition. Mis-counting after a restart would + // systematically under-report the savings. + if p, ok := obj.(*corev1.Pod); ok { + observePodFirstReady(nil, p) + } + c.enqueueIfEligible(obj) + }, + UpdateFunc: func(oldObj, newObj any) { + oldPod, _ := oldObj.(*corev1.Pod) + newPod, _ := newObj.(*corev1.Pod) + observePodFirstReady(oldPod, newPod) + c.enqueueIfEligible(newObj) + }, + // DeleteFunc intentionally omitted — a deleted pod has nothing + // to reconcile; the workqueue's existing entry (if any) will + // simply Get NotFound and log+drop. + }); err != nil { + return fmt.Errorf("install pod event handler: %w", err) + } + + factory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) { + return fmt.Errorf("controller: pod informer cache sync failed") + } + + c.Log.WithField("workers", c.Workers).Info("nvsnap controller starting") + for range c.Workers { + go c.runWorker(ctx, podInf.Lister()) + } + + // Pod-independent CFS recovery sweep (nvca#104 durable-warm). Runs + // on its own ticker, decoupled from the pod workqueue, so a capture + // whose live reconcile died before writeStatus still reaches Warm. + if c.SweepInterval == 0 { + c.SweepInterval = 60 * time.Second + } + if c.SweepInterval > 0 { + go c.runSweep(ctx) + } + + <-ctx.Done() + c.Log.Info("nvsnap controller stopping") + return ctx.Err() +} + +// enqueueIfEligible adds the pod's key to the workqueue if it carries +// the Hook A annotations AND is currently PodReady. The dual filter +// (annotation + Ready) avoids queueing pods we can't act on, keeping +// the queue's working set bounded by the number of in-flight warmups. +func (c *Controller) enqueueIfEligible(obj any) { + pod, ok := obj.(*corev1.Pod) + if !ok { + return + } + if !PodEligibleForCheckpointOnWarm(pod) { + return + } + if !IsPodReady(pod) { + return + } + key, err := cache.MetaNamespaceKeyFunc(pod) + if err != nil { + c.Log.WithError(err).Warn("nvsnap controller: build key failed; dropping") + return + } + c.queue.Add(key) +} + +// PodEligibleForCheckpointOnWarm is the static filter — checks that +// Hook A stamped both annotations the reconciler needs. Exported so +// the agent startup code can use the same predicate to install +// per-namespace informers conditionally. +func PodEligibleForCheckpointOnWarm(pod *corev1.Pod) bool { + if pod == nil || pod.Annotations == nil { + return false + } + if pod.Annotations[nvsnapreconciler.CheckpointOnWarmAnnotation] != "true" { + return false + } + if pod.Annotations[nvsnapreconciler.FunctionVersionIDAnnotation] == "" { + return false + } + return true +} + +// IsPodReady returns true iff the pod's PodReady condition is True. +// Duplicated from k8sutil so this package has no internal-only NVCA +// dependency (lets the controller be vendored standalone if needed). +func IsPodReady(pod *corev1.Pod) bool { + if pod == nil { + return false + } + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +func (c *Controller) runWorker(ctx context.Context, lister corev1listers.PodLister) { + for { + if shutdown := c.processNext(ctx, lister); shutdown { + return + } + } +} + +// runSweep drives the pod-independent CFS recovery sweep on a ticker +// until ctx is canceled (nvca#104 durable-warm). One pass runs +// immediately so a controller restart heals stuck-not-Warm CFS without +// waiting a full interval. SweepOnce is best-effort and never panics or +// returns an error, so this loop is just "tick → sweep". +func (c *Controller) runSweep(ctx context.Context) { + c.Log.WithField("interval", c.SweepInterval).Info("nvsnap CFS recovery sweep starting") + c.Reconciler.SweepOnce(ctx) + ticker := time.NewTicker(c.SweepInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.Reconciler.SweepOnce(ctx) + } + } +} + +// processNext pops one key, looks up the pod, and runs Reconcile. On +// error, re-queues with backoff. Returns true iff the queue was shut +// down (caller exits the loop). +func (c *Controller) processNext(ctx context.Context, lister corev1listers.PodLister) bool { + key, quit := c.queue.Get() + if quit { + return true + } + defer c.queue.Done(key) + + ns, name, err := cache.SplitMetaNamespaceKey(key.(string)) + if err != nil { + c.Log.WithError(err).WithField("key", key).Warn("nvsnap controller: bad key") + c.queue.Forget(key) + return false + } + + pod, err := lister.Pods(ns).Get(name) + if err != nil { + // Pod gone — nothing to reconcile. + c.Log.WithError(err).Debugf("nvsnap controller: pod %s/%s gone; dropping", ns, name) + c.queue.Forget(key) + return false + } + + // Re-check eligibility — informer can race with annotation removal + // (we drop the annotation in the reconciler on success). + if !PodEligibleForCheckpointOnWarm(pod) { + c.queue.Forget(key) + return false + } + + rctx, cancel := context.WithCancel(ctx) + if err := c.Reconciler.Reconcile(rctx, pod); err != nil { + cancel() + c.Log.WithError(err).WithFields(logrus.Fields{ + "pod": ns + "/" + name, + "retry_count": c.queue.NumRequeues(key), + }).Warn("nvsnap controller: reconcile failed; will retry") + c.queue.AddRateLimited(key) + return false + } + cancel() + c.queue.Forget(key) + return false +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/controller_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/controller_test.go new file mode 100644 index 000000000..af8c9e4f5 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/controller_test.go @@ -0,0 +1,118 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package controller + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nvsnapreconciler "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler" +) + +func TestPodEligibleForCheckpointOnWarm(t *testing.T) { + cases := []struct { + name string + pod *corev1.Pod + want bool + }{ + {"nil", nil, false}, + {"no annotations", &corev1.Pod{}, false}, + {"annotation false", podWithAnnotations(map[string]string{ + nvsnapreconciler.CheckpointOnWarmAnnotation: "false", + nvsnapreconciler.FunctionVersionIDAnnotation: "fv-1", + }), false}, + {"only checkpoint-on-warm", podWithAnnotations(map[string]string{ + nvsnapreconciler.CheckpointOnWarmAnnotation: "true", + }), false}, + {"only function-version", podWithAnnotations(map[string]string{ + nvsnapreconciler.FunctionVersionIDAnnotation: "fv-1", + }), false}, + {"both annotations set", podWithAnnotations(map[string]string{ + nvsnapreconciler.CheckpointOnWarmAnnotation: "true", + nvsnapreconciler.FunctionVersionIDAnnotation: "fv-1", + }), true}, + {"both set + extra unrelated annotations", podWithAnnotations(map[string]string{ + nvsnapreconciler.CheckpointOnWarmAnnotation: "true", + nvsnapreconciler.FunctionVersionIDAnnotation: "fv-1", + "some.other/annotation": "x", + }), true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := PodEligibleForCheckpointOnWarm(tc.pod); got != tc.want { + t.Errorf("PodEligibleForCheckpointOnWarm = %v, want %v", got, tc.want) + } + }) + } +} + +func TestIsPodReady(t *testing.T) { + cases := []struct { + name string + pod *corev1.Pod + want bool + }{ + {"nil", nil, false}, + {"no conditions", &corev1.Pod{}, false}, + {"ready true", &corev1.Pod{Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }}, true}, + {"ready false", &corev1.Pod{Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionFalse}, + }, + }}, false}, + {"ready unknown", &corev1.Pod{Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionUnknown}, + }, + }}, false}, + {"initialized but not ready", &corev1.Pod{Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodInitialized, Status: corev1.ConditionTrue}, + }, + }}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := IsPodReady(tc.pod); got != tc.want { + t.Errorf("IsPodReady = %v, want %v", got, tc.want) + } + }) + } +} + +func TestNewControllerDefaults(t *testing.T) { + c := NewController(nil, nil) + if c.Workers != 2 { + t.Errorf("Workers default = %d, want 2", c.Workers) + } +} + +func podWithAnnotations(ann map[string]string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "p", + Namespace: "n", + Annotations: ann, + }, + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/metrics.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/metrics.go new file mode 100644 index 000000000..79ef19c57 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/metrics.go @@ -0,0 +1,166 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package controller + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + corev1 "k8s.io/api/core/v1" +) + +// Annotation keys NVCA's Hook A stamps. Duplicated as constants here +// rather than imported from pkg/nvca to avoid a controller → nvca → +// controller cycle through nvsnap_controller_start.go. The values are +// stable contract — keep in sync with pkg/nvca/nvsnap_hook.go. +const ( + restoreFromAnnotation = "nvsnap.io/restore-from" + checkpointOnWarmAnnotation = "nvsnap.io/checkpoint-on-warm" + functionVersionIDAnnotation = "nvsnap.io/function-version-id" +) + +var _ = functionVersionIDAnnotation // reserved for future per-fvID metric labeling + +// Prometheus metrics for the NvSnap × NVCA observability story. +// +// nvca_nvsnap_pod_first_ready_seconds is the headline customer-facing +// number — the histogram comparing how long pods take to become +// Ready, split by whether NVCA's Hook A stamped a restore-from +// annotation on them. A side-by-side Grafana panel on this histogram +// is the "is NvSnap actually helping?" answer. +// +// Cardinality budget: only the `restored` label is exposed. We don't +// label by function_version_id even though the data exists — operators +// running thousands of FVs would blow the metric out, and per-FV +// drill-down is better served by ad-hoc PromQL with pod-label joins +// (kube-state-metrics gives us those). +var ( + podFirstReadySeconds = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "nvca_nvsnap_pod_first_ready_seconds", + Help: "Wall-clock seconds from Pod CreationTimestamp to first PodReady=True, " + + "split by whether NVCA's Hook A stamped nvsnap.io/restore-from on the pod. " + + "Cold-start vs NvSnap-restored side by side.", + // Buckets tuned for the typical span: a NvSnap restore in + // ~30-60s, a cold start in 1-5 minutes for a CPU function + // and 5-20 minutes for a GPU function pulling a multi-GB + // model. Wide tail catches misbehaving cold-starts. + Buckets: []float64{5, 10, 20, 30, 60, 120, 180, 300, 600, 900, 1800}, + }, + []string{"restored"}, + ) + + podFirstReadyTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "nvca_nvsnap_pods_first_ready_total", + Help: "Count of pod first-ready observations, partitioned by mode. " + + "The restored counter is the cold-starts you avoided.", + }, + []string{"mode"}, + ) +) + +// observePodFirstReady records a histogram observation + counter +// increment when newPod has just transitioned to PodReady=True +// (i.e., oldPod was not Ready, newPod is). No-ops in every other +// case, so safe to call from every UpdateFunc invocation. +// +// Pods without any NvSnap annotation (nvsnap.io/restore-from OR +// nvsnap.io/checkpoint-on-warm) are ignored — non-NVCA pods don't +// belong in this telemetry. This also avoids inflating the metric +// with kube-system / DaemonSet pods that share the same informer. +// +// Latency = newPod.PodReady.LastTransitionTime - newPod.CreationTimestamp. +// Using LastTransitionTime (rather than time.Now()) keeps the +// reported value stable across controller restarts: if the +// controller misses the Ready transition and catches up later, +// it still reports the true latency, not the catch-up delay. +func observePodFirstReady(oldPod, newPod *corev1.Pod) { + if newPod == nil || !podIsNvSnapTouched(newPod) { + return + } + if !isPodReadyTransition(oldPod, newPod) { + return + } + created := newPod.CreationTimestamp.Time + readyAt, ok := podReadyTransitionTime(newPod) + if !ok || created.IsZero() { + return + } + latency := readyAt.Sub(created).Seconds() + if latency < 0 { + // Shouldn't happen — defensive: a negative would skew the + // histogram and be visible as a long tail. Drop. + return + } + + mode := "cold-start" + restoredLabel := "false" + if rf := newPod.Annotations[restoreFromAnnotation]; rf != "" { + mode = "restored" + restoredLabel = "true" + } + podFirstReadySeconds.WithLabelValues(restoredLabel).Observe(latency) + podFirstReadyTotal.WithLabelValues(mode).Inc() +} + +// podIsNvSnapTouched reports whether NVCA's Hook A stamped any +// annotation on the pod (restore-from for the restored path, +// checkpoint-on-warm for the cold-start path). Either marks the +// pod as "in scope" for our histogram. +func podIsNvSnapTouched(p *corev1.Pod) bool { + if p == nil || p.Annotations == nil { + return false + } + if v := p.Annotations[restoreFromAnnotation]; v != "" { + return true + } + if v := p.Annotations[checkpointOnWarmAnnotation]; v != "" { + return true + } + return false +} + +// isPodReadyTransition returns true when newPod is Ready=True AND +// oldPod was not (nil oldPod counts as a transition — that's the +// "controller just started, caught a pod already Ready" path). +func isPodReadyTransition(oldPod, newPod *corev1.Pod) bool { + if !podReadyTrue(newPod) { + return false + } + if oldPod == nil { + return true + } + return !podReadyTrue(oldPod) +} + +func podReadyTrue(p *corev1.Pod) bool { + if p == nil { + return false + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +func podReadyTransitionTime(p *corev1.Pod) (time.Time, bool) { + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return c.LastTransitionTime.Time, true + } + } + return time.Time{}, false +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/metrics_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/metrics_test.go new file mode 100644 index 000000000..ff929cb88 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller/metrics_test.go @@ -0,0 +1,207 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package controller + +import ( + "testing" + "time" + + dto "github.com/prometheus/client_model/go" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func mkPodWithReady(annotations map[string]string, created time.Time, ready bool, readyAt time.Time) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + CreationTimestamp: metav1.NewTime(created), + }, + } + if ready { + p.Status.Conditions = []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.NewTime(readyAt), + }} + } else { + // Pod has the condition but Status=False (not yet ready). + // Mirrors what kubelet sets while warming. + p.Status.Conditions = []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: corev1.ConditionFalse, + }} + } + return p +} + +// counterByMode reads the current counter value for a given mode +// label off the global registry. Tests assert by delta to remain +// independent of other tests running first. +func counterByMode(t *testing.T, mode string) float64 { + t.Helper() + m := &dto.Metric{} + if err := podFirstReadyTotal.WithLabelValues(mode).Write(m); err != nil { + t.Fatalf("read counter: %v", err) + } + return m.GetCounter().GetValue() +} + +func histSampleCountByRestored(t *testing.T, restored string) uint64 { + t.Helper() + m := &dto.Metric{} + if err := podFirstReadySeconds.WithLabelValues(restored).(interface { + Write(*dto.Metric) error + }).Write(m); err != nil { + t.Fatalf("read histogram: %v", err) + } + return m.GetHistogram().GetSampleCount() +} + +func TestObserve_RestoredPodTransition(t *testing.T) { + beforeCount := counterByMode(t, "restored") + beforeHist := histSampleCountByRestored(t, "true") + + created := time.Date(2026, 5, 31, 22, 56, 0, 0, time.UTC) + old := mkPodWithReady(map[string]string{ + restoreFromAnnotation: "85ec4d75ee57c1be...", + functionVersionIDAnnotation: "fv-1", + }, created, false, time.Time{}) + new := mkPodWithReady(map[string]string{ + restoreFromAnnotation: "85ec4d75ee57c1be...", + functionVersionIDAnnotation: "fv-1", + }, created, true, created.Add(43*time.Second)) + + observePodFirstReady(old, new) + + if got := counterByMode(t, "restored"); got != beforeCount+1 { + t.Errorf("restored counter delta = %v, want +1", got-beforeCount) + } + if got := histSampleCountByRestored(t, "true"); got != beforeHist+1 { + t.Errorf("histogram(restored=true) sample-count delta = %v, want +1", got-beforeHist) + } +} + +func TestObserve_ColdStartPodTransition(t *testing.T) { + beforeCount := counterByMode(t, "cold-start") + beforeHist := histSampleCountByRestored(t, "false") + + created := time.Date(2026, 5, 31, 22, 50, 0, 0, time.UTC) + old := mkPodWithReady(map[string]string{ + checkpointOnWarmAnnotation: "true", + functionVersionIDAnnotation: "fv-1", + }, created, false, time.Time{}) + new := mkPodWithReady(map[string]string{ + checkpointOnWarmAnnotation: "true", + functionVersionIDAnnotation: "fv-1", + }, created, true, created.Add(188*time.Second)) + + observePodFirstReady(old, new) + + if got := counterByMode(t, "cold-start"); got != beforeCount+1 { + t.Errorf("cold-start counter delta = %v, want +1", got-beforeCount) + } + if got := histSampleCountByRestored(t, "false"); got != beforeHist+1 { + t.Errorf("histogram(restored=false) sample-count delta = %v, want +1", got-beforeHist) + } +} + +func TestObserve_PodAlreadyReadyOnOldDoesNotDoubleCount(t *testing.T) { + // Both old and new are Ready=True → no transition, no observation. + // Without this guard, every status field update on a Ready pod + // would re-observe and inflate the histogram. + before := counterByMode(t, "restored") + + created := time.Date(2026, 5, 31, 22, 0, 0, 0, time.UTC) + old := mkPodWithReady(map[string]string{restoreFromAnnotation: "h"}, created, true, created.Add(43*time.Second)) + new := mkPodWithReady(map[string]string{restoreFromAnnotation: "h"}, created, true, created.Add(43*time.Second)) + + observePodFirstReady(old, new) + + if got := counterByMode(t, "restored"); got != before { + t.Errorf("counter advanced on no-op update; delta=%v", got-before) + } +} + +func TestObserve_NilOldPodTreatedAsTransition(t *testing.T) { + // Pod was already Ready when the informer first saw it (controller + // restart / cache resync). We still observe — under-counting would + // systematically hide the savings after every controller bounce. + before := counterByMode(t, "restored") + + created := time.Date(2026, 5, 31, 22, 0, 0, 0, time.UTC) + new := mkPodWithReady(map[string]string{restoreFromAnnotation: "h"}, created, true, created.Add(43*time.Second)) + + observePodFirstReady(nil, new) + + if got := counterByMode(t, "restored"); got != before+1 { + t.Errorf("nil-old counter delta = %v, want +1", got-before) + } +} + +func TestObserve_PodWithoutNvSnapAnnotationsIgnored(t *testing.T) { + // kube-system / unrelated pods that happen to share the informer + // should NOT be counted. The annotation predicate is the only + // filter that keeps us out of those metrics. + before := counterByMode(t, "cold-start") + + created := time.Date(2026, 5, 31, 22, 0, 0, 0, time.UTC) + old := mkPodWithReady(nil, created, false, time.Time{}) + new := mkPodWithReady(nil, created, true, created.Add(60*time.Second)) + + observePodFirstReady(old, new) + + if got := counterByMode(t, "cold-start"); got != before { + t.Errorf("non-NVCA pod was counted; delta=%v", got-before) + } +} + +func TestObserve_NewPodNotReadyIsNoOp(t *testing.T) { + before := counterByMode(t, "restored") + + created := time.Date(2026, 5, 31, 22, 0, 0, 0, time.UTC) + old := mkPodWithReady(map[string]string{restoreFromAnnotation: "h"}, created, false, time.Time{}) + new := mkPodWithReady(map[string]string{restoreFromAnnotation: "h"}, created, false, time.Time{}) + + observePodFirstReady(old, new) + + if got := counterByMode(t, "restored"); got != before { + t.Errorf("not-ready pod was counted; delta=%v", got-before) + } +} + +func TestObserve_NegativeLatencyDropped(t *testing.T) { + // Defensive: a malformed pod with readyAt < createdAt would skew + // the histogram badly. We drop, not panic. + before := histSampleCountByRestored(t, "true") + + created := time.Date(2026, 5, 31, 22, 0, 0, 0, time.UTC) + old := mkPodWithReady(map[string]string{restoreFromAnnotation: "h"}, created, false, time.Time{}) + new := mkPodWithReady(map[string]string{restoreFromAnnotation: "h"}, created, true, created.Add(-5*time.Second)) + + observePodFirstReady(old, new) + + if got := histSampleCountByRestored(t, "true"); got != before { + t.Errorf("negative latency was observed; should drop. delta=%v", got-before) + } +} + +func TestObserve_NilNewPodNoPanic(t *testing.T) { + // Defensive: tests UpdateFunc cast-failure pattern (cast to *Pod + // returns nil if the object was a tombstone or wrong type). + observePodFirstReady(nil, nil) // must not panic +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/BUILD.bazel new file mode 100644 index 000000000..aaeb536f3 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/BUILD.bazel @@ -0,0 +1,67 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "reconciler", + srcs = [ + "backoff.go", + "health.go", + "metrics.go", + "reconciler.go", + "recover.go", + "state.go", + "sweep.go", + ], + importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler", + visibility = ["//visibility:public"], + deps = [ + "//internal/otel", + "//pkg/apis/nvsnap/v1alpha1", + "//pkg/nvca/nvsnap", + "//vendor/github.com/prometheus/client_golang/prometheus", + "//vendor/github.com/prometheus/client_golang/prometheus/promauto", + "//vendor/github.com/sirupsen/logrus", + "//vendor/go.opentelemetry.io/otel/attribute", + "//vendor/go.opentelemetry.io/otel/trace", + "//vendor/k8s.io/api/core/v1:core", + "//vendor/k8s.io/apimachinery/pkg/api/errors", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema", + "//vendor/k8s.io/apimachinery/pkg/types", + "//vendor/k8s.io/client-go/dynamic", + "//vendor/k8s.io/client-go/kubernetes", + ], +) + +alias( + name = "go_default_library", + actual = ":reconciler", + visibility = ["//visibility:public"], +) + +go_test( + name = "reconciler_test", + srcs = [ + "backoff_test.go", + "captureonce_test.go", + "coldstart_pioneer_test.go", + "reconciler_test.go", + "recover_test.go", + "sweep_test.go", + ], + embed = [":reconciler"], + deps = [ + "//pkg/apis/nvsnap/v1alpha1", + "//pkg/nvca/nvsnap", + "//vendor/github.com/prometheus/client_golang/prometheus/testutil", + "//vendor/github.com/sirupsen/logrus", + "//vendor/k8s.io/api/core/v1:core", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured", + "//vendor/k8s.io/apimachinery/pkg/runtime", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema", + "//vendor/k8s.io/client-go/dynamic", + "//vendor/k8s.io/client-go/dynamic/fake", + "//vendor/k8s.io/client-go/kubernetes/fake", + ], +) diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/backoff.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/backoff.go new file mode 100644 index 000000000..4ac6595a8 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/backoff.go @@ -0,0 +1,101 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// Per-function checkpoint-attempt backoff (nvca#167). +// +// Problem: prior to this code, every pod-ready event for a function +// with no warm cache triggered a fresh checkpoint attempt — even +// when the previous attempt had just failed seconds before. For +// untested or structurally-incompatible workloads (any new container +// a customer ships that NvSnap doesn't yet support), this produced an +// infinite ~10-minute capture loop: pod ready → POST → capture runs +// for ~10 min → fail → pod cycles → ready again → repeat. +// +// Customer-trust issue, not just an internal nuisance: a new BYOC +// workload that happens to incompatibly mix with NvSnap would be +// hammered with capture attempts forever, burning GPU minutes and +// blocking the operator's ability to even diagnose what was wrong. +// +// Fix: per-function exponential backoff keyed by CFS.AttemptCount. +// Reconcile consults the backoff window before issuing +// CreateCheckpoint; if we're still within the window after the +// previous failure, the attempt is suppressed (logged at Warn, no +// requeue, function continues cold) until the window elapses. +// +// AttemptCount resets to 0 on every successful checkpoint (already +// wired in reconciler.go's success path). So the backoff only +// applies to true failure streaks; a single bad attempt on an +// otherwise-healthy function doesn't impose any future penalty. + +package reconciler + +import ( + "time" +) + +// checkpointBackoffSchedule is the suppression window after the Nth +// consecutive failed attempt. After N=4, we cap at 24h — that gives +// the operator a full ops-cycle to notice and intervene without the +// function being abandoned forever (the 24h tick still lets a fixed +// agent or rebuilt cluster recover automatically). +// +// Exposed as a var (not const) so tests can shrink the windows +// without monkey-patching time.Now. +var checkpointBackoffSchedule = []time.Duration{ + 5 * time.Minute, // after 1st failure + 30 * time.Minute, // after 2nd + 2 * time.Hour, // after 3rd + 24 * time.Hour, // after 4th and beyond +} + +// backoffWindow returns the suppression duration appropriate for the +// given prior attempt count. attemptCount==0 → no backoff (first +// attempt has nothing to wait on). attemptCount>=len(schedule) → cap. +func backoffWindow(attemptCount int64) time.Duration { + if attemptCount <= 0 { + return 0 + } + idx := int(attemptCount) - 1 + if idx >= len(checkpointBackoffSchedule) { + idx = len(checkpointBackoffSchedule) - 1 + } + return checkpointBackoffSchedule[idx] +} + +// shouldSuppressAttempt reports whether a new checkpoint attempt +// should be skipped because the previous failure is still inside +// the backoff window. Returns the wall-clock time the suppression +// expires; callers log this so the operator can see when the next +// attempt is eligible. +// +// AttemptCount==0 OR LastAttemptAt==nil → never suppress (no prior +// failure on record). +// +// Suppression decision is "(now - LastAttemptAt) < window". We do +// NOT also gate on CapturedHere/LocalCacheState because NvSnap treats +// a Warm cache as a success path already: AttemptCount is reset on +// success, so a Warm function naturally has count=0 and skips this +// path. A function that's been Warm and then needs a fresh capture +// (hash invalidation, NVCA restart) starts fresh with count=0. +func shouldSuppressAttempt(prev cfsStatus, now time.Time) (suppress bool, until time.Time) { + if prev.AttemptCount <= 0 || prev.LastAttemptAt == nil { + return false, time.Time{} + } + window := backoffWindow(prev.AttemptCount) + if window <= 0 { + return false, time.Time{} + } + expires := prev.LastAttemptAt.Time.Add(window) + if now.Before(expires) { + return true, expires + } + return false, time.Time{} +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/backoff_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/backoff_test.go new file mode 100644 index 000000000..78fa14d5a --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/backoff_test.go @@ -0,0 +1,146 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package reconciler + +import ( + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// TestBackoffWindow_ScheduleMatchesIntent guards against silent +// schedule tweaks. If somebody changes the timings, the test fails +// loudly and forces a deliberate discussion in code review. +func TestBackoffWindow_ScheduleMatchesIntent(t *testing.T) { + cases := []struct { + attempts int64 + want time.Duration + }{ + {0, 0}, // no prior failure → no backoff + {-1, 0}, // defensive: negative should be no backoff + {1, 5 * time.Minute}, // 1st failure + {2, 30 * time.Minute}, // 2nd + {3, 2 * time.Hour}, // 3rd + {4, 24 * time.Hour}, // 4th — cap engages + {17, 24 * time.Hour}, // far above cap → still cap + {1_000_000, 24 * time.Hour}, // overflow stress → still cap + } + for _, tc := range cases { + got := backoffWindow(tc.attempts) + if got != tc.want { + t.Errorf("backoffWindow(%d) = %v, want %v", tc.attempts, got, tc.want) + } + } +} + +// TestShouldSuppressAttempt_NoPriorFailure verifies the gate is a +// no-op for a fresh function (the common case — most reconciles are +// for warm captures or first attempts). +func TestShouldSuppressAttempt_NoPriorFailure(t *testing.T) { + now := time.Now() + cases := []cfsStatus{ + {}, // brand new CFS + {AttemptCount: 0, LastAttemptAt: ptrTime(now)}, // count=0, attempt timestamp present (shouldn't happen but be tolerant) + {AttemptCount: 5, LastAttemptAt: nil}, // count>0 but timestamp lost (controller restart on old CRD) + } + for i, prev := range cases { + suppress, until := shouldSuppressAttempt(prev, now) + if suppress { + t.Errorf("case %d: shouldSuppressAttempt returned true for %+v; want false (no usable prior failure record)", i, prev) + } + if !until.IsZero() { + t.Errorf("case %d: until=%v, want zero", i, until) + } + } +} + +// TestShouldSuppressAttempt_WithinWindowSuppresses confirms the +// central case: a recent failure suppresses a new attempt and the +// `until` timestamp reflects window expiry. +func TestShouldSuppressAttempt_WithinWindowSuppresses(t *testing.T) { + now := time.Now() + prev := cfsStatus{ + AttemptCount: 1, + LastAttemptAt: ptrTime(now.Add(-1 * time.Minute)), // 1 min ago — well inside the 5 min window + } + suppress, until := shouldSuppressAttempt(prev, now) + if !suppress { + t.Fatal("expected suppression 1 min after failure with 5-min window") + } + want := prev.LastAttemptAt.Time.Add(5 * time.Minute) + if !until.Equal(want) { + t.Errorf("until = %v, want %v (last attempt + 5 min)", until, want) + } +} + +// TestShouldSuppressAttempt_WindowExpiredAllows verifies that +// suppression eventually clears. A failure 6 minutes ago (past the +// 5-min window for AttemptCount=1) must not block a new attempt. +func TestShouldSuppressAttempt_WindowExpiredAllows(t *testing.T) { + now := time.Now() + prev := cfsStatus{ + AttemptCount: 1, + LastAttemptAt: ptrTime(now.Add(-6 * time.Minute)), // past 5-min window + } + if suppress, _ := shouldSuppressAttempt(prev, now); suppress { + t.Fatal("expected NO suppression once 5-min window for attempt #1 has elapsed") + } +} + +// TestShouldSuppressAttempt_ExponentialEscalation verifies the +// schedule's exponential character: a 4th failure imposes a much +// longer window than a 1st failure, so a sustained-broken workload +// doesn't get re-attempted at the same cadence forever. +func TestShouldSuppressAttempt_ExponentialEscalation(t *testing.T) { + now := time.Now() + // 4th attempt failed 20 hours ago. Cap is 24h, so it should + // still suppress (20h < 24h). + prev := cfsStatus{ + AttemptCount: 4, + LastAttemptAt: ptrTime(now.Add(-20 * time.Hour)), + } + suppress, _ := shouldSuppressAttempt(prev, now) + if !suppress { + t.Fatal("4th failure 20h ago should still be within the 24h cap; expected suppress=true") + } + + // Same status but 25 hours ago — past the cap, should allow. + prev.LastAttemptAt = ptrTime(now.Add(-25 * time.Hour)) + if s, _ := shouldSuppressAttempt(prev, now); s { + t.Fatal("4th failure 25h ago should be past the 24h cap; expected suppress=false") + } +} + +// TestShouldSuppressAttempt_BoundaryAtExactExpiry pins behavior at +// the exact expiry instant (now == LastAttemptAt + window). Strict +// less-than means we ALLOW at the boundary — preferred so a polling +// loop with timestamp resolution >= 1s can't get stuck one tick on +// the suppress side forever. +func TestShouldSuppressAttempt_BoundaryAtExactExpiry(t *testing.T) { + last := time.Now().Add(-5 * time.Minute) + now := last.Add(5 * time.Minute) // exactly the boundary + prev := cfsStatus{ + AttemptCount: 1, + LastAttemptAt: ptrTime(last), + } + if s, _ := shouldSuppressAttempt(prev, now); s { + t.Fatal("boundary semantics: now==last+window should ALLOW (not suppress)") + } +} + +// ptrTime wraps t in metav1.Time and returns a pointer — matches how +// readStatus populates cfsStatus.LastAttemptAt. +func ptrTime(t time.Time) *metav1.Time { + mt := metav1.NewTime(t) + return &mt +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/captureonce_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/captureonce_test.go new file mode 100644 index 000000000..96d4bd002 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/captureonce_test.go @@ -0,0 +1,218 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package reconciler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" +) + +// capturingCFS seeds a NvSnapFunctionState already in the Capturing +// state, owned by `owner` with the given lease expiry — the shape the +// capture-once guard (nvca#189) writes when a reconcile wins the claim. +func capturingCFS(fvID, owner string, leaseExpiry time.Time) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": fvID}, + "spec": map[string]any{"functionVersionID": fvID}, + "status": map[string]any{ + "localCacheState": string(nvsnapv1alpha1.LocalCacheStateCapturing), + "captureOwner": owner, + "captureLeaseExpiry": leaseExpiry.UTC().Format(time.RFC3339), + }, + }} +} + +// TestTryClaimCapture covers the compare-and-swap state machine that +// makes capture single-flight per function-version (nvca#189). +func TestTryClaimCapture(t *testing.T) { + const fvID = "fv-claim" + ctx := context.Background() + now := time.Date(2026, 6, 20, 12, 0, 0, 0, time.UTC) + lease := now.Add(30 * time.Minute) + + t.Run("cold is claimable; stamps Capturing+owner+lease", func(t *testing.T) { + dyn := newFakeDynamic(coldCFS(fvID)) + claimed, err := tryClaimCapture(ctx, dyn, fvID, "ns1/podA", lease, now) + if err != nil { + t.Fatalf("tryClaimCapture: %v", err) + } + if !claimed { + t.Fatal("expected cold CFS to be claimable") + } + cur, err := dyn.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get: %v", err) + } + st := readStatus(cur) + if st.LocalCacheState != nvsnapv1alpha1.LocalCacheStateCapturing { + t.Errorf("state = %q, want Capturing", st.LocalCacheState) + } + if st.CaptureOwner != "ns1/podA" { + t.Errorf("owner = %q, want ns1/podA", st.CaptureOwner) + } + if st.CaptureLeaseExpiry == nil || !st.CaptureLeaseExpiry.Time.Equal(lease) { + t.Errorf("leaseExpiry = %v, want %v", st.CaptureLeaseExpiry, lease) + } + }) + + t.Run("a live claim by another owner is NOT claimable", func(t *testing.T) { + dyn := newFakeDynamic(capturingCFS(fvID, "ns1/podA", lease)) + claimed, err := tryClaimCapture(ctx, dyn, fvID, "ns1/podB", lease, now) + if err != nil { + t.Fatalf("tryClaimCapture: %v", err) + } + if claimed { + t.Fatal("podB must NOT claim while podA holds a live lease (thundering-herd guard)") + } + // Owner must be unchanged. + cur, _ := dyn.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if owner := readStatus(cur).CaptureOwner; owner != "ns1/podA" { + t.Errorf("owner = %q, want unchanged ns1/podA", owner) + } + }) + + t.Run("an EXPIRED claim is stealable by another owner", func(t *testing.T) { + expired := now.Add(-1 * time.Minute) // lease already elapsed + dyn := newFakeDynamic(capturingCFS(fvID, "ns1/podA", expired)) + claimed, err := tryClaimCapture(ctx, dyn, fvID, "ns1/podB", lease, now) + if err != nil { + t.Fatalf("tryClaimCapture: %v", err) + } + if !claimed { + t.Fatal("expired claim must be stealable (crash recovery)") + } + cur, _ := dyn.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if owner := readStatus(cur).CaptureOwner; owner != "ns1/podB" { + t.Errorf("owner = %q, want stolen to ns1/podB", owner) + } + }) + + t.Run("the owning pod re-claims re-entrantly (lease refresh)", func(t *testing.T) { + dyn := newFakeDynamic(capturingCFS(fvID, "ns1/podA", lease)) + newLease := now.Add(45 * time.Minute) + claimed, err := tryClaimCapture(ctx, dyn, fvID, "ns1/podA", newLease, now) + if err != nil { + t.Fatalf("tryClaimCapture: %v", err) + } + if !claimed { + t.Fatal("the owner must be able to re-claim its own live lease") + } + cur, _ := dyn.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if exp := readStatus(cur).CaptureLeaseExpiry; exp == nil || !exp.Time.Equal(newLease) { + t.Errorf("lease not refreshed: got %v want %v", exp, newLease) + } + }) + + t.Run("Warm is never claimable", func(t *testing.T) { + warm := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": fvID}, + "spec": map[string]any{"functionVersionID": fvID}, + "status": map[string]any{"localCacheState": string(nvsnapv1alpha1.LocalCacheStateWarm)}, + }} + dyn := newFakeDynamic(warm) + claimed, err := tryClaimCapture(ctx, dyn, fvID, "ns1/podB", lease, now) + if err != nil { + t.Fatalf("tryClaimCapture: %v", err) + } + if claimed { + t.Fatal("Warm CFS must not be claimed for capture") + } + }) +} + +// TestWriteStatusReleasesClaim verifies a terminal status write clears +// the Capturing claim — without this a completed/failed capture would +// leave captureOwner/leaseExpiry set and either pin the function or let +// readStatus mis-report a stale owner. +func TestWriteStatusReleasesClaim(t *testing.T) { + const fvID = "fv-release" + ctx := context.Background() + dyn := newFakeDynamic(capturingCFS(fvID, "ns1/podA", time.Now().Add(30*time.Minute))) + + if err := writeStatus(ctx, dyn, fvID, statusUpdate{ + CheckpointHash: "feedface", + CapturedHere: true, + LocalCacheState: nvsnapv1alpha1.LocalCacheStateWarm, + }); err != nil { + t.Fatalf("writeStatus: %v", err) + } + cur, _ := dyn.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + st := readStatus(cur) + if st.LocalCacheState != nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("state = %q, want Warm", st.LocalCacheState) + } + if st.CaptureOwner != "" || st.CaptureLeaseExpiry != nil { + t.Errorf("claim not released: owner=%q lease=%v", st.CaptureOwner, st.CaptureLeaseExpiry) + } +} + +// TestReconcileSkipsWhenCaptureInFlight is the end-to-end thundering- +// herd assertion: with a function-version already Capturing (owned by +// another pod, lease live), a second pod's reconcile must NOT POST a +// duplicate capture — it backs off on the capture-once claim (nvca#189). +func TestReconcileSkipsWhenCaptureInFlight(t *testing.T) { + const fvID = "fv-herd" + pod := inferencePod("podB", "ns1", fvID) // Ready, checkpoint-on-warm + + var capturePosted bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + // Recovery short-circuit (gate 0b) queries lookup; return no + // match so the reconcile falls through to the claim gate + // rather than recovering an existing capture. + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/api/v1/checkpoints/lookup"): + _ = json.NewEncoder(w).Encode(map[string]any{"matches": []map[string]any{}}) + // A capture POST here would be the duplicate we must prevent. + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/api/v1/checkpoints"): + capturePosted = true + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]any{"id": "ck-dup", "phase": "InProgress"}) + default: + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]any{}) + } + })) + defer srv.Close() + + // podA already holds a live Capturing claim on this fvID. + dyn := newFakeDynamic(capturingCFS(fvID, "ns1/podA", time.Now().Add(30*time.Minute))) + r := newTestReconciler(t, pod, srv, dyn) + + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if capturePosted { + t.Fatal("podB POSTed a duplicate capture while podA held the in-flight claim — thundering-herd guard failed") + } + // CFS must remain Capturing under podA (untouched by podB). + cur, _ := dyn.Resource(CFSResource).Get(context.Background(), fvID, metav1.GetOptions{}) + if st := readStatus(cur); st.CaptureOwner != "ns1/podA" { + t.Errorf("claim owner = %q, want unchanged ns1/podA", st.CaptureOwner) + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/coldstart_pioneer_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/coldstart_pioneer_test.go new file mode 100644 index 000000000..ab453c25d --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/coldstart_pioneer_test.go @@ -0,0 +1,177 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package reconciler + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" +) + +// mustGet reads CFS/ off the fake dynamic client or fails the test. +func mustGet(t *testing.T, dyn dynamic.Interface, fvID string) *unstructured.Unstructured { + t.Helper() + cur, err := dyn.Resource(CFSResource).Get(context.Background(), fvID, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get NvSnapFunctionState %s: %v", fvID, err) + } + return cur +} + +// pioneerCFS seeds a NvSnapFunctionState already holding a cold-start +// pioneer claim owned by `owner` with the given lease expiry — the shape +// TryClaimColdStartPioneer writes when a replica wins the election. +func pioneerCFS(fvID, owner string, leaseExpiry time.Time) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": fvID}, + "spec": map[string]any{"functionVersionID": fvID}, + "status": map[string]any{ + "localCacheState": string(nvsnapv1alpha1.LocalCacheStateCold), + "coldStartPioneer": owner, + "coldStartPioneerExpiry": leaseExpiry.UTC().Format(time.RFC3339), + }, + }} +} + +// stateCFS seeds a NvSnapFunctionState in a given localCacheState with no +// pioneer claim. +func stateCFS(fvID string, state nvsnapv1alpha1.NvSnapFunctionStateLocalCacheState) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": fvID}, + "spec": map[string]any{"functionVersionID": fvID}, + "status": map[string]any{"localCacheState": string(state)}, + }} +} + +// TestTryClaimColdStartPioneer covers the serialized-herd compare-and-swap +// that elects exactly one cold-start pioneer per function-version. +func TestTryClaimColdStartPioneer(t *testing.T) { + const fvID = "fv-pioneer" + ctx := context.Background() + now := time.Date(2026, 6, 24, 12, 0, 0, 0, time.UTC) + lease := now.Add(50 * time.Minute) + + t.Run("cold is claimable; stamps pioneer+expiry", func(t *testing.T) { + dyn := newFakeDynamic(coldCFS(fvID)) + claimed, err := TryClaimColdStartPioneer(ctx, dyn, fvID, "ns1/reqA", lease, now) + if err != nil { + t.Fatalf("TryClaimColdStartPioneer: %v", err) + } + if !claimed { + t.Fatal("expected cold CFS to be claimable as pioneer") + } + st := readStatus(mustGet(t, dyn, fvID)) + if st.ColdStartPioneer != "ns1/reqA" { + t.Errorf("pioneer = %q, want ns1/reqA", st.ColdStartPioneer) + } + if st.ColdStartPioneerExpiry == nil || !st.ColdStartPioneerExpiry.Time.Equal(lease) { + t.Errorf("pioneerExpiry = %v, want %v", st.ColdStartPioneerExpiry, lease) + } + }) + + t.Run("a live claim by another owner is NOT claimable", func(t *testing.T) { + dyn := newFakeDynamic(pioneerCFS(fvID, "ns1/reqA", lease)) + claimed, err := TryClaimColdStartPioneer(ctx, dyn, fvID, "ns1/reqB", lease, now) + if err != nil { + t.Fatalf("TryClaimColdStartPioneer: %v", err) + } + if claimed { + t.Fatal("reqB must NOT claim while reqA holds a live pioneer lease (serialized-herd)") + } + if p := readStatus(mustGet(t, dyn, fvID)).ColdStartPioneer; p != "ns1/reqA" { + t.Errorf("pioneer = %q, want unchanged ns1/reqA", p) + } + }) + + t.Run("an EXPIRED claim is stealable by another owner", func(t *testing.T) { + expired := now.Add(-1 * time.Minute) + dyn := newFakeDynamic(pioneerCFS(fvID, "ns1/reqA", expired)) + claimed, err := TryClaimColdStartPioneer(ctx, dyn, fvID, "ns1/reqB", lease, now) + if err != nil { + t.Fatalf("TryClaimColdStartPioneer: %v", err) + } + if !claimed { + t.Fatal("expired pioneer claim must be stealable (crash recovery)") + } + if p := readStatus(mustGet(t, dyn, fvID)).ColdStartPioneer; p != "ns1/reqB" { + t.Errorf("pioneer = %q, want stolen to ns1/reqB", p) + } + }) + + t.Run("the owning request re-claims re-entrantly (lease refresh)", func(t *testing.T) { + dyn := newFakeDynamic(pioneerCFS(fvID, "ns1/reqA", lease)) + newLease := now.Add(75 * time.Minute) + claimed, err := TryClaimColdStartPioneer(ctx, dyn, fvID, "ns1/reqA", newLease, now) + if err != nil { + t.Fatalf("TryClaimColdStartPioneer: %v", err) + } + if !claimed { + t.Fatal("the pioneer must be able to re-claim its own live lease") + } + if exp := readStatus(mustGet(t, dyn, fvID)).ColdStartPioneerExpiry; exp == nil || !exp.Time.Equal(newLease) { + t.Errorf("lease not refreshed: got %v want %v", exp, newLease) + } + }) + + t.Run("Warm is never claimable (caller proceeds to warm-restore)", func(t *testing.T) { + dyn := newFakeDynamic(stateCFS(fvID, nvsnapv1alpha1.LocalCacheStateWarm)) + claimed, err := TryClaimColdStartPioneer(ctx, dyn, fvID, "ns1/reqB", lease, now) + if err != nil { + t.Fatalf("TryClaimColdStartPioneer: %v", err) + } + if claimed { + t.Fatal("Warm CFS must not be claimed as pioneer") + } + }) + + t.Run("Failed is never claimable (caller fails open to cold)", func(t *testing.T) { + dyn := newFakeDynamic(stateCFS(fvID, nvsnapv1alpha1.LocalCacheStateFailed)) + claimed, err := TryClaimColdStartPioneer(ctx, dyn, fvID, "ns1/reqB", lease, now) + if err != nil { + t.Fatalf("TryClaimColdStartPioneer: %v", err) + } + if claimed { + t.Fatal("Failed CFS must not be claimed as pioneer") + } + }) +} + +// TestWriteStatusReleasesPioneerClaim verifies a terminal status write +// vacates the cold-start pioneer slot — without this a warmed function +// would keep a stale pioneer owner on status. +func TestWriteStatusReleasesPioneerClaim(t *testing.T) { + const fvID = "fv-pioneer-release" + ctx := context.Background() + dyn := newFakeDynamic(pioneerCFS(fvID, "ns1/reqA", time.Now().Add(50*time.Minute))) + + if err := writeStatus(ctx, dyn, fvID, statusUpdate{ + CheckpointHash: "feedface", + CapturedHere: true, + LocalCacheState: nvsnapv1alpha1.LocalCacheStateWarm, + }); err != nil { + t.Fatalf("writeStatus: %v", err) + } + st := readStatus(mustGet(t, dyn, fvID)) + if st.ColdStartPioneer != "" || st.ColdStartPioneerExpiry != nil { + t.Errorf("pioneer claim not released: owner=%q expiry=%v", st.ColdStartPioneer, st.ColdStartPioneerExpiry) + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/health.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/health.go new file mode 100644 index 000000000..8522ec8b1 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/health.go @@ -0,0 +1,47 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package reconciler + +import ( + corev1 "k8s.io/api/core/v1" +) + +// podReady reports whether the pod has condition Ready=True. +// +// Hook B reconciler trusts this as the "workload is warm enough" signal. +// K8s computes pod.Ready by aggregating each container's readinessProbe; +// for NVCF function pods the utils sidecar runs a httpGet probe against +// the inference container's /health endpoint, so PodReady=true implies +// inference /health returned 200. Re-polling /health from the +// reconciler would duplicate that exact RTT against the exact same +// endpoint — and would fail when the inference container has no +// containerPort declared (NVCA's default, since INFERENCE_PORT is +// stamped only on the utils sidecar's env). +// +// Duplicated from controller.IsPodReady to avoid the cyclic +// controller→reconciler→controller import. +func podReady(pod *corev1.Pod) bool { + if pod == nil { + return false + } + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/metrics.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/metrics.go new file mode 100644 index 000000000..58801286d --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/metrics.go @@ -0,0 +1,157 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package reconciler + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// captureProtocolErrors counts protocol-level failures returned by +// nvsnap-server that the reconciler can't act on — typically a bug in +// the agent or nvsnap-server contract (e.g., Completed phase without a +// content hash, see nvca#15 + nvnvsnap#61). Every increment is a +// situation the operator should investigate; wire a Prometheus alert +// rule on rate(...[5m]) > 0. +// +// Cardinality budget: one label `reason`, expected values +// {"empty_hash"}. New reasons must be enumerated below and documented +// — high-cardinality labels (e.g., per-checkpointID) belong in logs +// or traces, not here. +var captureProtocolErrors = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "nvca_nvsnap_capture_protocol_error_total", + Help: "Count of nvsnap-server / agent protocol violations the reconciler swallowed " + + "to avoid a retry storm. Non-zero means the nvsnap control-plane has a bug " + + "the operator should investigate. Labeled by reason: empty_hash = Completed " + + "checkpoint returned with no content hash (see nvnvsnap#61).", + }, + []string{"reason"}, +) + +// checkpointAttemptFailures counts checkpoint attempts that ended in a +// terminal failure and were SWALLOWED by the reconciler (returned nil +// up the controller stack so controller-runtime does NOT requeue). +// +// Why we swallow: a single failed checkpoint attempt should not trigger +// an infinite re-checkpoint loop. The pod is still running; the +// operator wants logs + a counter + the next pod-ready cycle to make +// the call, not a tight retry that re-attempts an 80 GB CRIU dump +// every 30 seconds. See nvsnap-h100-a 2026-06-03: a single L2 promote +// timeout drove 3 back-to-back full captures before manual +// intervention. +// +// Cardinality budget: one label `reason`, fixed set: +// +// create_failed — nvsnap-server CreateCheckpoint API returned error +// poll_failed — pollCheckpointTerminal returned error (timeout +// or non-transient API error) +// completed_failed — checkpoint reached terminal Phase=Failed +// promote_poll_failed — L2 promote-state poll timed out or hit a +// non-transient API error (nvca#179). The CRIU +// capture itself succeeded; only the L2 fan-out +// wasn't confirmed within the deadline. +// promote_failed — L2 promote terminally failed (Job error, +// snapshot failure, rox PVC bind failure). +// Capture is durable on L1 / peer cascade. +// +// Every increment is a situation the operator should investigate; wire +// a Prometheus alert rule on rate(...[5m]) > 0. +var checkpointAttemptFailures = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "nvca_nvsnap_checkpoint_attempt_failure_total", + Help: "Count of checkpoint attempts that failed terminally and were swallowed " + + "by the reconciler (no requeue). The pod continues to run cold; the next " + + "pod-ready event decides whether to re-attempt. Non-zero means an operator " + + "should investigate the underlying capture failure. Labels: " + + "reason={create_failed,poll_failed,completed_failed,promote_poll_failed,promote_failed}.", + }, + []string{"reason"}, +) + +// checkpointAttemptsSuppressed counts checkpoint attempts that were +// skipped because the per-function failure backoff (nvca#167) window +// has not yet elapsed. Sustained non-zero rate means the cluster has +// at least one workload NvSnap cannot capture — typically the workload +// is structurally incompatible (no LD_PRELOAD intercept, weird +// signal handling, kernel-version-specific bug) and the operator +// should either opt the function out (spec.optOut=true) or fix the +// agent. The label is intentionally minimal — distinguishing +// individual functions belongs in traces/logs, not here. +var checkpointAttemptsSuppressed = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "nvca_nvsnap_checkpoint_attempt_suppressed_total", + Help: "Count of checkpoint attempts skipped by the per-function failure-backoff " + + "gate. Sustained non-zero rate indicates at least one workload that NvSnap " + + "cannot capture; check CFS lastError and consider opting out.", + }, +) + +// checkpointAttemptsSkippedWarm counts attempts that were skipped +// because NvSnapFunctionState.status.localCacheState was already Warm +// (nvca#178). This fires when a duplicate pod-ready event reaches +// the reconciler after a successful capture — e.g. CRIU's freeze → +// thaw causes kubelet to flip Ready=true a second time, and the +// workqueue picks up the queued event before the informer-cached pod +// has the annotation-removal patch applied. The CFS check at the +// gate catches this; without it we observed a full duplicate ~3 min +// capture for the same hash on nvsnap-h100-a 2026-06-03. +// +// Non-zero rate is benign (the gate is doing its job). A SUSTAINED +// high rate per function would indicate a flapping pod ready signal +// independent of CRIU, which is worth investigating but doesn't +// damage anything. +var checkpointAttemptsSkippedWarm = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "nvca_nvsnap_checkpoint_attempt_skipped_warm_total", + Help: "Count of checkpoint attempts skipped because CFS LocalCacheState was " + + "already Warm at the reconciler gate (duplicate pod-ready trigger). " + + "Benign — the gate prevented a full duplicate capture.", + }, +) + +// checkpointAttemptsSkippedInflight counts attempts skipped by the +// capture-once claim (nvca#189): another pod of the same function- +// version already holds the Capturing claim, so this reconcile backed +// off instead of firing a duplicate capture. When N pods of a function +// deploy simultaneously cold, this counter rises to ~N-1 per capture — +// that's the thundering-herd guard doing its job (one capture, N-1 +// skips). A sustained high rate with NO completing capture would point +// at a stuck claim (owner crashing before lease expiry); investigate +// alongside the capture-failure counter. +var checkpointAttemptsSkippedInflight = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "nvca_nvsnap_checkpoint_attempt_skipped_inflight_total", + Help: "Count of checkpoint attempts skipped because another pod holds the " + + "in-flight capture claim for the same function-version (capture-once, " + + "nvca#189). Expected to rise to ~N-1 per capture when N pods deploy cold " + + "simultaneously — the guard prevented N-1 duplicate captures.", + }, +) + +// cfsSweepRecovered counts NvSnapFunctionStates that the controller's +// periodic recovery sweep flipped to Warm from an existing nvsnap-server +// capture WITHOUT a live source pod (nvca#104 durable-warm). A non-zero +// rate is healthy — the sweep is catching captures whose live reconcile +// died before writeStatus. A sustained HIGH rate suggests the happy-path +// reconcile is frequently interrupted (controller restarts, source pods +// scaled away mid-capture) and is worth investigating. See +// docs/users/nvsnap/DURABLE-WARM-SWEEP.md. +var cfsSweepRecovered = promauto.NewCounter( + prometheus.CounterOpts{ + Name: "nvca_nvsnap_cfs_sweep_recovered_total", + Help: "Count of NvSnapFunctionStates the periodic recovery sweep flipped to " + + "Warm from an existing capture without a live source pod (nvca#104). " + + "Closes the gap where an interrupted Hook B reconcile leaves CFS stuck " + + "not-Warm.", + }, +) diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/reconciler.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/reconciler.go new file mode 100644 index 000000000..62300c407 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/reconciler.go @@ -0,0 +1,724 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// Package reconciler implements Hook B of the NVCA × NvSnap integration: +// the post-Ready checkpoint loop. Pods stamped by Hook A (in +// pkg/nvca/nvsnap_hook.go) with nvsnap.io/checkpoint-on-warm: "true" are +// fed to this reconciler. It polls the inference container's health +// endpoint until it returns 200, waits the warmup buffer, POSTs a +// checkpoint request to nvsnap-server, polls until terminal, and +// updates NvSnapFunctionState.status. +// +// This package is the state machine only. The wiring (Pod informer + +// workqueue + agent startup) lands in a follow-up — keeps this PR +// scoped to the logic that's worth testing. +// +// Design: docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md §"Hook B". +package reconciler + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/sirupsen/logrus" + otelattr "go.opentelemetry.io/otel/attribute" + oteltrace "go.opentelemetry.io/otel/trace" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + + nvcaotel "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/otel" + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap" +) + +// Annotation keys driven by this reconciler. Must match the constants +// in pkg/nvca/nvsnap_hook.go — duplicated here to avoid an import cycle +// (nvsnap_hook.go lives in package nvca, which would otherwise import +// this package). +const ( + CheckpointOnWarmAnnotation = "nvsnap.io/checkpoint-on-warm" + RestoreFromAnnotation = "nvsnap.io/restore-from" +) + +// Default timing knobs. Mirror nvcfv1.DefaultNvSnapWarmup{Timeout,Buffer}Seconds — +// duplicated to avoid an import dep on the operator CRD package +// (which would require all transitive deps to be in this scope). +// Tests should override via Reconciler fields; production wires from +// the agent's runtime config (PR-6). +const ( + defaultWarmupBuffer = 10 * time.Second +) + +// FunctionVersionIDAnnotation carries the NGC function-version UUID +// on the pod. NVCA stamps this via labels-for-request elsewhere +// (nvsnap.io/source-function-version-id). The reconciler reads it off +// the pod rather than re-resolving from the ICMSRequest. +// +// NOTE: Hook A in PR-4 looks up FunctionVersionID from the +// ICMSRequest passed to CreatePodArtifactInstances. The reconciler +// doesn't have that request handy at watch time; an extra annotation +// on the pod is the bridge. PR-4 should stamp this annotation when it +// stamps CheckpointOnWarm — listed as a TODO in the integration doc. +const FunctionVersionIDAnnotation = "nvsnap.io/function-version-id" + +// Reconciler runs the post-Ready checkpoint state machine for one +// pod at a time. Construct via New + Reconcile. +// +// Concurrency: safe for many concurrent Reconcile calls on different +// pods (each call is self-contained). +type Reconciler struct { + // KubeClient mutates the pod (annotation removal at completion). + KubeClient kubernetes.Interface + + // DynClient reads/writes NvSnapFunctionState. + DynClient dynamic.Interface + + // NvSnapClient is the nvsnap-server HTTP client (PR-1). + NvSnapClient *nvsnap.Client + + // InferenceContainerName is the container the nvsnap-server should + // snapshot. Defaults to "inference" (NVCA convention). + InferenceContainerName string + + // WarmupBuffer is the dwell between PodReady=true and POST + // checkpoint. Default 10s. Lets post-Ready setup (CUDA graph + // capture, JIT warmup) land before the snapshot. NVCA's utils + // sidecar already runs the /health probe that gates PodReady, + // so the reconciler trusts that signal and only adds this dwell. + WarmupBuffer time.Duration + + // CheckpointPollInterval is how often the reconciler polls the + // nvsnap-server checkpoint by id. Default 5s. + CheckpointPollInterval time.Duration + + // CheckpointTimeout caps how long the reconciler waits for the + // checkpoint to reach terminal status. Default 30 min — large + // rootfs captures (NIM Qwen3-32B / vLLM 70B) can run minutes, + // plus uploads to nvsnap-blobstore. + CheckpointTimeout time.Duration + + // PromotePollInterval is how often the reconciler polls + // nvsnap-server's pvc-state endpoint after the checkpoint reaches + // terminal Completed (nvca#179). Default 5 s — same cadence as + // CheckpointPollInterval; the L2 snap+clone is comparable in + // duration to the CRIU dump itself for any non-trivial capture. + PromotePollInterval time.Duration + + // PromotePollTimeout caps how long the reconciler waits for the + // agent's L2 snap+clone to reach a terminal state. Default 15 + // min — Hyperdisk-ML snap+clone takes ~6 min for a 90 GB rwx + // PVC on nvsnap-h100-a 2026-06-03; 15 min gives 2.5× headroom for + // the largest captures and accommodates slower default snapshot + // classes. Beyond this, the reconciler logs and records a + // promote-poll failure on CFS — the underlying capture is still + // durable on L1 / peer cascade, only the L2 fan-out is missed. + PromotePollTimeout time.Duration + + // CaptureLeaseTTL bounds the capture-once Capturing claim + // (nvca#189). When a reconcile wins the Cold→Capturing CAS it + // stamps captureLeaseExpiry = now + CaptureLeaseTTL; other pods of + // the same function-version observe the live claim and back off. To + // guarantee a legitimately-running capture is never stolen + // mid-flight, this must exceed the maximum time one reconcile can + // hold the claim: WarmupBuffer + CheckpointTimeout + + // PromotePollTimeout, plus margin. Zero = derive that sum in + // applyDefaults. Only a crashed/hung capturer (one that exceeds the + // whole budget) has its claim expire and become stealable. + CaptureLeaseTTL time.Duration + + Log logrus.FieldLogger +} + +// applyDefaults fills zero fields with sane defaults. Called at the +// top of Reconcile so callers can use a partially-populated +// Reconciler without thinking about every knob. +func (r *Reconciler) applyDefaults() { + if r.InferenceContainerName == "" { + r.InferenceContainerName = "inference" + } + if r.WarmupBuffer == 0 { + r.WarmupBuffer = defaultWarmupBuffer + } + if r.CheckpointPollInterval == 0 { + r.CheckpointPollInterval = 5 * time.Second + } + if r.CheckpointTimeout == 0 { + r.CheckpointTimeout = 30 * time.Minute + } + if r.PromotePollInterval == 0 { + r.PromotePollInterval = 5 * time.Second + } + if r.PromotePollTimeout == 0 { + r.PromotePollTimeout = 15 * time.Minute + } + if r.CaptureLeaseTTL == 0 { + // Derive a lease that provably covers the longest a single + // reconcile can legitimately hold the Capturing claim — the + // full capture budget plus a 5-minute margin — so a valid + // in-flight capture is never stolen by a peer pod (nvca#189). + r.CaptureLeaseTTL = r.WarmupBuffer + r.CheckpointTimeout + r.PromotePollTimeout + 5*time.Minute + } + if r.Log == nil { + r.Log = logrus.NewEntry(logrus.New()) + } +} + +// DefaultColdStartPioneerLeaseTTL is the lease TTL the MiniService-creation +// gate (package nvca) stamps when it elects a cold-start pioneer +// (serialized-herd). It reuses the same concept as CaptureLeaseTTL: a +// deferred replica must not steal the pioneer slot while the pioneer is +// still legitimately cold-starting + capturing, so the lease must cover +// the full cold-start-to-Warm budget — warmup buffer + checkpoint +// timeout + L2 promote timeout — plus margin. The gate has no Reconciler +// instance, so this exposes the same default sum applyDefaults derives, +// computed from the package defaults. +func DefaultColdStartPioneerLeaseTTL() time.Duration { + return defaultWarmupBuffer + 30*time.Minute + 15*time.Minute + 5*time.Minute +} + +// Reconcile drives the checkpoint state machine for one pod, end to +// end. Blocks until the checkpoint reaches a terminal state, the +// context is canceled, or a timeout fires. +// +// The pod must: +// - have annotation CheckpointOnWarmAnnotation = "true" +// - have annotation FunctionVersionIDAnnotation set to the NGC FV id +// - be PodReady (caller responsibility — wire this from a Ready event) +// +// On terminal success: NvSnapFunctionState.status reflects the new +// hash + Warm cache state + CapturedHere=true; pod's +// checkpoint-on-warm annotation is removed (idempotency). +// +// On terminal failure: NvSnapFunctionState.status records the error +// and increments AttemptCount; pod's annotation remains so a future +// reconciler can retry. +func (r *Reconciler) Reconcile(ctx context.Context, pod *corev1.Pod) error { + r.applyDefaults() + if pod == nil { + return errors.New("reconciler: pod is nil") + } + + // Span covers the full Hook B reconcile — warmup buffer, POST + // nvsnap-server, poll until terminal, write CFS status. Pod uid + // is the link key so nvsnap-side spans (CRIU dump, cascade fetch) + // can be correlated to the NVCA admission/reconcile that + // triggered them. See docs/users/nvsnap/NVSNAP-INTEGRATION-DESIGN.md + // §"Observability" for the span hierarchy. + tracer := nvcaotel.NewTracer(nvcaotel.WithName("nvca.nvsnap.reconciler")) + ctx, span := tracer.Start(ctx, "nvsnap.hook_b.reconcile", + oteltrace.WithSpanKind(oteltrace.SpanKindInternal), + oteltrace.WithAttributes( + otelattr.String("k8s.pod.name", pod.Name), + otelattr.String("k8s.pod.namespace", pod.Namespace), + otelattr.String("k8s.pod.uid", string(pod.UID)), + ), + ) + defer span.End() + + log := r.Log.WithFields(logrus.Fields{ + "pod": pod.Namespace + "/" + pod.Name, + "component": "nvsnap-reconciler", + }) + + if pod.Annotations[CheckpointOnWarmAnnotation] != "true" { + log.Debug("skip: pod is not marked for checkpoint-on-warm") + return nil + } + fvID := pod.Annotations[FunctionVersionIDAnnotation] + if fvID == "" { + log.Warn("skip: pod has checkpoint-on-warm but no function-version annotation") + return nil + } + log = log.WithField("functionVersionID", fvID) + + cfs, err := getOrCreateCFS(ctx, r.DynClient, fvID) + if err != nil { + return fmt.Errorf("reconciler: NvSnapFunctionState bootstrap failed: %w", err) + } + if optedOut(cfs) { + log.Info("skip: function-version is opted out") + return r.removeCheckpointOnWarm(ctx, pod, log) // stop watching this pod + } + + // Persist this pod's workload-lookup inputs onto CFS.spec so the + // controller's recovery sweep can flip Warm from an existing capture + // even if THIS reconcile dies before writeStatus (nvca#104 + // durable-warm). Best-effort — a failure here only forgoes the sweep + // safety net for this function-version; the live reconcile below is + // unaffected. Done before the long poll so an interruption mid-poll + // still leaves the inputs persisted. + if c := findInferenceContainer(pod, r.InferenceContainerName); c != nil && c.Image != "" { + if err := writeWorkloadLookup(ctx, r.DynClient, fvID, c.Image, extractModelID(c)); err != nil { + log.WithError(err).Debug("could not persist spec.workloadLookup; recovery sweep won't cover this fvID until next reconcile") + } + } + + prev := readStatus(cfs) + + // 0a. Already-Warm short-circuit (nvca#178). The annotation alone + // is not enough to decide "should I capture?" — CFS state is the + // source of truth. Without this gate we observed a duplicate + // capture on nvsnap-h100-a 2026-06-03: + // + // T+0:00 capture #1 starts (pod transitions Ready=true) + // T+2:36 capture #1 finishes; writeStatus(Warm) + removeAnnotation + // T+2:36 CRIU's freeze→thaw caused kubelet to flip Ready=true + // AGAIN as the inference container resumed; that second + // Ready event was already queued in the workqueue + // T+2:36 capture #2 dispatched — annotation just removed but the + // informer-cached pod object that this Reconcile got still + // had it. Full duplicate ~3 min capture for the same hash. + // + // The failure backoff (nvca#167) doesn't help because the first + // attempt succeeded (AttemptCount=0). The annotation guard at the + // top of Reconcile() reads from the informer-cached pod object and + // can lag. The CFS read above goes straight to the API server via + // the dynamic client and is strongly consistent — that's the + // trustworthy gate. + // + // removeCheckpointOnWarm is idempotent (no-op when annotation + // already gone), so calling it here drains a stale annotation if + // one is still around and stops the workqueue from re-firing. + if prev.LocalCacheState == nvsnapv1alpha1.LocalCacheStateWarm { + checkpointAttemptsSkippedWarm.Inc() + capturedAt := "(unknown)" + if prev.CapturedAt != nil { + capturedAt = prev.CapturedAt.UTC().Format(time.RFC3339) + } + log.WithFields(logrus.Fields{ + "hash": prev.CheckpointHash, + "captured_at": capturedAt, + "reason": "cfs_already_warm", + }).Info("skip: CFS LocalCacheState=Warm — duplicate trigger ignored (likely informer lag after a successful capture)") + return r.removeCheckpointOnWarm(ctx, pod, log) + } + + // 0b. Recovery short-circuit (nvca#104). CFS is not Warm, but a + // usable capture may already exist — the previous reconcile could + // have completed the dump+promote and then been interrupted before + // writeStatus (controller restart, leader-election change, ctx + // cancel, source-pod redeploy). Rather than re-capture, ask + // nvsnap-server (the same content-addressed lookup Hook A uses) and, + // on a usable match, flip CFS=Warm directly. Idempotent; also closes + // the duplicate-capture race past the already-Warm gate (nvca#178/#189). + // Runs before the failure-backoff gate so a real, usable capture + // recovers even if a prior attempt was recorded as failed. + if hash := r.recoverExistingCapture(ctx, pod, log); hash != "" { + log.WithField("hash", hash). + Info("recovery: usable capture already exists for this workload — flipping CFS=Warm without re-capture (nvca#104)") + now := time.Now() + if err := writeStatus(ctx, r.DynClient, fvID, statusUpdate{ + CheckpointHash: hash, + CapturedHere: true, + CapturedAt: now, + LocalCacheState: nvsnapv1alpha1.LocalCacheStateWarm, + AttemptCount: 0, + LastError: "", + LastAttemptAt: now, + }); err != nil { + return fmt.Errorf("recovery: writeStatus Warm: %w", err) + } + return r.removeCheckpointOnWarm(ctx, pod, log) + } + + // 0c. Per-function failure backoff (nvca#167). If the previous + // attempt failed and we're still within the suppression window, + // don't fire another capture. Without this, every pod-ready + // event reconsiders — an incompatible workload gets hammered + // indefinitely. AttemptCount is reset to 0 on success so a + // healthy function never hits this path. + if suppress, until := shouldSuppressAttempt(prev, time.Now()); suppress { + checkpointAttemptsSuppressed.Inc() + log.WithFields(logrus.Fields{ + "attempt_count": prev.AttemptCount, + "last_error": prev.LastError, + "last_attempt_at": prev.LastAttemptAt.Time.UTC().Format(time.RFC3339), + "next_attempt_after": until.UTC().Format(time.RFC3339), + "backoff_window_sec": int64(until.Sub(prev.LastAttemptAt.Time).Seconds()), + }).Warn("checkpoint attempt suppressed by failure backoff; function continues cold until window elapses") + return nil + } + + // 1. Workload-ready signal: defer to K8s PodReady. + // + // The pod's own readinessProbe (utils sidecar's httpGet against + // the inference container's /health endpoint, per NVCA's + // convention) already fires only when the engine is loaded and + // serving. PodReady=true means K8s aggregated that signal and + // considers the pod ready. Re-polling /health from here would + // duplicate the same RTT, on the same port, against the same + // endpoint — and would fail when the inference container has + // no containerPort declared (which is the default for NVCF + // function pods: NVCA stamps INFERENCE_PORT on the utils + // sidecar's env, not on the inference container's ports[]). + // + // The controller already gates the workqueue on IsPodReady, so + // this defensive check just protects against direct Reconcile() + // calls (tests, future entrypoints) that bypass the controller. + if !podReady(pod) { + log.Debug("skip: pod not Ready; controller will re-enqueue on next event") + return nil + } + + // 1b. Capture-once claim (nvca#189 thundering-herd guard). With N + // pods of the same function-version admitted cold, each reaches + // here. Atomically transition CFS Cold→Capturing via Kubernetes + // optimistic concurrency; exactly one reconcile wins and runs the + // single capture. Losers observe the live claim and return without + // firing a duplicate (multi-GB) CRIU dump. A crashed/hung owner's + // claim expires after CaptureLeaseTTL and becomes stealable so the + // function isn't pinned Capturing forever. Placed after the + // podReady gate (only Ready pods capture) and before the warmup + // buffer so losers skip immediately rather than each dwelling. + claimNow := time.Now() + owner := pod.Namespace + "/" + pod.Name + claimed, err := tryClaimCapture(ctx, r.DynClient, fvID, owner, claimNow.Add(r.CaptureLeaseTTL), claimNow) + if err != nil { + return fmt.Errorf("capture-once claim for %s: %w", fvID, err) + } + if !claimed { + checkpointAttemptsSkippedInflight.Inc() + log.WithField("reason", "capture_in_flight"). + Info("skip: another pod holds the in-flight capture claim for this function-version (capture-once, nvca#189)") + return nil + } + + // 2. Dwell before snapshot so post-Ready setup completes. + if r.WarmupBuffer > 0 { + log.WithField("buffer", r.WarmupBuffer).Info("phase: warmup buffer") + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(r.WarmupBuffer): + } + } + + // 3. POST checkpoint to nvsnap-server. + log.Info("phase: POST nvsnap-server /api/v1/checkpoints") + ckpt, err := r.NvSnapClient.CreateCheckpoint(ctx, nvsnap.CheckpointRequest{ + Namespace: pod.Namespace, + PodName: pod.Name, + ContainerName: r.InferenceContainerName, + LeaveRunning: true, + }) + if err != nil { + return r.recordFailure(ctx, fvID, "create_failed", prev, fmt.Errorf("nvsnap CreateCheckpoint: %w", err), log) + } + log = log.WithField("checkpointID", ckpt.ID) + + // 4. Poll GetCheckpoint until terminal (or checkpoint-timeout). + pollCtx, pollCancel := context.WithTimeout(ctx, r.CheckpointTimeout) + defer pollCancel() + final, err := r.pollCheckpointTerminal(pollCtx, ckpt.ID, log) + if err != nil { + return r.recordFailure(ctx, fvID, "poll_failed", prev, err, log) + } + if final.Phase == nvsnap.PhaseFailed { + return r.recordFailure(ctx, fvID, "completed_failed", prev, + fmt.Errorf("nvsnap-server reported checkpoint %s Phase=Failed: %s", ckpt.ID, final.Error), log) + } + + // Defense in depth (nvca#14 + nvca#15). nvsnap-server should always + // return a non-empty content hash on a Completed checkpoint; an + // empty hash means the agent or nvsnap-server has a hash-propagation + // bug (see nvnvsnap#61). Behavior here used to be "return an error + // and let controller-runtime requeue" — but that caused a retry + // storm on nvsnap-h100-a (2026-05-31): 3 back-to-back 80 GB captures + // looped before manual intervention, because every requeue POSTs a + // fresh GPUCheckpoint job to nvsnap-server. + // + // Empty hash on Completed is a software bug, not a transient. + // Treat it as if the capture never happened: log loudly, bump an + // alertable counter so the operator notices, and return nil. The + // CFS stays in its previous state (no Warm transition, no Failed + // state-change either); the next pod-ready event will retry once. + // The defense in depth — refusing to stamp restore-from="" — stays + // in place; we just stop the storm. + if final.Hash == "" { + captureProtocolErrors.WithLabelValues("empty_hash").Inc() + log.WithFields(logrus.Fields{ + "checkpointID": ckpt.ID, + "phase": final.Phase, + "size": final.Size, + "duration": final.Duration, + "reason": "empty_hash", + }).Error("nvsnap-server returned Completed with empty hash — likely agent or nvsnap-server bug (see nvnvsnap#61). " + + "Skipping CFS update; pod stays cold. Next pod-ready event will retry.") + return nil + } + + // 5. L2 promote-state gate (nvca#179). The CRIU checkpoint reached + // terminal Completed, but the agent's L2 snap+clone runs ASYNC — + // rox- may not be Bound yet (or may have failed). If we + // flip CFS=Warm now, future function pods get nvsnap.io/restore-from + // and admit, but the rox PVC reference points at a not-yet-ready + // (or never-going-to-be-ready) PVC. Restore stalls or fails. + // + // Solution: poll nvsnap-server's /pvc-state until the agent reaches + // a terminal L2 state. Three outcomes: + // - state=ready: L2 done, fall through to write CFS=Warm + // - state=failed: L2 promote terminally failed. Record the + // failure on CFS but DON'T flip Warm — future pods will + // cold-start (no nvsnap.io/restore-from stamped). + // - state="" (or 404): agent has no L2 backend configured. + // Fall through to write CFS=Warm — restore uses L1/peer + // cascade, no L2 to wait on. + // + // Time budget: capped by PromotePollTimeout (default 15 min, + // covers the largest captures we've seen — Qwen3-32B ~6 min + // snap+clone on Hyperdisk-ML). A timeout records the partial + // state on CFS as "L2 stalled" so the operator can see it; it + // does NOT crash the reconcile — checkpoint capture was a + // success, we just don't get the L2 fan-out benefit this round. + promoteCtx, promoteCancel := context.WithTimeout(ctx, r.PromotePollTimeout) + defer promoteCancel() + promote, err := r.pollPVCPromoteTerminal(promoteCtx, final.Hash, log) + if err != nil { + // Transient/timeout — record but proceed cold. The capture + // itself succeeded; the L2 fan-out simply won't be available + // for the first restore wave. Future captures will retry. + log.WithError(err).WithField("hash", final.Hash). + Warn("L2 promote-state poll did not reach terminal — falling back to cold start (capture itself succeeded)") + return r.recordFailure(ctx, fvID, "promote_poll_failed", prev, + fmt.Errorf("L2 promote poll: %w", err), log) + } + switch promote.State { + case "failed": + // L2 promote terminally failed (Job error, snapshot failure, + // rox PVC bind failure). Capture is durable on L1 / peer + // cascade, but no L2 fan-out for this hash. Function pods + // will admit without restore-from and cold-start; next + // capture attempt may succeed L2 (e.g. transient SC issue). + log.WithFields(logrus.Fields{ + "hash": final.Hash, + "pvc_name": promote.PVCName, + }).Warn("L2 promote terminally failed — pods will cold-start until next successful capture") + return r.recordFailure(ctx, fvID, "promote_failed", prev, + fmt.Errorf("L2 promote state=failed for hash %s", final.Hash), log) + case "ready": + log.WithFields(logrus.Fields{ + "hash": final.Hash, + "pvc_name": promote.PVCName, + }).Info("L2 promote ready — rox PVC bound, safe to flip CFS=Warm") + default: + // Empty state or terminal-but-no-L2-configured. The capture + // is on L1 / peer cascade only; restore will work via the + // existing fallback path. Proceed to flip Warm. + log.WithField("hash", final.Hash). + Info("L2 not in use for this capture (no agent backend configured) — flipping CFS=Warm via peer-cascade fallback") + } + + // 6. Success — update CFS, drop the annotation. + now := time.Now() + if err := writeStatus(ctx, r.DynClient, fvID, statusUpdate{ + CheckpointHash: final.Hash, + CapturedHere: true, + CapturedAt: now, + LocalCacheState: nvsnapv1alpha1.LocalCacheStateWarm, + AttemptCount: 0, + LastError: "", + LastAttemptAt: now, + }); err != nil { + log.WithError(err).Error("status update failed after successful checkpoint; will retry on next reconcile") + return err + } + log.WithFields(logrus.Fields{ + "hash": final.Hash, + "size": final.Size, + "duration": final.Duration, + }).Info("checkpoint committed") + + return r.removeCheckpointOnWarm(ctx, pod, log) +} + +// pollCheckpointTerminal polls nvsnap-server for the checkpoint's +// terminal phase. Returns the final Checkpoint when reached, or an +// error if the context expires, the API returns a non-transient +// failure (404 = checkpoint no longer exists; 4xx other than 429 = +// permanent client error), or transient transport errors persist +// past the context deadline. +// +// Greptile P2 (nvca!1748): the old behavior was to silently retry +// every error — including a permanent 404 — until CheckpointTimeout +// (default 30 min) expired. A nvsnap-server that lost the row, a +// retention sweep that deleted the in-flight checkpoint, or a +// bad-id bug would all spin instead of failing fast. +func (r *Reconciler) pollCheckpointTerminal(ctx context.Context, id string, log logrus.FieldLogger) (*nvsnap.Checkpoint, error) { + ticker := time.NewTicker(r.CheckpointPollInterval) + defer ticker.Stop() + for { + c, err := r.NvSnapClient.GetCheckpoint(ctx, id) + if err != nil { + if isNonTransientAPIError(err) { + return nil, fmt.Errorf("poll checkpoint %s: non-transient error: %w", id, err) + } + // Transient (transport error, 5xx, 429). Log + keep polling + // until the context deadline. + log.WithError(err).Debug("GetCheckpoint failed; will retry until deadline") + } else if c.IsTerminal() { + return c, nil + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("poll checkpoint %s: %w", id, ctx.Err()) + case <-ticker.C: + } + } +} + +// pollPVCPromoteTerminal polls nvsnap-server's /pvc-state until the +// agent's async L2 snap+clone reaches a terminal state (ready, +// failed, or "" / 404 = no L2 backend). Returns the final state on +// terminal, or an error on context expiry / non-transient API +// failure. +// +// Symmetric to pollCheckpointTerminal — same retry policy, same +// fail-fast on permanent 4xx, same transient-error swallow on +// transport hiccups. The only difference: 404 on /pvc-state is a +// terminal "no L2" signal (caller proceeds with cold-start / +// peer-cascade restore), NOT a fatal error. +func (r *Reconciler) pollPVCPromoteTerminal(ctx context.Context, hash string, log logrus.FieldLogger) (*nvsnap.PVCPromoteState, error) { + ticker := time.NewTicker(r.PromotePollInterval) + defer ticker.Stop() + for { + s, err := r.NvSnapClient.GetPVCPromoteState(ctx, hash) + if err != nil { + var apiErr *nvsnap.APIError + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound { + // 404 → catalog has no row for this hash. Agent has no + // L2 backend configured for this capture. Return a + // synthetic "no L2" state so the caller falls through + // to the cold/peer-cascade fallback without erroring. + log.WithField("hash", hash). + Debug("pvc-state 404 — agent has no L2 backend; proceed without L2") + return &nvsnap.PVCPromoteState{Hash: hash, State: ""}, nil + } + if isNonTransientAPIError(err) { + return nil, fmt.Errorf("poll pvc-state %s: non-transient error: %w", hash, err) + } + log.WithError(err).Debug("GetPVCPromoteState failed; will retry until deadline") + } else if s.IsTerminal() { + return s, nil + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("poll pvc-state %s: %w", hash, ctx.Err()) + case <-ticker.C: + } + } +} + +// isNonTransientAPIError returns true when err is an *APIError whose +// status code indicates the request will never succeed on retry +// (4xx, except 429 which IS transient). Transport errors (network +// failures) are treated as transient by definition. +func isNonTransientAPIError(err error) bool { + var apiErr *nvsnap.APIError + if !errors.As(err, &apiErr) { + return false // transport error → transient + } + if apiErr.StatusCode == http.StatusTooManyRequests { + return false + } + return apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 +} + +// recordFailure writes the error into NvSnapFunctionState.status, logs +// the full failure context at Error level, bumps an alertable counter, +// and returns nil so controller-runtime does NOT requeue. The next +// pod-ready event will decide whether to re-attempt — we explicitly +// do NOT trigger a tight retry loop here. +// +// Why no requeue (instead of returning the error like a normal +// reconciler would): a checkpoint attempt is heavy (an 80 GB CRIU +// dump can take 2-3 minutes; an L2 promote can pile on another +// 5-8 minutes of GCP storage operations). controller-runtime's +// rate-limited requeue would re-fire every ~30 seconds — fast enough +// to wedge the source pod's GPU under repeat capture pressure but +// slow enough that the operator wouldn't notice for hours. Empirically +// reproduced on nvsnap-h100-a 2026-06-03: a single L2 timeout drove +// 3 back-to-back full captures of the same hash before manual +// intervention. The defense-in-depth pattern (nvca#15) for the +// empty-hash case is the same: log loudly, count, return nil. +// +// AttemptCount on CFS is still incremented so operators can see how +// many attempts a given function has burned across pod-ready events. +// +// `reason` is a small enum that lands in the metric label — keep the +// set bounded (see metrics.go). +func (r *Reconciler) recordFailure(ctx context.Context, fvID, reason string, prev cfsStatus, cause error, log logrus.FieldLogger) error { + now := time.Now() + upd := statusUpdate{ + CheckpointHash: prev.CheckpointHash, + CapturedHere: prev.CapturedHere, + LocalCacheState: prev.LocalCacheState, + AttemptCount: prev.AttemptCount + 1, + LastError: cause.Error(), + LastAttemptAt: now, + } + // Forward CapturedAt from prior status. Without this, every + // failed re-checkpoint wipes the timestamp set by an earlier + // successful capture, hiding it from operators debugging a CFS + // that was once Warm. + if prev.CapturedAt != nil { + upd.CapturedAt = prev.CapturedAt.Time + } + if err := writeStatus(ctx, r.DynClient, fvID, upd); err != nil { + log.WithError(err).Error("failed to write failure status; original error preserved in log") + } + checkpointAttemptFailures.WithLabelValues(reason).Inc() + // Loud, structured log so the operator has everything in one line: + // reason, full error chain, attempt count, prior CFS state. Error + // level (not Warn) because every entry is an attempt that won't be + // retried — the operator must look. + log.WithError(cause).WithFields(logrus.Fields{ + "reason": reason, + "attempt_count": upd.AttemptCount, + "prev_cache_state": prev.LocalCacheState, + "prev_hash": prev.CheckpointHash, + "function_version": fvID, + "swallowed_retry": true, + }).Error("checkpoint attempt failed; NOT retrying — next pod-ready event will reconsider") + // Return nil: no requeue. The pod stays running cold; controller- + // runtime drops the work item. Defense-in-depth against the + // retry storm pattern (nvca#15). + return nil +} + +// removeCheckpointOnWarm patches the pod to drop the annotation, so +// a re-reconcile (e.g. controller restart) doesn't checkpoint again. +// Failure to patch is logged but not fatal — the next reconciler can +// observe the warm cache state and skip via Hook A's gating instead. +func (r *Reconciler) removeCheckpointOnWarm(ctx context.Context, pod *corev1.Pod, log logrus.FieldLogger) error { + if _, ok := pod.Annotations[CheckpointOnWarmAnnotation]; !ok { + return nil + } + // Strategic-merge-patch with null deletes the annotation key. + patch := []byte(`{"metadata":{"annotations":{"nvsnap.io/checkpoint-on-warm":null}}}`) + _, err := r.KubeClient.CoreV1().Pods(pod.Namespace).Patch( + ctx, pod.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // Pod is gone — nothing to do. + return nil + } + log.WithError(err).Warn("failed to remove nvsnap.io/checkpoint-on-warm annotation") + return err + } + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/reconciler_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/reconciler_test.go new file mode 100644 index 000000000..c56a1a30a --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/reconciler_test.go @@ -0,0 +1,991 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package reconciler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + fakedynamic "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap" +) + +// newFakeDynamic returns a fake dynamic client wired for the +// NvSnapFunctionState resource. The unstructured scheme registration +// must match what fakedynamic.NewSimpleDynamicClientWithCustomListKinds +// expects: GVR → list-kind name. +func newFakeDynamic(seed ...*unstructured.Unstructured) *fakedynamic.FakeDynamicClient { + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(schema.GroupVersionKind{ + Group: CFSResource.Group, + Version: CFSResource.Version, + Kind: "NvSnapFunctionState", + }, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(schema.GroupVersionKind{ + Group: CFSResource.Group, + Version: CFSResource.Version, + Kind: "NvSnapFunctionStateList", + }, &unstructured.UnstructuredList{}) + objs := make([]runtime.Object, 0, len(seed)) + for _, o := range seed { + objs = append(objs, o) + } + return fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, + map[schema.GroupVersionResource]string{CFSResource: "NvSnapFunctionStateList"}, + objs..., + ) +} + +// inferencePod builds a pod with the annotations Hook A would have +// stamped, the inference container, and PodReady=True (which the +// reconciler now trusts as the workload-warm signal). +func inferencePod(podName, namespace, fvID string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + Annotations: map[string]string{ + CheckpointOnWarmAnnotation: "true", + FunctionVersionIDAnnotation: fvID, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "inference"}, + }, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + } +} + +// nvsnapServerStub simulates nvsnap-server's POST/GET /api/v1/checkpoints. +// First POST returns InProgress; subsequent GETs return Completed +// after pollsBeforeCompleted polls. +type nvsnapServerStub struct { + pollsBeforeCompleted int32 + polls int32 + hash string + size int64 + failAt string // empty | "create" | "status" | "pvc-state" + statusCode int // optional override (defaults to 500 when failAt=="status") + + // nvca#179: L2 /pvc-state simulation. + promotePolls int32 + promotePollsBeforeTerminal int32 // 0 = first poll is terminal + promoteTerminalState string // "ready" | "failed" | "" (default ready) + promote404 bool // if true, /pvc-state returns 404 (no-L2 case) +} + +func (s *nvsnapServerStub) handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // nvca#179: /pvc-state must be matched BEFORE the catchall + // status handler — the path also contains "/api/v1/checkpoints/". + isPVCState := r.Method == http.MethodGet && + strings.Contains(r.URL.Path, "/api/v1/checkpoints/by-hash/") && + strings.HasSuffix(r.URL.Path, "/pvc-state") + switch { + case isPVCState: + if s.failAt == "pvc-state" { + code := s.statusCode + if code == 0 { + code = http.StatusInternalServerError + } + http.Error(w, "boom", code) + return + } + if s.promote404 { + http.NotFound(w, r) + return + } + n := atomic.AddInt32(&s.promotePolls, 1) + state := "pending" + if n > s.promotePollsBeforeTerminal { + state = s.promoteTerminalState + if state == "" { + state = "ready" + } + } + body := struct { + Hash string `json:"hash"` + State string `json:"state"` + PVCName string `json:"pvc_name,omitempty"` + }{Hash: s.hash, State: state} + if state == "ready" { + // Mirror agent naming: rox-; guard for + // short test hashes so the stub doesn't panic. + short := s.hash + if len(short) > 8 { + short = short[:8] + } + body.PVCName = "rox-" + short + } + _ = json.NewEncoder(w).Encode(body) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/api/v1/checkpoints"): + if s.failAt == "create" { + http.Error(w, "boom", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(nvsnap.Checkpoint{ID: "ck-1", Phase: nvsnap.PhaseInProgress}) + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/api/v1/checkpoints/"): + if s.failAt == "status" { + code := s.statusCode + if code == 0 { + code = http.StatusInternalServerError + } + http.Error(w, "boom", code) + return + } + n := atomic.AddInt32(&s.polls, 1) + c := nvsnap.Checkpoint{ID: "ck-1", Phase: nvsnap.PhaseInProgress} + if n >= s.pollsBeforeCompleted { + c.Phase = nvsnap.PhaseCompleted + c.Hash = s.hash + c.Size = s.size + } + _ = json.NewEncoder(w).Encode(c) + default: + http.NotFound(w, r) + } + }) +} + +func newTestReconciler(t *testing.T, pod *corev1.Pod, srv *httptest.Server, dyn *fakedynamic.FakeDynamicClient) *Reconciler { + t.Helper() + return &Reconciler{ + KubeClient: fake.NewSimpleClientset(pod), + DynClient: dyn, + NvSnapClient: nvsnap.NewClient(nvsnap.WithBaseURL(srv.URL)), + // applyDefaults treats 0 as "unset"; use a tiny non-zero value + // so the test doesn't wait the production default (10s). + WarmupBuffer: 1 * time.Microsecond, + CheckpointPollInterval: 10 * time.Millisecond, + CheckpointTimeout: 2 * time.Second, + // nvca#179 L2 promote-state poll. Tests use tiny intervals + // so they don't wait the production default (15 min). The + // happy-path stub returns "ready" on first poll, so 100 ms + // timeout is plenty. + PromotePollInterval: 10 * time.Millisecond, + PromotePollTimeout: 500 * time.Millisecond, + Log: logrus.NewEntry(logrus.New()), + } +} + +func TestReconcileHappyPath(t *testing.T) { + // Stand up the inference /health server and override the pod's + // PodIP+port to point at it. + pod := inferencePod("p1", "ns1", "fv-1") + + stub := &nvsnapServerStub{pollsBeforeCompleted: 2, hash: "abcdef", size: 12345} + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() // CFS doesn't exist yet — reconciler creates it + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + // CFS should be created and status filled. + got, err := dyn.Resource(CFSResource).Get(context.Background(), "fv-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + s := readStatus(got) + if s.CheckpointHash != "abcdef" { + t.Errorf("hash = %q, want abcdef", s.CheckpointHash) + } + if !s.CapturedHere { + t.Error("CapturedHere should be true after a self-driven capture") + } + if s.LocalCacheState != nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("LocalCacheState = %q, want Warm", s.LocalCacheState) + } + if s.AttemptCount != 0 { + t.Errorf("AttemptCount = %d, want 0 (reset on success)", s.AttemptCount) + } + + // Annotation must be removed so a re-reconcile is a no-op. + updatedPod, _ := r.KubeClient.CoreV1().Pods("ns1").Get(context.Background(), "p1", metav1.GetOptions{}) + if _, present := updatedPod.Annotations[CheckpointOnWarmAnnotation]; present { + t.Error("nvsnap.io/checkpoint-on-warm annotation should be removed after success") + } +} + +// Greptile P2 on nvca!1748: pollCheckpointTerminal used to silently +// retry every error (including a permanent 404) until the full +// CheckpointTimeout (30 min default). A nvsnap-server that lost the +// row, or a retention sweep that deleted the in-flight checkpoint, +// would spin instead of failing fast. The fix routes 4xx (non-429) +// errors through isNonTransientAPIError → recordFailure. +func TestPollCheckpointTerminal_BreaksOn404(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-poll404") + stub := &nvsnapServerStub{failAt: "status", statusCode: http.StatusNotFound} + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + // Set a long timeout — the test should NOT take that long. + // If pollCheckpointTerminal spins on the 404, the test times out + // at the t.Deadline; if it breaks fast, we finish in milliseconds. + r.CheckpointTimeout = 30 * time.Second + + start := time.Now() + err := r.Reconcile(context.Background(), pod) + elapsed := time.Since(start) + + // Reconcile MUST return nil (no controller-runtime requeue). The + // failure detail lives on CFS.status.lastError. + if err != nil { + t.Fatalf("Reconcile must return nil (no requeue on swallowed failure); got %v", err) + } + got, gerr := dyn.Resource(CFSResource).Get(context.Background(), "fv-poll404", metav1.GetOptions{}) + if gerr != nil { + t.Fatalf("get CFS: %v", gerr) + } + lastErr, _, _ := unstructured.NestedString(got.Object, "status", "lastError") + if !strings.Contains(lastErr, "non-transient") { + t.Errorf("CFS.status.lastError should mention non-transient; got: %q", lastErr) + } + // Must fail fast, not wait for CheckpointTimeout — fast-fail on + // 404 is the behavior under test; the error-swallowing fix didn't + // change that. + if elapsed > 2*time.Second { + t.Errorf("took %v — should fast-fail well under CheckpointTimeout=30s", elapsed) + } +} + +// 5xx errors are transient and SHOULD keep polling. Confirms we +// didn't over-eagerly mark every error as non-transient. +func TestPollCheckpointTerminal_RetriesOn5xx(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-poll5xx") + stub := &nvsnapServerStub{failAt: "status", statusCode: http.StatusBadGateway} + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + // Tight deadline — we expect this to spin and time out (proving + // 5xx is treated as transient, not fast-failed like 404). + r.CheckpointTimeout = 100 * time.Millisecond + + // Reconcile MUST return nil (no requeue). Behavior under test + // (transient retry until timeout) is reflected in CFS.lastError. + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile must return nil (no requeue); got %v", err) + } + got, gerr := dyn.Resource(CFSResource).Get(context.Background(), "fv-poll5xx", metav1.GetOptions{}) + if gerr != nil { + t.Fatalf("get CFS: %v", gerr) + } + lastErr, _, _ := unstructured.NestedString(got.Object, "status", "lastError") + // Should be a context-deadline-style error (transient retry path), + // not a non-transient fast-fail. + if strings.Contains(lastErr, "non-transient") { + t.Errorf("5xx should NOT be marked non-transient in CFS.lastError; got: %q", lastErr) + } + if !strings.Contains(lastErr, "context deadline") { + t.Errorf("CFS.lastError should mention context deadline (transient retry hit timeout); got: %q", lastErr) + } +} + +// 429 (Too Many Requests) is transient — server is asking us to back +// off, not telling us the request is permanently rejected. +func TestPollCheckpointTerminal_RetriesOn429(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-poll429") + stub := &nvsnapServerStub{failAt: "status", statusCode: http.StatusTooManyRequests} + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + r.CheckpointTimeout = 100 * time.Millisecond + + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile must return nil (no requeue); got %v", err) + } + got, gerr := dyn.Resource(CFSResource).Get(context.Background(), "fv-poll429", metav1.GetOptions{}) + if gerr != nil { + t.Fatalf("get CFS: %v", gerr) + } + lastErr, _, _ := unstructured.NestedString(got.Object, "status", "lastError") + if strings.Contains(lastErr, "non-transient") { + t.Errorf("429 should NOT be marked non-transient in CFS.lastError; got: %q", lastErr) + } + if !strings.Contains(lastErr, "context deadline") { + t.Errorf("CFS.lastError should mention context deadline (transient retry hit timeout); got: %q", lastErr) + } +} + +// nvca#14 + nvca#15: nvsnap-server returning Completed + empty hash is +// a control-plane bug (likely agent hash-propagation, nvnvsnap#61). The +// reconciler must refuse to mark Warm — that defense in depth stays — +// but it must also break the retry storm: returning an error here +// re-queues, which on nvsnap-h100-a (2026-05-31) caused 3 back-to-back +// 80 GB captures before manual intervention. New contract: +// +// - return nil (no requeue) +// - CFS state untouched (no Warm transition, no Failed flag-flip) +// - increment nvca_nvsnap_capture_protocol_error_total{reason="empty_hash"} +// so the operator sees an alertable signal +func TestReconcileSwallowsEmptyHashOnCompleted(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-empty-hash") + + // Stub returns Completed terminal phase but with an empty hash. + stub := &nvsnapServerStub{pollsBeforeCompleted: 1, hash: "", size: 12345} + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + // Snapshot the counter before so we can assert the increment is + // attributable to this Reconcile call. + before := testutil.ToFloat64(captureProtocolErrors.WithLabelValues("empty_hash")) + + err := r.Reconcile(context.Background(), pod) + if err != nil { + t.Fatalf("Reconcile should swallow empty-hash (return nil) to break retry storm; got: %v", err) + } + + after := testutil.ToFloat64(captureProtocolErrors.WithLabelValues("empty_hash")) + if after-before != 1 { + t.Errorf("captureProtocolErrors{reason=empty_hash} delta = %v, want 1", after-before) + } + + // CFS must NOT transition to Warm (defense in depth). + got, err := dyn.Resource(CFSResource).Get(context.Background(), "fv-empty-hash", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + s := readStatus(got) + if s.LocalCacheState == nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("LocalCacheState = Warm despite empty hash; defense in depth failed") + } + if s.CheckpointHash != "" { + t.Errorf("CheckpointHash = %q; should remain empty", s.CheckpointHash) + } + // CFS should also NOT have a Failed flag-flip or LastError set — + // the reconciler treats this as "capture never happened," so the + // CFS stays in whatever state it was. Pod-ready event triggers a + // fresh attempt. + if s.LocalCacheState == nvsnapv1alpha1.LocalCacheStateFailed { + t.Errorf("LocalCacheState = Failed; should be untouched when swallowing empty-hash") + } + if s.LastError != "" { + t.Errorf("LastError should stay empty (we don't surface as a CFS error); got %q", s.LastError) + } +} + +func TestReconcileSkipsOptOut(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-1") + + // Seed an opted-out CFS. + cfs := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": "fv-1"}, + "spec": map[string]any{"functionVersionID": "fv-1", "optOut": true}, + "status": map[string]any{}, + }} + dyn := newFakeDynamic(cfs) + nvsnapSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("nvsnap-server should NOT be called for opted-out FV; got %s %s", r.Method, r.URL.Path) + })) + defer nvsnapSrv.Close() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile: %v", err) + } + // Annotation must be removed even when skipping, so the reconciler + // doesn't re-attempt every time the pod's status updates. + updatedPod, _ := r.KubeClient.CoreV1().Pods("ns1").Get(context.Background(), "p1", metav1.GetOptions{}) + if _, present := updatedPod.Annotations[CheckpointOnWarmAnnotation]; present { + t.Error("annotation should be removed on opt-out skip") + } +} + +// TestReconcileSkipsWhenCFSAlreadyWarm is the regression test for the +// duplicate-capture race observed on nvsnap-h100-a 2026-06-03 (nvca#178): +// after capture #1 succeeded, CRIU's freeze→thaw caused kubelet to +// flip Ready=true a second time; controller-runtime had already +// queued the second event; when the worker picked it up, the +// informer-cached pod still had the annotation present (the +// annotation-removal patch hadn't propagated). Without a CFS-Warm +// gate, the reconciler proceeded with a full duplicate ~3 min +// capture. +// +// The fix: read CFS fresh (already does via dynamic client direct +// Get), and skip the rest of Reconcile when LocalCacheState=Warm. +// Annotation removal is still called as a safety net to drain the +// queued trigger. +func TestReconcileSkipsWhenCFSAlreadyWarm(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-1") + + // Seed a Warm CFS to simulate "capture #1 already finished". + cfs := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": "fv-1"}, + "spec": map[string]any{"functionVersionID": "fv-1"}, + "status": map[string]any{ + "localCacheState": string(nvsnapv1alpha1.LocalCacheStateWarm), + "checkpointHash": "abcdef0123456789", + "capturedHere": true, + "capturedAt": time.Now().UTC().Format(time.RFC3339), + "attemptCount": int64(0), + }, + }} + dyn := newFakeDynamic(cfs) + + // nvsnap-server MUST NOT be contacted for a Warm function — that's + // the entire point of the gate. Any request fails the test. + nvsnapSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("nvsnap-server should NOT be called for Warm CFS; got %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + })) + defer nvsnapSrv.Close() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + // Take a baseline of the counter so we can assert it incremented + // exactly once after Reconcile. + beforeWarm := testutil.ToFloat64(checkpointAttemptsSkippedWarm) + beforeSuppressed := testutil.ToFloat64(checkpointAttemptsSuppressed) + + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile must return nil on Warm skip; got %v", err) + } + + // Metric assertions: + // - checkpointAttemptsSkippedWarm: exactly +1 (the new gate) + // - checkpointAttemptsSuppressed: unchanged (different gate, must not double-count) + afterWarm := testutil.ToFloat64(checkpointAttemptsSkippedWarm) + if afterWarm-beforeWarm != 1 { + t.Errorf("checkpointAttemptsSkippedWarm: +%v, want +1", afterWarm-beforeWarm) + } + afterSuppressed := testutil.ToFloat64(checkpointAttemptsSuppressed) + if afterSuppressed != beforeSuppressed { + t.Errorf("checkpointAttemptsSuppressed should NOT increment on Warm skip; got delta=%v", afterSuppressed-beforeSuppressed) + } + + // CFS state must be untouched — no AttemptCount bump, no + // LastError set, no LastAttemptAt timestamp. The Warm skip is a + // no-op on state, only on the trigger. + got, _ := dyn.Resource(CFSResource).Get(context.Background(), "fv-1", metav1.GetOptions{}) + s := readStatus(got) + if s.LocalCacheState != nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("LocalCacheState changed: got %q, want Warm", s.LocalCacheState) + } + if s.AttemptCount != 0 { + t.Errorf("AttemptCount bumped to %d on Warm skip — should be 0", s.AttemptCount) + } + if s.LastError != "" { + t.Errorf("LastError set to %q on Warm skip — should remain empty", s.LastError) + } + if s.CheckpointHash != "abcdef0123456789" { + t.Errorf("CheckpointHash changed: got %q, want abcdef0123456789", s.CheckpointHash) + } + + // Annotation MUST be removed even on the Warm skip — this is the + // stale-annotation drain that stops the workqueue from re-firing + // indefinitely. Same semantics as the opt-out skip path. + updatedPod, _ := r.KubeClient.CoreV1().Pods("ns1").Get(context.Background(), "p1", metav1.GetOptions{}) + if _, present := updatedPod.Annotations[CheckpointOnWarmAnnotation]; present { + t.Error("annotation should be removed on Warm skip so the workqueue stops re-firing for this pod") + } +} + +// TestReconcileWarmGate_DoesNotFireOnCold confirms the gate is +// specific: a Cold CFS must NOT take the Warm short-circuit. Without +// this assertion the new gate could regress into "skip everything" +// if LocalCacheState's zero-value matched the comparison. +func TestReconcileWarmGate_DoesNotFireOnCold(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-1") + dyn := newFakeDynamic() // CFS doesn't exist; reconciler creates Cold + + stub := &nvsnapServerStub{pollsBeforeCompleted: 1, hash: "newhash", size: 1} + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + beforeWarm := testutil.ToFloat64(checkpointAttemptsSkippedWarm) + + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + // Counter MUST NOT increment on Cold path. CFS must end Warm + // (the happy path). + afterWarm := testutil.ToFloat64(checkpointAttemptsSkippedWarm) + if afterWarm != beforeWarm { + t.Errorf("checkpointAttemptsSkippedWarm fired on Cold CFS — gate is too eager (delta=%v)", afterWarm-beforeWarm) + } + got, _ := dyn.Resource(CFSResource).Get(context.Background(), "fv-1", metav1.GetOptions{}) + if state := readStatus(got).LocalCacheState; state != nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("CFS not transitioned to Warm; got %q", state) + } +} + +func TestReconcileRecordsFailureOnCreateError(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-1") + + stub := &nvsnapServerStub{failAt: "create"} + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + // MUST return nil. Returning the error would re-queue via + // controller.go:255 (AddRateLimited) and trigger the nvsnap-h100-a + // 2026-06-03 retry storm. recordFailure swallows + counts + logs. + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile must return nil to suppress controller-runtime requeue; got %v", err) + } + + got, err := dyn.Resource(CFSResource).Get(context.Background(), "fv-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + s := readStatus(got) + if s.AttemptCount != 1 { + t.Errorf("AttemptCount = %d, want 1 (first failure)", s.AttemptCount) + } + if !strings.Contains(s.LastError, "nvsnap CreateCheckpoint") { + t.Errorf("LastError = %q, want CreateCheckpoint mention", s.LastError) + } + if s.LocalCacheState != "" && s.LocalCacheState != nvsnapv1alpha1.LocalCacheStateCold { + t.Errorf("LocalCacheState = %q, want empty or Cold (failure preserves prior state)", s.LocalCacheState) + } + + // Annotation stays so a future pod-ready event (e.g., a sibling + // pod for the same function) can reconsider. We deliberately do + // NOT requeue THIS work item — the next legitimate event will. + // Same pattern as the nvca#15 empty-hash path. + updatedPod, _ := r.KubeClient.CoreV1().Pods("ns1").Get(context.Background(), "p1", metav1.GetOptions{}) + if updatedPod.Annotations[CheckpointOnWarmAnnotation] != "true" { + t.Error("annotation should remain on failure so next pod-ready event can reconsider (no auto-retry, but eligible for re-attempt)") + } +} + +func TestReconcileMissingFunctionVersionAnnotation(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "p1", + Namespace: "ns1", + Annotations: map[string]string{ + CheckpointOnWarmAnnotation: "true", + // no FunctionVersionIDAnnotation + }, + }, + } + nvsnapSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("nvsnap-server should NOT be called when FV annotation is missing") + })) + defer nvsnapSrv.Close() + r := newTestReconciler(t, pod, nvsnapSrv, newFakeDynamic()) + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile should silently skip, got %v", err) + } +} + +func TestReconcileSkipsWhenAnnotationAbsent(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p1", Namespace: "ns1"}} + nvsnapSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.NotFound(w, nil) + })) + defer nvsnapSrv.Close() + r := newTestReconciler(t, pod, nvsnapSrv, newFakeDynamic()) + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile should be a no-op without the annotation: %v", err) + } +} + +// TestReconcileSkipsWhenNotReady covers the defensive guard added when +// Hook B was refactored to trust K8s PodReady instead of self-polling +// /health. The controller already enqueues only Ready pods, but a +// direct Reconcile() call with a not-Ready pod must early-return +// cleanly instead of barreling into a checkpoint POST. +func TestReconcileSkipsWhenNotReady(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-1") + // Force NotReady. + pod.Status.Conditions = []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionFalse}, + } + nvsnapSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + t.Errorf("nvsnap-server must NOT be called when pod is not Ready") + http.NotFound(w, nil) + })) + defer nvsnapSrv.Close() + r := newTestReconciler(t, pod, nvsnapSrv, newFakeDynamic()) + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile should be a no-op when pod not Ready: %v", err) + } +} + +// TestRecordFailurePreservesCapturedAt is a regression test for +// Greptile P1 on MR !1698. writeStatus does a per-key patch (the +// fix landed in the same commit), but recordFailure must still +// forward prev.CapturedAt explicitly so an operator debugging a +// CFS that was once Warm sees the original capture timestamp after +// a failed re-checkpoint attempt. +func TestRecordFailurePreservesCapturedAt(t *testing.T) { + captured := metav1.NewTime(time.Date(2026, 5, 1, 10, 0, 0, 0, time.UTC)) + seed := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": "fv-failtest"}, + "spec": map[string]any{"functionVersionID": "fv-failtest"}, + "status": map[string]any{ + "checkpointHash": "abc123", + "capturedHere": true, + "capturedAt": captured.UTC().Format(time.RFC3339), + "localCacheState": string(nvsnapv1alpha1.LocalCacheStateWarm), + "attemptCount": int64(1), + }, + }} + dyn := newFakeDynamic(seed) + r := &Reconciler{DynClient: dyn} + + prev := cfsStatus{ + CheckpointHash: "abc123", + CapturedHere: true, + CapturedAt: &captured, + LocalCacheState: nvsnapv1alpha1.LocalCacheStateWarm, + AttemptCount: 1, + } + _ = r.recordFailure(context.Background(), "fv-failtest", "completed_failed", prev, + fmtErrorf("simulated"), logrus.NewEntry(logrus.New())) + + got, err := dyn.Resource(CFSResource).Get(context.Background(), "fv-failtest", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + gotCapturedAt, found, _ := unstructured.NestedString(got.Object, "status", "capturedAt") + if !found || gotCapturedAt == "" { + t.Fatal("CapturedAt was dropped by recordFailure (Greptile P1 regression)") + } + if gotCapturedAt != captured.UTC().Format(time.RFC3339) { + t.Errorf("CapturedAt drifted: got %q, want %q", gotCapturedAt, captured.UTC().Format(time.RFC3339)) + } + // AttemptCount should have bumped. + gotAttempt, _, _ := unstructured.NestedInt64(got.Object, "status", "attemptCount") + if gotAttempt != 2 { + t.Errorf("AttemptCount = %d, want 2 (1 prev + 1 from this failure)", gotAttempt) + } +} + +// TestWriteStatusPreservesUnmanagedKeys is a regression test for +// Greptile P1 on MR !1698. writeStatus must NOT replace the whole +// status subtree — it must merge by key so unmanaged fields like +// `conditions` (which other controllers or future revisions of +// this one may set) survive a status update. +func TestWriteStatusPreservesUnmanagedKeys(t *testing.T) { + preExistingConditions := []any{ + map[string]any{ + "type": "Ready", + "status": "True", + }, + } + seed := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": "fv-conditions"}, + "spec": map[string]any{"functionVersionID": "fv-conditions"}, + "status": map[string]any{ + "conditions": preExistingConditions, + // Plus a key writeStatus never sets — verify it survives too. + "customOperatorField": "must-survive", + }, + }} + dyn := newFakeDynamic(seed) + + if err := writeStatus(context.Background(), dyn, "fv-conditions", statusUpdate{ + CheckpointHash: "deadbeef", + LocalCacheState: nvsnapv1alpha1.LocalCacheStateWarm, + }); err != nil { + t.Fatalf("writeStatus: %v", err) + } + + got, err := dyn.Resource(CFSResource).Get(context.Background(), "fv-conditions", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get: %v", err) + } + // Managed key landed. + gotHash, _, _ := unstructured.NestedString(got.Object, "status", "checkpointHash") + if gotHash != "deadbeef" { + t.Errorf("checkpointHash = %q, want deadbeef", gotHash) + } + // Unmanaged conditions preserved. + gotConditions, found, _ := unstructured.NestedSlice(got.Object, "status", "conditions") + if !found || len(gotConditions) != 1 { + t.Fatal("status.conditions was wiped by writeStatus (Greptile P1 regression)") + } + cond := gotConditions[0].(map[string]any) + if cond["type"] != "Ready" { + t.Errorf("condition type = %v, want Ready", cond["type"]) + } + // Arbitrary unmanaged key preserved. + gotCustom, _, _ := unstructured.NestedString(got.Object, "status", "customOperatorField") + if gotCustom != "must-survive" { + t.Errorf("customOperatorField = %q, want must-survive (unmanaged keys must survive a writeStatus call)", gotCustom) + } +} + +// fmtErrorf is a tiny helper to keep the import set lean (avoid +// pulling in fmt just for fmt.Errorf in one test). +func fmtErrorf(s string) error { return &simpleErr{s} } + +type simpleErr struct{ s string } + +func (e *simpleErr) Error() string { return e.s } + +// recordFailure MUST return nil (not the cause) so controller-runtime +// drops the work item instead of requeueing. Returning the error +// triggers the retry-storm pattern that wedged nvsnap-h100-a +// 2026-06-03 (3 back-to-back 80 GB captures from a single L2 timeout). +// Regression test against that behavior. +func TestRecordFailureReturnsNil_NoRequeue(t *testing.T) { + seed := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": "fv-noretry"}, + "spec": map[string]any{"functionVersionID": "fv-noretry"}, + "status": map[string]any{"attemptCount": int64(0)}, + }} + dyn := newFakeDynamic(seed) + r := &Reconciler{DynClient: dyn} + + got := r.recordFailure(context.Background(), "fv-noretry", "completed_failed", + cfsStatus{}, fmtErrorf("simulated capture failure"), logrus.NewEntry(logrus.New())) + if got != nil { + t.Errorf("recordFailure returned %v; want nil so controller-runtime DOES NOT requeue (nvsnap-h100-a 2026-06-03 retry storm)", got) + } +} + +// Each reason swallows independently and bumps the metric with the +// right label so the operator can tell which failure stage fired. +func TestRecordFailureBumpsCounterByReason(t *testing.T) { + for _, reason := range []string{"create_failed", "poll_failed", "completed_failed"} { + t.Run(reason, func(t *testing.T) { + seed := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": "fv-" + reason}, + "spec": map[string]any{"functionVersionID": "fv-" + reason}, + "status": map[string]any{"attemptCount": int64(0)}, + }} + dyn := newFakeDynamic(seed) + r := &Reconciler{DynClient: dyn} + before := testutil.ToFloat64(checkpointAttemptFailures.WithLabelValues(reason)) + + err := r.recordFailure(context.Background(), "fv-"+reason, reason, + cfsStatus{}, fmtErrorf("simulated"), logrus.NewEntry(logrus.New())) + if err != nil { + t.Fatalf("recordFailure returned %v; must be nil", err) + } + after := testutil.ToFloat64(checkpointAttemptFailures.WithLabelValues(reason)) + if after-before != 1 { + t.Errorf("checkpointAttemptFailures{reason=%s} delta = %v, want 1", reason, after-before) + } + }) + } +} + +// TestIsHealthyDrainsBody was removed alongside the Hook B +// /health poll. The earlier Greptile P2 fix (drain body before close) +// applied to isHealthy(); after the refactor to trust K8s PodReady, +// that function — and the body-drain concern — no longer exists. + +// ─── nvca#179: L2 promote-state gate ───────────────────────────────────── + +// TestReconcileL2PromoteReady_FlipsWarm — happy path: checkpoint +// reaches Completed, /pvc-state returns "ready" immediately, CFS +// flips to Warm. This is the case that eliminates the need for +// nvsnap-l2-wait (#179 design). +func TestReconcileL2PromoteReady_FlipsWarm(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-179a") + + stub := &nvsnapServerStub{ + pollsBeforeCompleted: 2, + hash: "abcdef0123456789", + size: 12345, + // promoteTerminalState empty defaults to "ready" + // promotePollsBeforeTerminal=0 → first poll terminal + } + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + got, err := dyn.Resource(CFSResource).Get(context.Background(), "fv-179a", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + s := readStatus(got) + if s.LocalCacheState != nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("LocalCacheState = %q, want Warm (L2 ready → flip)", s.LocalCacheState) + } + if s.CheckpointHash != "abcdef0123456789" { + t.Errorf("hash = %q, want abcdef0123456789", s.CheckpointHash) + } + if atomic.LoadInt32(&stub.promotePolls) == 0 { + t.Errorf("reconciler must have polled /pvc-state at least once") + } +} + +// TestReconcileL2PromoteFailed_RecordsFailure — terminal failure +// path: /pvc-state returns "failed". CFS must NOT flip to Warm; the +// reconciler must record a promote_failed failure so future pods +// cold-start instead of getting nvsnap.io/restore-from with a broken +// PVC reference. +func TestReconcileL2PromoteFailed_RecordsFailure(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-179b") + + stub := &nvsnapServerStub{ + pollsBeforeCompleted: 2, + hash: "deadbeef00000000", + size: 12345, + promoteTerminalState: "failed", + } + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + // recordFailure returns nil — reconciler swallows so controller- + // runtime doesn't requeue (nvca#15 / #167 pattern). + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile must swallow promote-failed; got %v", err) + } + + got, err := dyn.Resource(CFSResource).Get(context.Background(), "fv-179b", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + s := readStatus(got) + if s.LocalCacheState == nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("LocalCacheState = Warm despite L2 promote=failed; should stay Cold") + } + if s.AttemptCount == 0 { + t.Errorf("AttemptCount = 0; promote_failed should bump it for backoff") + } + if s.LastError == "" { + t.Errorf("LastError empty; promote_failed should populate it") + } +} + +// TestReconcileL2PromoteNoBackend_FlipsWarm — agent has no L2 +// backend configured. nvsnap-server's /pvc-state returns 404 (catalog +// row exists but no L2 columns). Reconciler treats this as "no L2, +// proceed via peer cascade" and flips CFS=Warm. +func TestReconcileL2PromoteNoBackend_FlipsWarm(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-179c") + + stub := &nvsnapServerStub{ + pollsBeforeCompleted: 2, + hash: "cafebabe00000000", + size: 12345, + promote404: true, + } + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + got, err := dyn.Resource(CFSResource).Get(context.Background(), "fv-179c", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + s := readStatus(got) + if s.LocalCacheState != nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("LocalCacheState = %q, want Warm (no-L2 fallback)", s.LocalCacheState) + } +} + +// TestReconcileL2PromoteTimeout_RecordsFailure — /pvc-state never +// reaches terminal within PromotePollTimeout. The CRIU checkpoint +// itself was a success, but L2 stalled. Reconciler records a +// promote_poll_failed failure (CFS stays Cold; future pods cold-start) +// and returns nil so controller-runtime doesn't requeue. +func TestReconcileL2PromoteTimeout_RecordsFailure(t *testing.T) { + pod := inferencePod("p1", "ns1", "fv-179d") + + stub := &nvsnapServerStub{ + pollsBeforeCompleted: 2, + hash: "feedface00000000", + size: 12345, + // Never reach terminal — first 10000 polls return "pending" + promotePollsBeforeTerminal: 10000, + promoteTerminalState: "ready", + } + nvsnapSrv := httptest.NewServer(stub.handler()) + defer nvsnapSrv.Close() + + dyn := newFakeDynamic() + r := newTestReconciler(t, pod, nvsnapSrv, dyn) + + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile must swallow promote-timeout; got %v", err) + } + + got, err := dyn.Resource(CFSResource).Get(context.Background(), "fv-179d", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + s := readStatus(got) + if s.LocalCacheState == nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("LocalCacheState = Warm despite L2 promote timeout; should stay Cold") + } + if s.LastError == "" || !strings.Contains(s.LastError, "L2 promote poll") { + t.Errorf("LastError = %q, want one mentioning L2 promote poll", s.LastError) + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/recover.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/recover.go new file mode 100644 index 000000000..a0c56bb73 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/recover.go @@ -0,0 +1,161 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// recover.go — the nvca#104 recovery short-circuit for Hook B. +// +// The happy-path Warm flip is reachable only by ONE uninterrupted +// reconcile: warmup → POST → pollCheckpointTerminal → pollPVCPromoteTerminal +// → writeStatus(Warm). pollPVCPromoteTerminal can block for minutes +// (Hyperdisk-ML snap+clone). If that reconcile is interrupted AFTER the +// capture succeeds but BEFORE writeStatus — controller restart, leader +// election change, ctx cancel, the source pod being redeployed, or an +// earlier transient error that returned — CFS stays not-Warm. The only +// existing short-circuit is "already Warm", so the next reconcile would +// re-capture from scratch (and, in the worst case where nothing +// re-triggers it, CFS never goes Warm → silent cold start). +// +// recoverExistingCapture closes that gap: before POSTing a fresh +// capture, ask nvsnap-server (the same content-addressed lookup Hook A +// uses for restore) whether a usable capture already exists for this +// pod's workload identity. If so, the caller flips CFS=Warm directly — +// idempotent, no re-dump. This also closes the duplicate-capture race +// (nvca#178/#189): a capture that already exists is never re-taken. + +package reconciler + +import ( + "context" + "errors" + "net/http" + "strings" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap" +) + +// recoverExistingCapture returns the content hash of a usable existing +// capture for this pod's workload identity, or "" if none exists or +// none is usable yet (so the caller proceeds with the normal capture +// flow). Best-effort: any lookup error returns "" — recovery never +// blocks a fresh capture. +func (r *Reconciler) recoverExistingCapture(ctx context.Context, pod *corev1.Pod, log logrus.FieldLogger) string { + c := findInferenceContainer(pod, r.InferenceContainerName) + if c == nil || c.Image == "" { + return "" + } + return r.findUsableCapture(ctx, c.Image, extractModelID(c), log) +} + +// findUsableCapture asks nvsnap-server whether a usable capture exists for +// the given workload identity (image + model) and returns its content +// hash, or "" if none exists or none is usable yet. Best-effort: any +// lookup error returns "". Shared by the pod-triggered recovery +// short-circuit (recoverExistingCapture) and the controller's +// pod-independent CFS sweep (SweepOnce) so both use one definition of +// "usable". +func (r *Reconciler) findUsableCapture(ctx context.Context, imageRef, modelID string, log logrus.FieldLogger) string { + if imageRef == "" { + return "" + } + resp, err := r.NvSnapClient.LookupCheckpoints(ctx, nvsnap.LookupRequest{ + ImageRef: imageRef, + ModelID: modelID, + }) + if err != nil { + log.WithError(err).Debug("recovery: lookup failed; proceeding with normal capture flow") + return "" + } + if resp == nil || len(resp.Matches) == 0 { + return "" + } + // Matches are freshest-first. Take the newest whose L2 promote is + // terminal-usable: "ready" (rox bound) or no-L2 (404 / "" — the + // capture lives on L1 / peer cascade and Hook A restores via the + // fallback path). A "failed" or still-in-progress promote is NOT + // recoverable here — leave it to the normal flow. + for _, m := range resp.Matches { + if m.Hash == "" { + continue + } + ps, err := r.NvSnapClient.GetPVCPromoteState(ctx, m.Hash) + if err != nil { + var apiErr *nvsnap.APIError + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound { + return m.Hash // no L2 row → usable via L1/peer cascade + } + log.WithError(err).WithField("hash", m.Hash). + Debug("recovery: pvc-state check failed; skipping this match") + continue + } + switch ps.State { + case "ready", "": + return m.Hash + default: // "failed" | "pending" | "writing" | "snapshotting" + continue + } + } + return "" +} + +// findInferenceContainer returns the container named `name`, falling +// back to the first GPU-requesting container, then the first. Mirrors +// pkg/nvca/nvsnap_hook.go::inferenceContainer — duplicated here to avoid +// an import cycle with package nvca. +func findInferenceContainer(pod *corev1.Pod, name string) *corev1.Container { + if pod == nil { + return nil + } + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == name { + return &pod.Spec.Containers[i] + } + } + for i := range pod.Spec.Containers { + if _, ok := pod.Spec.Containers[i].Resources.Limits["nvidia.com/gpu"]; ok { + return &pod.Spec.Containers[i] + } + } + if len(pod.Spec.Containers) > 0 { + return &pod.Spec.Containers[0] + } + return nil +} + +// extractModelID inspects the container's env + args for the well-known +// model-identifier keys. Mirrors pkg/nvca/nvsnap_hook.go::extractModelID +// (same import-cycle reason). +func extractModelID(c *corev1.Container) string { + for _, e := range c.Env { + switch e.Name { + case "NIM_MODEL_NAME", "MODEL_NAME", "MODEL_REPO", "HF_MODEL_ID", "HUGGINGFACE_MODEL": + if e.Value != "" { + return e.Value + } + } + } + for i, arg := range c.Args { + if (arg == "--model" || arg == "--model-path") && i+1 < len(c.Args) { + return c.Args[i+1] + } + if strings.HasPrefix(arg, "--model=") { + return strings.TrimPrefix(arg, "--model=") + } + if strings.HasPrefix(arg, "--model-path=") { + return strings.TrimPrefix(arg, "--model-path=") + } + } + return "" +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/recover_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/recover_test.go new file mode 100644 index 000000000..c839d901f --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/recover_test.go @@ -0,0 +1,131 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package reconciler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" +) + +// recoverablePod is inferencePod + an image (the lookup key) on the +// inference container. +func recoverablePod(podName, ns, fvID, image string) *corev1.Pod { + p := inferencePod(podName, ns, fvID) + p.Spec.Containers[0].Image = image + return p +} + +// coldCFS seeds a NvSnapFunctionState that exists but is NOT Warm. +func coldCFS(fvID string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": fvID}, + "spec": map[string]any{"functionVersionID": fvID}, + }} +} + +// nvca#104: if a usable capture already exists, the reconcile must flip +// CFS=Warm via the recovery short-circuit and must NOT POST a new +// capture (which would be the duplicate-capture re-dump). +func TestReconcileRecoversExistingCapture(t *testing.T) { + const fvID, image, hash = "fv-rec", "ngc.io/fn:1", "deadbeefcafe0001" + pod := recoverablePod("p-rec", "ns1", fvID, image) + dyn := newFakeDynamic(coldCFS(fvID)) + + var capturePosted bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/api/v1/checkpoints/lookup"): + _ = json.NewEncoder(w).Encode(map[string]any{ + "matches": []map[string]any{{"hash": hash, "checkpointId": "ck-rec", "imageRef": image}}, + }) + case strings.Contains(r.URL.Path, "/pvc-state"): + _ = json.NewEncoder(w).Encode(map[string]any{"hash": hash, "state": "ready", "pvc_name": "rox-deadbeef"}) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/api/v1/checkpoints"): + capturePosted = true // recovery should have prevented this + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]any{"id": "ck-new", "phase": "InProgress"}) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + r := newTestReconciler(t, pod, srv, dyn) + if err := r.Reconcile(context.Background(), pod); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + if capturePosted { + t.Error("recovery should NOT POST a new capture when a usable one already exists") + } + got, _ := dyn.Resource(CFSResource).Get(context.Background(), fvID, metav1.GetOptions{}) + s := readStatus(got) + if s.LocalCacheState != nvsnapv1alpha1.LocalCacheStateWarm { + t.Errorf("CFS not flipped Warm: got %q", s.LocalCacheState) + } + if s.CheckpointHash != hash { + t.Errorf("CheckpointHash: got %q, want %q", s.CheckpointHash, hash) + } +} + +// A "failed" L2 promote on the matched capture is NOT recoverable — +// the reconcile must fall through to the normal capture flow (POST). +func TestReconcileRecovery_SkipsFailedPromote(t *testing.T) { + const fvID, image, hash = "fv-rec2", "ngc.io/fn:2", "deadbeefcafe0002" + pod := recoverablePod("p-rec2", "ns1", fvID, image) + dyn := newFakeDynamic(coldCFS(fvID)) + + var capturePosted bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/api/v1/checkpoints/lookup"): + _ = json.NewEncoder(w).Encode(map[string]any{ + "matches": []map[string]any{{"hash": hash, "imageRef": image}}, + }) + case strings.Contains(r.URL.Path, "/pvc-state"): + _ = json.NewEncoder(w).Encode(map[string]any{"hash": hash, "state": "failed"}) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/api/v1/checkpoints"): + capturePosted = true + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]any{"id": "ck-new", "phase": "InProgress"}) + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/api/v1/checkpoints/"): + _ = json.NewEncoder(w).Encode(map[string]any{"id": "ck-new", "phase": "Completed", "hash": hash}) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + r := newTestReconciler(t, pod, srv, dyn) + // promote poll for the fresh capture would block; the new capture's + // pvc-state also returns "failed", which the normal flow records as + // a failure (no Warm). We only assert recovery did NOT short-circuit. + _ = r.Reconcile(context.Background(), pod) + if !capturePosted { + t.Error("a failed-promote match must NOT be recovered; reconcile should fall through to a fresh capture POST") + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/state.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/state.go new file mode 100644 index 000000000..dc9524ab6 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/state.go @@ -0,0 +1,473 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package reconciler + +import ( + "context" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" +) + +// CFSResource is the GroupVersionResource the reconciler reads/writes +// on. Lives here so the reconciler and Hook A share one source of +// truth. +var CFSResource = schema.GroupVersionResource{ + Group: nvsnapv1alpha1.SchemeGroupVersion.Group, + Version: nvsnapv1alpha1.SchemeGroupVersion.Version, + Resource: "nvsnapfunctionstates", +} + +// cfsStatus captures the fields the reconciler reads from /writes to +// status on NvSnapFunctionState. Using a struct rather than poking +// unstructured.NestedX directly at every call site keeps the state +// machine in reconciler.go readable. +type cfsStatus struct { + CheckpointHash string + CapturedHere bool + LocalCacheState nvsnapv1alpha1.NvSnapFunctionStateLocalCacheState + AttemptCount int64 + LastError string + CapturedAt *metav1.Time + // LastAttemptAt is the wall-clock time of the most recent + // CheckpointRequest the reconciler issued for this CFS — set on + // both successful and failed terminal attempts. Used by the + // reconciler's per-function backoff gate (nvca#167) to suppress + // re-attempts on pod-ready events that arrive too quickly after + // a previous failure. nil = never attempted. + LastAttemptAt *metav1.Time + // CaptureOwner / CaptureLeaseExpiry back the capture-once claim + // (nvca#189). CaptureOwner is the namespace/name of the pod holding + // an in-flight Capturing claim; CaptureLeaseExpiry bounds it for + // crash recovery. Both zero-valued when not Capturing. + CaptureOwner string + CaptureLeaseExpiry *metav1.Time + // ColdStartPioneer / ColdStartPioneerExpiry back the serialized-herd + // cold-start pioneer election. ColdStartPioneer is the namespace/name + // of the ICMSRequest that won the right to cold-start + capture while + // its peers defer; ColdStartPioneerExpiry bounds it for crash + // recovery. Both zero-valued when no pioneer is elected. Analogous to + // CaptureOwner/CaptureLeaseExpiry but claimed one step earlier (at + // MiniService creation, not at capture). + ColdStartPioneer string + ColdStartPioneerExpiry *metav1.Time +} + +// getOrCreateCFS reads NvSnapFunctionState/. If missing, creates +// an empty one (cold state) and returns it. The reconciler needs the +// object to exist so it can update status; this is the standard +// "controller materializes its CR on first touch" pattern. +func getOrCreateCFS(ctx context.Context, dc dynamic.Interface, fvID string) (*unstructured.Unstructured, error) { + cfs, err := dc.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if err == nil { + return cfs, nil + } + if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("get NvSnapFunctionState %s: %w", fvID, err) + } + // Create with minimal spec; Hook B will populate status as it + // progresses through Cold → Fetching → Warm. + fresh := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": fvID}, + "spec": map[string]any{ + "functionVersionID": fvID, + }, + "status": map[string]any{ + "localCacheState": string(nvsnapv1alpha1.LocalCacheStateCold), + }, + }, + } + created, err := dc.Resource(CFSResource).Create(ctx, fresh, metav1.CreateOptions{}) + if err != nil { + // If somebody else created it between our Get and Create + // (race with a parallel reconciler), re-read instead of + // failing — the state we want exists, just not from us. + if apierrors.IsAlreadyExists(err) { + return dc.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + } + return nil, fmt.Errorf("create NvSnapFunctionState %s: %w", fvID, err) + } + return created, nil +} + +// readStatus extracts the typed status fields from the unstructured +// object. Unset fields return their zero value; the reconciler +// already treats those correctly (empty hash = no checkpoint yet, +// empty state = Cold, etc.). +func readStatus(cfs *unstructured.Unstructured) cfsStatus { + var out cfsStatus + if cfs == nil { + return out + } + out.CheckpointHash, _, _ = unstructured.NestedString(cfs.Object, "status", "checkpointHash") + out.CapturedHere, _, _ = unstructured.NestedBool(cfs.Object, "status", "capturedHere") + stateStr, _, _ := unstructured.NestedString(cfs.Object, "status", "localCacheState") + out.LocalCacheState = nvsnapv1alpha1.NvSnapFunctionStateLocalCacheState(stateStr) + out.AttemptCount, _, _ = unstructured.NestedInt64(cfs.Object, "status", "attemptCount") + out.LastError, _, _ = unstructured.NestedString(cfs.Object, "status", "lastError") + if rfc, found, _ := unstructured.NestedString(cfs.Object, "status", "capturedAt"); found { + if t, err := time.Parse(time.RFC3339, rfc); err == nil { + mt := metav1.NewTime(t) + out.CapturedAt = &mt + } + } + if rfc, found, _ := unstructured.NestedString(cfs.Object, "status", "lastAttemptAt"); found { + if t, err := time.Parse(time.RFC3339, rfc); err == nil { + mt := metav1.NewTime(t) + out.LastAttemptAt = &mt + } + } + out.CaptureOwner, _, _ = unstructured.NestedString(cfs.Object, "status", "captureOwner") + if rfc, found, _ := unstructured.NestedString(cfs.Object, "status", "captureLeaseExpiry"); found { + if t, err := time.Parse(time.RFC3339, rfc); err == nil { + mt := metav1.NewTime(t) + out.CaptureLeaseExpiry = &mt + } + } + out.ColdStartPioneer, _, _ = unstructured.NestedString(cfs.Object, "status", "coldStartPioneer") + if rfc, found, _ := unstructured.NestedString(cfs.Object, "status", "coldStartPioneerExpiry"); found { + if t, err := time.Parse(time.RFC3339, rfc); err == nil { + mt := metav1.NewTime(t) + out.ColdStartPioneerExpiry = &mt + } + } + return out +} + +// optedOut returns true iff spec.optOut is set on the CR. +func optedOut(cfs *unstructured.Unstructured) bool { + if cfs == nil { + return false + } + v, _, _ := unstructured.NestedBool(cfs.Object, "spec", "optOut") + return v +} + +// readWorkloadLookup extracts spec.workloadLookup. Empty imageRef means +// it was never persisted (the CFS predates this writer, or Hook B died +// before reaching the persist step) — the sweep skips such CFS. +func readWorkloadLookup(cfs *unstructured.Unstructured) (imageRef, modelID string) { + if cfs == nil { + return "", "" + } + imageRef, _, _ = unstructured.NestedString(cfs.Object, "spec", "workloadLookup", "imageRef") + modelID, _, _ = unstructured.NestedString(cfs.Object, "spec", "workloadLookup", "modelId") + return imageRef, modelID +} + +// writeWorkloadLookup persists spec.workloadLookup so the controller's +// CFS recovery sweep can look up an existing capture without a live pod +// (nvca#104 durable-warm). No-op when imageRef is empty or the value is +// already current — avoids a needless write on every reconcile. Writes +// the spec subtree via Update (not UpdateStatus); only the +// workloadLookup keys are touched, so a concurrent optOut toggle on a +// different key survives the read-modify-write. +func writeWorkloadLookup(ctx context.Context, dc dynamic.Interface, fvID, imageRef, modelID string) error { + if imageRef == "" { + return nil + } + cur, err := dc.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get NvSnapFunctionState %s: %w", fvID, err) + } + curImage, _, _ := unstructured.NestedString(cur.Object, "spec", "workloadLookup", "imageRef") + curModel, _, _ := unstructured.NestedString(cur.Object, "spec", "workloadLookup", "modelId") + if curImage == imageRef && curModel == modelID { + return nil // already current + } + lookup := map[string]any{"imageRef": imageRef} + if modelID != "" { + lookup["modelId"] = modelID + } + if err := unstructured.SetNestedMap(cur.Object, lookup, "spec", "workloadLookup"); err != nil { + return fmt.Errorf("set spec.workloadLookup: %w", err) + } + if _, err := dc.Resource(CFSResource).Update(ctx, cur, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("update NvSnapFunctionState %s spec: %w", fvID, err) + } + return nil +} + +// statusUpdate is what writeStatus marshals onto the CR. Only the +// fields the reconciler actually writes — leaves everything else +// (Conditions, LastAttemptAt) untouched so future writers can extend. +type statusUpdate struct { + CheckpointHash string + CapturedHere bool + CapturedAt time.Time // zero = don't set + LocalCacheState nvsnapv1alpha1.NvSnapFunctionStateLocalCacheState + AttemptCount int64 + LastError string + LastAttemptAt time.Time // zero = don't set +} + +// writeStatus patches the status subresource via the dynamic client's +// UpdateStatus. NvSnapFunctionState exposes /status (kubebuilder marker +// status:status added in a follow-up CRD manifest PR); until that +// lands, Update is the only path. +// +// Known TOCTOU on the Update fallback (Greptile P2 on nvca!1748): if +// the operator toggles spec.optOut between our Get and our Update, +// our Update will overwrite spec back to the pre-toggle value. The +// fallback is only taken when the CRD doesn't have its /status +// subresource registered yet — once the follow-up CRD-manifest PR +// lands (registering subresource:status), UpdateStatus succeeds and +// only the /status subtree is written, eliminating the race. +// +// We can't conditionally skip the fallback today because the +// in-tree CRD manifest used in tests + nvsnap-h100-a doesn't have +// /status, and the alternative — failing every status write — +// would break Hook B entirely. Tracking removal: nvca-nvsnap +// status-subresource-registration follow-up. +func writeStatus(ctx context.Context, dc dynamic.Interface, fvID string, upd statusUpdate) error { + cur, err := dc.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get NvSnapFunctionState %s: %w", fvID, err) + } + // Read the existing status map and patch only the keys we manage. + // SetNestedField(...,"status") would otherwise replace the whole + // subtree, silently wiping unmanaged keys like `conditions` that + // other controllers (or future versions of this one) may write. + status, _, _ := unstructured.NestedMap(cur.Object, "status") + if status == nil { + status = map[string]any{} + } + status["checkpointHash"] = upd.CheckpointHash + status["capturedHere"] = upd.CapturedHere + status["localCacheState"] = string(upd.LocalCacheState) + status["attemptCount"] = upd.AttemptCount + status["lastError"] = upd.LastError + if !upd.CapturedAt.IsZero() { + status["capturedAt"] = upd.CapturedAt.UTC().Format(time.RFC3339) + } + if !upd.LastAttemptAt.IsZero() { + status["lastAttemptAt"] = upd.LastAttemptAt.UTC().Format(time.RFC3339) + } + // Release the capture-once claim (nvca#189). writeStatus is only + // called on terminal transitions (Warm on success, or back to the + // prior state on failure) — in both cases the in-flight capture is + // over, so the Capturing claim must be dropped. Leaving these set + // would either pin the function until lease expiry or let readStatus + // mis-report a stale owner. + delete(status, "captureOwner") + delete(status, "captureLeaseExpiry") + // Release the cold-start pioneer claim too (serialized-herd). A + // terminal status write means the pioneer's cold-start + capture is + // over: on success the function is now Warm (deferred replicas + // proceed and warm-restore), on failure it falls back to a + // non-Warm state where a fresh pioneer should be electable. Either + // way the slot must be vacated so it doesn't linger past its purpose. + delete(status, "coldStartPioneer") + delete(status, "coldStartPioneerExpiry") + if err := unstructured.SetNestedField(cur.Object, status, "status"); err != nil { + return fmt.Errorf("set status: %w", err) + } + if _, err := dc.Resource(CFSResource).UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err != nil { + // Some test envs (and CRDs without status subresource) reject + // UpdateStatus — fall back to Update for those. + if apierrors.IsNotFound(err) || apierrors.IsMethodNotSupported(err) { + if _, err := dc.Resource(CFSResource).Update(ctx, cur, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("update NvSnapFunctionState %s: %w", fvID, err) + } + return nil + } + return fmt.Errorf("update status %s: %w", fvID, err) + } + return nil +} + +// tryClaimCapture is the capture-once / thundering-herd guard +// (nvca#189). It atomically transitions a function-version's CFS to +// LocalCacheStateCapturing — claiming the right to run the single +// in-flight checkpoint — using Kubernetes optimistic concurrency: the +// object Get carries a resourceVersion, and the subsequent Update is +// admitted by the API server only if no one else mutated the object in +// between. When N pods of the same function-version reconcile cold at +// once, exactly one Update succeeds; the rest get a 409 Conflict and +// observe claimed=false. No double-capture. +// +// Claimability (evaluated against the freshly-read object): +// - Warm / Fetching: NOT claimable — capture is unnecessary (Warm is +// handled by the reconciler's already-Warm gate before we get here; +// Fetching means cross-cluster replication owns the bytes). +// - Capturing held by ANOTHER owner with an unexpired lease: NOT +// claimable — that pod is mid-capture; back off. +// - everything else (Cold / Failed / empty / Capturing with an +// EXPIRED lease / Capturing already owned by us): claimable. The +// expired-lease case lets a fresh pod steal a crashed capturer's +// claim; the own-owner case makes a re-reconcile of the capturing +// pod re-entrant and refreshes (heartbeats) its lease. +// +// Returns (true, nil) iff this caller now owns the claim. (false, nil) +// means another caller owns it (lost the race or unexpired foreign +// claim) — a non-error, expected outcome under concurrency. A non-nil +// error is a real API failure the caller should surface/requeue on. +// +// owner is the claiming pod's namespace/name; leaseExpiry is the +// wall-clock deadline to stamp; now is injected for deterministic +// tests. +func tryClaimCapture(ctx context.Context, dc dynamic.Interface, fvID, owner string, leaseExpiry, now time.Time) (bool, error) { + cur, err := dc.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("get NvSnapFunctionState %s: %w", fvID, err) + } + st := readStatus(cur) + + switch st.LocalCacheState { + case nvsnapv1alpha1.LocalCacheStateWarm, nvsnapv1alpha1.LocalCacheStateFetching: + return false, nil + case nvsnapv1alpha1.LocalCacheStateCapturing: + leaseLive := st.CaptureLeaseExpiry != nil && now.Before(st.CaptureLeaseExpiry.Time) + if leaseLive && st.CaptureOwner != owner { + return false, nil // another pod holds a live claim + } + // expired lease (steal) or our own claim (re-entrant) → fall through + } + + // Patch only the keys we manage; preserve checkpointHash/attemptCount/etc. + status, _, _ := unstructured.NestedMap(cur.Object, "status") + if status == nil { + status = map[string]any{} + } + status["localCacheState"] = string(nvsnapv1alpha1.LocalCacheStateCapturing) + status["captureOwner"] = owner + status["captureLeaseExpiry"] = leaseExpiry.UTC().Format(time.RFC3339) + if err := unstructured.SetNestedField(cur.Object, status, "status"); err != nil { + return false, fmt.Errorf("set status: %w", err) + } + + // Update carries cur's resourceVersion → optimistic concurrency. + // A concurrent winner bumps the resourceVersion, so our Update + // returns Conflict — that's the lost-race signal, not an error. + if _, err := dc.Resource(CFSResource).UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err != nil { + if apierrors.IsNotFound(err) || apierrors.IsMethodNotSupported(err) { + if _, err := dc.Resource(CFSResource).Update(ctx, cur, metav1.UpdateOptions{}); err != nil { + if apierrors.IsConflict(err) { + return false, nil + } + return false, fmt.Errorf("claim update NvSnapFunctionState %s: %w", fvID, err) + } + return true, nil + } + if apierrors.IsConflict(err) { + return false, nil + } + return false, fmt.Errorf("claim updateStatus %s: %w", fvID, err) + } + return true, nil +} + +// TryClaimColdStartPioneer is the serialized-herd cold-start guard +// (pioneer election). It is the exact analogue of tryClaimCapture +// (nvca#189) applied one step earlier — at MiniService creation rather +// than at capture: when N replicas of one function-version deploy cold +// (no Warm checkpoint yet), exactly one reconcile is allowed to claim +// the "pioneer" slot and cold-start; the others observe a live foreign +// claim and back off (the gate in pkg/nvca defers their MiniService +// creation). Once the pioneer's capture flips LocalCacheState=Warm the +// deferred replicas proceed and warm-restore via Hook A. +// +// Exported so the MiniService-creation gate (package nvca) can call it; +// the gate can't reach the unexported tryClaimCapture but the +// compare-and-swap mechanics are identical (the API server's optimistic +// concurrency on UpdateStatus elects exactly one winner under a herd). +// +// Claimability (evaluated against the freshly-read object): +// - Warm / Failed: NOT claimable, returns (false, nil). The caller +// treats both as "proceed" (Warm => warm-restore; Failed => +// fail-open cold) — not as a pioneer claim. Keeping the policy in +// the caller (rather than claiming here) lets the gate log the two +// cases distinctly. +// - ColdStartPioneer held by ANOTHER owner with an unexpired lease: +// NOT claimable — that replica is the pioneer; defer. +// - everything else (no claim / our own claim / expired lease / +// Cold / Fetching / Capturing / absent-status): claimable. The +// expired-lease case lets a fresh replica steal a crashed pioneer's +// slot so the function isn't deferred forever; the own-owner case +// makes a re-reconcile of the pioneer re-entrant (lease refresh). +// +// STRICT single pioneer (K=1) for v1. +// TODO(nvsnap): K>1 multi-pioneer slots — the MaxConcurrentColdStarts +// config knob exists but v1 honors K=1 only; multi-slot election needs +// a slot-count field on status, not a single owner string. +// +// Returns (true, nil) iff this caller now owns the pioneer slot. +// (false, nil) means proceed-or-defer per the cases above — a non-error +// outcome. A non-nil error is a real API failure to surface/requeue on. +// +// owner is the claiming ICMSRequest's namespace/name; leaseExpiry is the +// wall-clock deadline to stamp; now is injected for deterministic tests. +func TryClaimColdStartPioneer(ctx context.Context, dc dynamic.Interface, fvID, owner string, leaseExpiry, now time.Time) (bool, error) { + cur, err := dc.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("get NvSnapFunctionState %s: %w", fvID, err) + } + st := readStatus(cur) + + switch st.LocalCacheState { + case nvsnapv1alpha1.LocalCacheStateWarm, nvsnapv1alpha1.LocalCacheStateFailed: + // Caller proceeds (warm-restore or fail-open cold) — not a claim. + return false, nil + } + + leaseLive := st.ColdStartPioneerExpiry != nil && now.Before(st.ColdStartPioneerExpiry.Time) + if leaseLive && st.ColdStartPioneer != "" && st.ColdStartPioneer != owner { + return false, nil // another replica is the pioneer + } + // no claim / our own claim / expired lease → fall through and claim. + + // Patch only the keys we manage; preserve every other status field. + status, _, _ := unstructured.NestedMap(cur.Object, "status") + if status == nil { + status = map[string]any{} + } + status["coldStartPioneer"] = owner + status["coldStartPioneerExpiry"] = leaseExpiry.UTC().Format(time.RFC3339) + if err := unstructured.SetNestedField(cur.Object, status, "status"); err != nil { + return false, fmt.Errorf("set status: %w", err) + } + + // Update carries cur's resourceVersion → optimistic concurrency. A + // concurrent winner bumps the resourceVersion, so our Update returns + // Conflict — the lost-race signal, not an error. + if _, err := dc.Resource(CFSResource).UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err != nil { + if apierrors.IsNotFound(err) || apierrors.IsMethodNotSupported(err) { + if _, err := dc.Resource(CFSResource).Update(ctx, cur, metav1.UpdateOptions{}); err != nil { + if apierrors.IsConflict(err) { + return false, nil + } + return false, fmt.Errorf("pioneer claim update NvSnapFunctionState %s: %w", fvID, err) + } + return true, nil + } + if apierrors.IsConflict(err) { + return false, nil + } + return false, fmt.Errorf("pioneer claim updateStatus %s: %w", fvID, err) + } + return true, nil +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/sweep.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/sweep.go new file mode 100644 index 000000000..f239a2c0f --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/sweep.go @@ -0,0 +1,106 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +// sweep.go — the pod-independent CFS recovery sweep (nvca#104 +// durable-warm). See docs/users/nvsnap/DURABLE-WARM-SWEEP.md. +// +// The happy-path Warm flip and the recoverExistingCapture short-circuit +// both run inside a pod-triggered reconcile. If the reconcile that +// captured a checkpoint dies before writeStatus (controller restart, +// lost goroutine, source pod scaled away mid-poll) AND no new +// checkpoint-on-warm pod ever appears for that function-version — e.g. +// the next pod is a restore attempt, or NVCF scaled the function to +// zero — nothing re-evaluates the CFS and it stays not-Warm forever, +// silently cold-starting every restore. +// +// SweepOnce closes that gap: it reconciles every NvSnapFunctionState +// against nvsnap-server (the source of truth for "does a usable capture +// exist") with no dependence on a live pod, flipping Warm for any CFS +// whose capture has landed. + +package reconciler + +import ( + "context" + "time" + + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" +) + +// SweepOnce runs one pass of the CFS recovery sweep: list every +// NvSnapFunctionState and, for each that is not Warm, not opted out, and +// carries persisted workload-lookup inputs, ask nvsnap-server whether a +// usable capture exists and flip LocalCacheState=Warm if so. Idempotent +// and best-effort — list/lookup/write errors are logged and the sweep +// continues to the next CFS. Never returns an error (the controller +// just runs it again next tick). +func (r *Reconciler) SweepOnce(ctx context.Context) { + r.applyDefaults() + log := r.Log.WithField("component", "nvsnap-cfs-sweep") + + list, err := r.DynClient.Resource(CFSResource).List(ctx, metav1.ListOptions{}) + if err != nil { + log.WithError(err).Debug("sweep: list NvSnapFunctionState failed; will retry next tick") + return + } + + for i := range list.Items { + cfs := &list.Items[i] + fvID := cfs.GetName() + + st := readStatus(cfs) + if st.LocalCacheState == nvsnapv1alpha1.LocalCacheStateWarm { + continue // already Warm — nothing to recover + } + if optedOut(cfs) { + continue + } + imageRef, modelID := readWorkloadLookup(cfs) + if imageRef == "" { + continue // never persisted (CFS predates the writer, or Hook B + // died before persisting) — can't look up without inputs + } + + hash := r.findUsableCapture(ctx, imageRef, modelID, log) + if hash == "" { + continue // no usable capture yet — leave for a future tick + } + + now := time.Now() + if err := writeStatus(ctx, r.DynClient, fvID, statusUpdate{ + CheckpointHash: hash, + CapturedHere: true, + CapturedAt: now, + LocalCacheState: nvsnapv1alpha1.LocalCacheStateWarm, + AttemptCount: 0, + LastError: "", + LastAttemptAt: now, + }); err != nil { + log.WithError(err).WithField("functionVersionID", fvID). + Warn("sweep: writeStatus Warm failed; will retry next tick") + continue + } + cfsSweepRecovered.Inc() + log.WithFields(logrus.Fields{ + "functionVersionID": fvID, + "hash": hash, + "prev_state": st.LocalCacheState, + }).Info("sweep: recovered a usable capture and flipped CFS=Warm without a live pod (nvca#104 durable-warm)") + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/sweep_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/sweep_test.go new file mode 100644 index 000000000..69d901124 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler/sweep_test.go @@ -0,0 +1,230 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Tests for the pod-independent CFS recovery sweep (nvca#104 +// durable-warm). The sweep must flip a not-Warm CFS to Warm from an +// existing usable capture WITHOUT a live pod, and must skip CFS that +// are already Warm, opted out, have no persisted lookup inputs, or have +// no usable capture in nvsnap-server. + +package reconciler + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + fakedynamic "k8s.io/client-go/dynamic/fake" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap" +) + +// cfsWithLookup seeds a not-Warm CFS carrying persisted workload-lookup +// inputs (what Hook B writes before the risky poll). extra lets a test +// set additional spec/status keys (optOut, an already-Warm state). +func cfsWithLookup(fvID, image, model string, mutate func(obj map[string]any)) *unstructured.Unstructured { + obj := map[string]any{ + "apiVersion": "nvsnap.nvcf.nvidia.io/v1alpha1", + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": fvID}, + "spec": map[string]any{ + "functionVersionID": fvID, + "workloadLookup": map[string]any{"imageRef": image, "modelId": model}, + }, + } + if mutate != nil { + mutate(obj) + } + return &unstructured.Unstructured{Object: obj} +} + +// sweepReconciler builds a Reconciler pointed at srv + dyn, with no pod +// (the sweep never needs one). +func sweepReconciler(srv *httptest.Server, dyn *fakedynamic.FakeDynamicClient) *Reconciler { + return &Reconciler{ + DynClient: dyn, + NvSnapClient: nvsnap.NewClient(nvsnap.WithBaseURL(srv.URL)), + Log: logrus.NewEntry(logrus.New()), + } +} + +// usableCaptureServer answers lookup with one match whose pvc-state is +// "ready" — a recoverable capture. +func usableCaptureServer(image, hash string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/api/v1/checkpoints/lookup"): + _ = json.NewEncoder(w).Encode(map[string]any{ + "matches": []map[string]any{{"hash": hash, "checkpointId": "ck", "imageRef": image}}, + }) + case strings.Contains(r.URL.Path, "/pvc-state"): + _ = json.NewEncoder(w).Encode(map[string]any{"hash": hash, "state": "ready", "pvc_name": "rox-" + hash}) + default: + http.NotFound(w, r) + } + })) +} + +// The core fix: a not-Warm CFS with a usable capture in nvsnap-server is +// flipped to Warm by the sweep, no pod involved. +func TestSweep_FlipsWarmFromUsableCapture(t *testing.T) { + const fvID, image, hash = "fv-sweep", "ngc.io/fn:1", "deadbeefcafe1001" + srv := usableCaptureServer(image, hash) + defer srv.Close() + dyn := newFakeDynamic(cfsWithLookup(fvID, image, "", nil)) + + sweepReconciler(srv, dyn).SweepOnce(context.Background()) + + got, err := dyn.Resource(CFSResource).Get(context.Background(), fvID, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + s := readStatus(got) + if s.LocalCacheState != "Warm" { + t.Errorf("CFS not flipped Warm: got %q", s.LocalCacheState) + } + if s.CheckpointHash != hash { + t.Errorf("hash: got %q want %q", s.CheckpointHash, hash) + } + if !s.CapturedHere { + t.Error("capturedHere should be true (this cluster holds the L2 rox)") + } +} + +// An already-Warm CFS must be left untouched — and the sweep must not +// even hit nvsnap-server for it. +func TestSweep_SkipsAlreadyWarm(t *testing.T) { + const fvID, image = "fv-warm", "ngc.io/fn:2" + var lookupCalled bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + lookupCalled = true + http.NotFound(w, r) + })) + defer srv.Close() + dyn := newFakeDynamic(cfsWithLookup(fvID, image, "", func(obj map[string]any) { + obj["status"] = map[string]any{"localCacheState": "Warm", "checkpointHash": "existing"} + })) + + sweepReconciler(srv, dyn).SweepOnce(context.Background()) + + if lookupCalled { + t.Error("sweep must not query nvsnap-server for an already-Warm CFS") + } + got, _ := dyn.Resource(CFSResource).Get(context.Background(), fvID, metav1.GetOptions{}) + if h := readStatus(got).CheckpointHash; h != "existing" { + t.Errorf("already-Warm hash mutated: got %q", h) + } +} + +// A CFS with no persisted workloadLookup can't be looked up — skip it +// (no panic, no nvsnap-server call). +func TestSweep_SkipsWhenNoLookupInputs(t *testing.T) { + const fvID = "fv-nolookup" + var lookupCalled bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + lookupCalled = true + http.NotFound(w, r) + })) + defer srv.Close() + dyn := newFakeDynamic(coldCFS(fvID)) // no workloadLookup + + sweepReconciler(srv, dyn).SweepOnce(context.Background()) + + if lookupCalled { + t.Error("sweep must not query nvsnap-server when workloadLookup is unset") + } + got, _ := dyn.Resource(CFSResource).Get(context.Background(), fvID, metav1.GetOptions{}) + if s := readStatus(got).LocalCacheState; s == "Warm" { + t.Error("CFS without lookup inputs must not be flipped Warm") + } +} + +// An opted-out CFS must never be flipped Warm, even with a usable +// capture present. +func TestSweep_SkipsOptedOut(t *testing.T) { + const fvID, image, hash = "fv-optout", "ngc.io/fn:3", "deadbeefcafe1003" + srv := usableCaptureServer(image, hash) + defer srv.Close() + dyn := newFakeDynamic(cfsWithLookup(fvID, image, "", func(obj map[string]any) { + obj["spec"].(map[string]any)["optOut"] = true + })) + + sweepReconciler(srv, dyn).SweepOnce(context.Background()) + + got, _ := dyn.Resource(CFSResource).Get(context.Background(), fvID, metav1.GetOptions{}) + if s := readStatus(got).LocalCacheState; s == "Warm" { + t.Error("opted-out CFS must not be flipped Warm") + } +} + +// No usable capture yet (lookup returns empty) → CFS stays not-Warm, +// the sweep will retry next tick. +func TestSweep_LeavesNotWarmWhenNoCapture(t *testing.T) { + const fvID, image = "fv-nocap", "ngc.io/fn:4" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/api/v1/checkpoints/lookup") { + _ = json.NewEncoder(w).Encode(map[string]any{"matches": []map[string]any{}}) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + dyn := newFakeDynamic(cfsWithLookup(fvID, image, "", nil)) + + sweepReconciler(srv, dyn).SweepOnce(context.Background()) + + got, _ := dyn.Resource(CFSResource).Get(context.Background(), fvID, metav1.GetOptions{}) + if s := readStatus(got).LocalCacheState; s == "Warm" { + t.Error("CFS must stay not-Warm when no usable capture exists") + } +} + +// writeWorkloadLookup persists the inputs and is a no-op when already +// current (the reconciler calls it on every reconcile). +func TestWriteWorkloadLookup_PersistsAndIdempotent(t *testing.T) { + const fvID, image, model = "fv-wl", "ngc.io/fn:5", "meta-llama/x" + dyn := newFakeDynamic(coldCFS(fvID)) + ctx := context.Background() + + if err := writeWorkloadLookup(ctx, dyn, fvID, image, model); err != nil { + t.Fatalf("writeWorkloadLookup: %v", err) + } + got, _ := dyn.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + gotImage, gotModel := readWorkloadLookup(got) + if gotImage != image || gotModel != model { + t.Fatalf("persisted lookup: got (%q,%q) want (%q,%q)", gotImage, gotModel, image, model) + } + rv := got.GetResourceVersion() + + // Second call with identical values must not write (resourceVersion + // unchanged). + if err := writeWorkloadLookup(ctx, dyn, fvID, image, model); err != nil { + t.Fatalf("writeWorkloadLookup (2nd): %v", err) + } + got2, _ := dyn.Resource(CFSResource).Get(ctx, fvID, metav1.GetOptions{}) + if got2.GetResourceVersion() != rv { + t.Errorf("idempotent write should not bump resourceVersion: %q -> %q", rv, got2.GetResourceVersion()) + } + + // Empty imageRef is a no-op. + if err := writeWorkloadLookup(ctx, dyn, fvID, "", ""); err != nil { + t.Errorf("empty imageRef should be a no-op, got %v", err) + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap_coldstart_gate.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_coldstart_gate.go new file mode 100644 index 000000000..f9c2f3654 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_coldstart_gate.go @@ -0,0 +1,153 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package nvca + +import ( + "context" + "fmt" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag" + nvsnapreconciler "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler" +) + +// maxConcurrentColdStarts is the number of cold-start pioneers allowed to +// proceed concurrently per function-version (the "K" in the serialized- +// herd design). Hidden internal knob — not surfaced in any helm value or +// UI. v1 honors K=1 only (strict single pioneer); the field exists so +// the call sites read as configurable. +// +// TODO(nvsnap): expose this (per-cluster ClusterConfig.NvSnap) and honor +// K>1 once TryClaimColdStartPioneer grows multi-slot election. Until +// then this MUST stay 1 — the election machinery elects exactly one +// pioneer regardless, so any other value would be a silent lie. +const maxConcurrentColdStarts = 1 + +// deferColdStartReplica is the sentinel-style non-terminal error the +// MiniService-creation gate returns to requeue a deferred replica. +// applyMiniServiceCreationMessage's caller (backendk8scache.go's +// ApplyCreationMessage path) requeues on any non-terminal error, so +// returning this (wrapped, NOT a nvcaerrors.TerminalError) parks the +// replica until the pioneer warms the function. Mirrors the +// "...will be requeued, err: %w" requeue convention in that path. +func deferColdStartReplica(fvID, pioneer string) error { + return fmt.Errorf("nvsnap: deferring replica until cold-start pioneer warms "+ + "(fvID=%s, pioneer=%s); MiniService creation will be requeued", fvID, pioneer) +} + +// shouldDeferColdStart is the serialized-herd cold-start gate (pioneer +// election), evaluated immediately BEFORE the MiniService is created in +// applyMiniServiceCreationMessage. It is the exact analogue of the +// capture-once guard (nvca#189) applied one step earlier: when N replicas +// of one function-version deploy cold (no Warm checkpoint yet), exactly +// one "pioneer" is allowed through to cold-start + capture; the rest are +// deferred (requeued) until that pioneer's capture flips +// LocalCacheState=Warm, at which point they proceed and warm-restore via +// Hook A (stampNvSnapAnnotations). +// +// Returns a non-nil non-terminal error ONLY when this replica must defer. +// In every other case it returns nil ("proceed") — the gate is an +// optimization, never a hard gate on deployment: +// +// - NvSnap integration disabled (feature flag off): proceed. +// - fvID empty (no function-version on the request): proceed. +// - CFS lookup hits a real API error: proceed (fail-open). A transient +// API blip must not block a deployment. +// - CFS Warm: proceed (warm-restore via Hook A). +// - CFS Failed: proceed (cold; fail-open — a prior capture failed, do +// not pin the herd on a slot that can't warm). +// - CFS Cold / Fetching / Capturing / absent: run the pioneer +// election. Win the slot (or an expired-lease steal) → proceed as +// pioneer; lose to a live foreign pioneer → defer. +// +// Replica count: N replicas of one function-version arrive as N +// independent ICMSRequests sharing an fvID — the per-request +// InstanceCount is not the fleet-wide replica count, so the gate does +// NOT try to know N. The election is correct without it: it lets exactly +// one request through and defers the others, which is the desired +// behavior whether N is 2 or 200. A genuinely single-replica function +// simply wins its own election immediately (one pioneer, nobody to +// defer) — no added latency. +func (c K8sComputeBackend) shouldDeferColdStart(ctx context.Context, req *nvcav2beta1.ICMSRequest) error { + log := core.GetLogger(ctx) + + if !featureflag.NvSnapCheckpointRestore.Enabled() { + return nil + } + + fvID := req.Spec.FunctionDetails.FunctionVersionID + if fvID == "" { + return nil // no function-version, nothing to serialize on + } + + cfsObj, err := c.dynClient.Resource(nvsnapFunctionStateGVR).Get(ctx, fvID, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + // Real API error (not a plain 404) — fail open. Blocking a + // deployment on a transient CFS read failure is never correct; + // NvSnap is an optimization. + log.WithError(err).WithField("functionVersionID", fvID). + Info("nvsnap: cold-start gate CFS lookup failed; proceeding (fail-open)") + return nil + } + + // Read state (NotFound → empty object → Cold-equivalent, falls into + // the election branch below where an absent CFS is claimable). + var state string + if cfsObj != nil { + state, _, _ = unstructured.NestedString(cfsObj.Object, "status", "localCacheState") + } + + switch nvsnapv1alpha1.NvSnapFunctionStateLocalCacheState(state) { + case nvsnapv1alpha1.LocalCacheStateWarm: + return nil // warm-restore via Hook A + case nvsnapv1alpha1.LocalCacheStateFailed: + return nil // fail-open cold; don't pin the herd on a failed slot + } + + // Cold / Fetching / Capturing / absent → elect a pioneer. owner is + // the requesting ICMSRequest's namespace/name (its identity for the + // claim; mirrors how the capture-once guard keys CaptureOwner by the + // pod's namespace/name). + owner := req.Namespace + "/" + req.Name + now := core.GetCurrentTime(ctx) + leaseExpiry := now.Add(nvsnapreconciler.DefaultColdStartPioneerLeaseTTL()) + + claimed, err := nvsnapreconciler.TryClaimColdStartPioneer(ctx, c.dynClient, fvID, owner, leaseExpiry, now) + if err != nil { + // CAS itself errored (API failure) — fail open rather than block. + log.WithError(err).WithFields(map[string]any{ + "functionVersionID": fvID, "owner": owner, + }).Info("nvsnap: cold-start pioneer claim failed; proceeding (fail-open)") + return nil + } + if claimed { + coldStartPioneersElected.Inc() + log.WithFields(map[string]any{ + "functionVersionID": fvID, "owner": owner, + }).Info("nvsnap: elected cold-start pioneer; proceeding to cold-start + capture") + return nil + } + + // Lost the election to a live foreign pioneer → defer (requeue). + pioneer, _, _ := unstructured.NestedString(cfsObj.Object, "status", "coldStartPioneer") + coldStartReplicasDeferred.Inc() + log.WithFields(map[string]any{ + "functionVersionID": fvID, "owner": owner, "pioneer": pioneer, + }).Info("nvsnap: deferring replica until cold-start pioneer warms the function") + return deferColdStartReplica(fvID, pioneer) +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap_coldstart_metrics.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_coldstart_metrics.go new file mode 100644 index 000000000..005e4ca51 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_coldstart_metrics.go @@ -0,0 +1,47 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package nvca + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Serialized-herd cold-start (pioneer election) metrics. The +// MiniService-creation gate elects exactly one cold-start pioneer per +// function-version and defers the rest until the pioneer's capture warms +// the function. Both counters are package-level (same pattern as the +// reconciler's metrics.go) so the gate can record without threading a +// metrics handle through applyMiniServiceCreationMessage. +var ( + // coldStartPioneersElected counts MiniService-creation reconciles + // that won the pioneer slot and were allowed to cold-start. Roughly + // one increment per cold function-version (plus one per stolen + // expired-lease slot under pioneer crash recovery). + coldStartPioneersElected = promauto.NewCounter(prometheus.CounterOpts{ + Name: "nvca_nvsnap_coldstart_pioneers_elected_total", + Help: "Count of MiniService-creation reconciles that won the cold-start " + + "pioneer slot and were allowed to cold-start + capture (serialized-herd).", + }) + + // coldStartRepliacasDeferred counts MiniService-creation reconciles + // that observed a live foreign pioneer claim and deferred (requeued) + // rather than creating their MiniService. A healthy herd produces + // (N-1) deferrals per cold function-version of N replicas, each + // repeated until the pioneer warms the function. + coldStartReplicasDeferred = promauto.NewCounter(prometheus.CounterOpts{ + Name: "nvca_nvsnap_coldstart_replicas_deferred_total", + Help: "Count of MiniService-creation reconciles deferred (requeued) because " + + "another replica holds the cold-start pioneer claim (serialized-herd). " + + "Counts every requeue, so it is cumulative across a deferred replica's retries.", + }) +) diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap_controller_start.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_controller_start.go new file mode 100644 index 000000000..5c35fb26f --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_controller_start.go @@ -0,0 +1,78 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package nvca + +import ( + "context" + + "github.com/sirupsen/logrus" + "k8s.io/client-go/dynamic" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/kubeclients" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap" + nvsnapcontroller "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap/controller" + nvsnapreconciler "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap/reconciler" +) + +// startNvSnapController boots Hook B (the checkpoint-after-warm +// reconciler + Pod informer + workqueue). Returns nil immediately +// when featureflag.NvSnapCheckpointRestore is disabled, so the call +// from agent.Start is a safe no-op in the default config. +// +// Runs in its own goroutine; lifecycle bound to ctx. Failure inside +// the controller (e.g., cache sync) logs but doesn't kill the agent +// — NvSnap is an optimization, never a gate on pod creation. +// +// Wiring: +// - dynamic.Interface for NvSnapFunctionState reads/writes. +// - nvsnap HTTP client (in-cluster Service URL by default). +// - Pod informer scoped to a.RequestsNamespace (where tenant pods +// land). Empty namespace = cluster-wide; production NVCA uses +// nvcf-backend. +func (a *Agent) startNvSnapController(ctx context.Context, k8sclients *kubeclients.KubeClients, log *logrus.Entry) error { + if !featureflag.NvSnapCheckpointRestore.Enabled() { + log.Info("nvsnap: NvSnapCheckpointRestore feature flag is off; controller not started") + return nil + } + + dynClient, err := dynamic.NewForConfig(k8sclients.Config) + if err != nil { + log.WithError(err).Warn("nvsnap: dynamic client build failed; controller not started") + return nil // fail-open + } + + // nvsnap.NewClient with no options points at the in-cluster + // Service URL (nvsnap-server.nvsnap-system.svc.cluster.local:8080). + // Per-cluster ServerURL override will land in PR-2's + // ClusterConfig.NvSnap wiring once that propagates to the agent + // runtime config (PR-8 territory). + nvsnapClient := nvsnap.NewClient() + + r := &nvsnapreconciler.Reconciler{ + KubeClient: k8sclients.K8s, + DynClient: dynClient, + NvSnapClient: nvsnapClient, + Log: log.WithField("subcomponent", "nvsnap-reconciler"), + } + ctrl := nvsnapcontroller.NewController(k8sclients.K8s, r) + ctrl.Namespace = a.RequestsNamespace // empty falls back to cluster-wide + ctrl.Log = log.WithField("subcomponent", "nvsnap-controller") + + go func() { + log.WithField("namespace", ctrl.Namespace).Info("nvsnap controller starting") + if err := ctrl.Run(ctx); err != nil && err != context.Canceled { + log.WithError(err).Warn("nvsnap controller exited with error") + } + }() + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap_hook.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_hook.go new file mode 100644 index 000000000..2bdbe1153 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_hook.go @@ -0,0 +1,532 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +package nvca + +import ( + "context" + "errors" + "fmt" + "net/http" + "sort" + "strings" + "time" + + "github.com/sirupsen/logrus" + otelattr "go.opentelemetry.io/otel/attribute" + oteltrace "go.opentelemetry.io/otel/trace" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + + nvcaotel "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/otel" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap" +) + +// inferenceContainerName is the canonical name of the inference +// container in an NVCF pod. Hook A reads its image/env/args to +// build the content-addressed lookup inputs. +const inferenceContainerName = "inference" + +// Annotation keys NVCA stamps on workload pods to drive the NvSnap +// integration. NvSnap's mutating webhook reads RestoreFromAnnotation +// at admission time; Hook B's reconciler (PR-5) watches for pods +// with CheckpointOnWarmAnnotation to know which to checkpoint after +// readiness. +const ( + // NvSnapRestoreFromAnnotation, when present and non-empty, tells + // NvSnap's mutating webhook to inject the restore mounts for + // the given content-addressed checkpoint hash. NvSnap's webhook + // resolves the hash to a manifest ConfigMap, picks a node where + // the cache lives (or override target node), and injects the + // rootfs volume + sitecustomize plumbing. + NvSnapRestoreFromAnnotation = "nvsnap.io/restore-from" + + // NvSnapCheckpointOnWarmAnnotation marks pods that NVCA's + // post-Ready reconciler (Hook B, PR-5) should checkpoint after + // the warmup window. NVCA stamps this at pod-create time on + // every function-version pod whose NvSnapFunctionState is not + // opted out, so the reconciler doesn't need to re-resolve the + // FV state at every Ready event. + NvSnapCheckpointOnWarmAnnotation = "nvsnap.io/checkpoint-on-warm" + + // NvSnapFunctionVersionIDAnnotation is the bridge between + // ICMSRequest-driven Hook A (which knows the FV id from + // req.Spec.FunctionDetails) and the Pod-watching reconciler + // (which doesn't see the originating ICMSRequest). Stamped at + // the same time as CheckpointOnWarmAnnotation. + NvSnapFunctionVersionIDAnnotation = "nvsnap.io/function-version-id" +) + +// nvsnapFunctionStateGVR is the cluster-scoped CR holding per-cluster +// NvSnap state for one NGC function-version. Defined in +// pkg/apis/nvsnap/v1alpha1. +var nvsnapFunctionStateGVR = nvsnapv1alpha1.SchemeGroupVersion.WithResource("nvsnapfunctionstates") + +// stampNvSnapAnnotations is Hook A (restore-on-create). For each pod +// CreatePodArtifactInstances is about to apply, if the NvSnap +// integration is enabled and the function-version has a usable +// checkpoint cached locally, stamp nvsnap.io/restore-from on the pod +// so NvSnap's webhook injects the restore mounts. Always stamps +// nvsnap.io/checkpoint-on-warm when not opted out so Hook B can pick +// up newly-Ready pods. +// +// Fail-open: any error short-circuits to "no stamping" — the pod is +// applied unchanged and cold-starts as before. The NvSnap integration +// is an optimization, never a gate. +// +// Gating (any false short-circuits): +// - featureflag.NvSnapCheckpointRestore is the global kill switch. +// - NvSnapFunctionState.spec.optOut is the per-function-version +// opt-out. +// - Per-cluster ClusterConfig.NvSnap gating is TODO; will land when +// the NVCFBackend CR → NVCA agent runtime config wiring is +// extended (separate PR). Today the global flag is the only +// non-FV gate. +// +// req.Spec.FunctionDetails.FunctionVersionID is the canonical key +// for NvSnapFunctionState lookup — one CR per NGC function-version. +func (c K8sComputeBackend) stampNvSnapAnnotations(ctx context.Context, pod *corev1.Pod, req *nvcav2beta1.ICMSRequest, log *logrus.Entry) { + if !featureflag.NvSnapCheckpointRestore.Enabled() { + return + } + + // Hook A span covers: CFS lookup, optional content-addressed + // lookup RPC to nvsnap-server, annotation stamping decision. The + // span attribute "nvsnap.decision" is set right before return — + // stamped_restore_from | stamped_checkpoint_on_warm | skipped + // — so trace consumers can quickly group by what Hook A + // decided per admission. + tracer := nvcaotel.NewTracer(nvcaotel.WithName("nvca.nvsnap.hook_a")) + ctx, span := tracer.Start(ctx, "nvsnap.hook_a.stamp", + oteltrace.WithSpanKind(oteltrace.SpanKindInternal), + oteltrace.WithAttributes( + otelattr.String("k8s.pod.name", pod.Name), + otelattr.String("k8s.pod.namespace", pod.Namespace), + // pod.UID is empty at admission time (assigned by apiserver + // after the webhook returns), so we skip it here. + ), + ) + defer span.End() + + fvID := req.Spec.FunctionDetails.FunctionVersionID + if fvID == "" { + span.SetAttributes(otelattr.String("nvsnap.decision", "skipped_no_fvid")) + return // no function-version, nothing to look up + } + span.SetAttributes(otelattr.String("nvsnap.function_version_id", fvID)) + + cfsObj, err := c.dynClient.Resource(nvsnapFunctionStateGVR).Get(ctx, fvID, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + log.WithError(err).WithField("functionVersionID", fvID). + Debug("nvsnap: NvSnapFunctionState lookup failed; pod will cold-start") + return + } + if apierrors.IsNotFound(err) { + // First-touch for this function-version. Without this branch + // we'd have a chicken-and-egg: Hook A skips stamping + // checkpoint-on-warm because no CFS exists, Hook B never + // sees a pod with the annotation so no checkpoint fires, + // CFS never gets created. Materialize an empty CFS at + // state=Cold so the next steps stamp the annotation and Hook B + // can pick it up after PodReady. + cfsObj, err = createInitialCFS(ctx, c.dynClient, fvID) + if err != nil { + log.WithError(err).WithField("functionVersionID", fvID). + Debug("nvsnap: failed to create initial NvSnapFunctionState; pod will cold-start") + return + } + log.WithField("functionVersionID", fvID). + Info("nvsnap: created initial NvSnapFunctionState at Cold") + } + + if optOut, _, _ := unstructured.NestedBool(cfsObj.Object, "spec", "optOut"); optOut { + log.WithField("functionVersionID", fvID).Debug("nvsnap: function-version opted out; pod will cold-start") + span.SetAttributes(otelattr.String("nvsnap.decision", "skipped_optout")) + return + } + + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + // FunctionVersionID is needed in both branches below — Hook A + // (the nvsnap webhook reads it back when resolving the restore + // manifest) and Hook B (the reconciler watches Pods, not + // ICMSRequests, so it reads FV id off the pod). + pod.Annotations[NvSnapFunctionVersionIDAnnotation] = fvID + + hash, _, _ := unstructured.NestedString(cfsObj.Object, "status", "checkpointHash") + state, _, _ := unstructured.NestedString(cfsObj.Object, "status", "localCacheState") + + if hash != "" && state == string(nvsnapv1alpha1.LocalCacheStateWarm) { + // Fast path: this function-version already has its own + // checkpoint cached locally. Stamp restore-from; do NOT + // stamp checkpoint-on-warm, otherwise the restore pod would + // also be queued by Hook B and N restored replicas would each + // POST a redundant CreateCheckpoint, overwriting each other's + // hash in CFS. (Greptile P2 on MR !1698.) Refresh-driven + // re-checkpoint is a separate flow (CFS-level state machine, + // not per-pod), so unconditional re-stamping here is wrong. + // + // Self-heal (nvca#190): the artifact can be deleted out from + // under us — UI delete, retention GC, cross-cluster eviction — + // without anything resetting this CFS (the nvsnap-side delete + // cascade owns nvsnap tiers, not NVCA's CFS). A stale Warm then + // makes us stamp restore-from for a hash whose bytes are gone: + // the pod silently cold-starts AND never re-captures (stuck + // Warm forever). So verify the artifact still exists on + // nvsnap-server before honoring Warm. Fail-OPEN: a transient + // nvsnap-server error must not break otherwise-valid restores. + stale := false + if c.nvsnapClient != nil { + vctx, cancel := context.WithTimeout(ctx, 2*time.Second) + exists, checkErr := nvsnapArtifactExists(vctx, c.nvsnapClient, hash) + cancel() + if checkErr != nil { + log.WithError(checkErr).WithFields(logrus.Fields{ + "functionVersionID": fvID, "hash": hash, + }).Debug("nvsnap: artifact existence check failed; failing open (honoring Warm)") + } else if !exists { + stale = true + } + } + if stale { + // Definitive: nvsnap-server has no record of this hash. Reset + // CFS to Cold and fall through to the cold/capture path so + // Hook B re-captures. Do NOT stamp restore-from. + if err := resetCFSToCold(ctx, c.dynClient, fvID); err != nil { + log.WithError(err).WithField("functionVersionID", fvID). + Warn("nvsnap: failed to reset stale-Warm CFS to Cold; pod will still cold-start") + } + span.SetAttributes( + otelattr.String("nvsnap.decision", "warm_artifact_missing_recapture"), + otelattr.String("nvsnap.hash", hash), + ) + log.WithFields(logrus.Fields{ + "functionVersionID": fvID, + "hash": hash, + }).Warn("nvsnap: CFS=Warm but nvsnap-server has no artifact for hash; reset to Cold, will re-capture") + // fall through to the cold/capture path below (no return). + } else { + pod.Annotations[NvSnapRestoreFromAnnotation] = hash + span.SetAttributes( + otelattr.String("nvsnap.decision", "stamped_restore_from_fvid"), + otelattr.String("nvsnap.hash", hash), + ) + log.WithFields(logrus.Fields{ + "functionVersionID": fvID, + "hash": hash, + }).Info("nvsnap: stamped nvsnap.io/restore-from (fvID-keyed)") + return + } + } + + // Content-addressed dedup (nvnvsnap#59). The fvID-keyed lookup + // missed — either this is a true cold start, or this fvID is new + // but another fvID with the same canonical content has already + // captured. Ask nvsnap-server "do you have a checkpoint for this + // (image, model, flags, driver)?" — if yes, restore from it + // instead of cold-starting + re-capturing. + if match := c.lookupContentAddressedMatch(ctx, pod, fvID, log); match != nil { + pod.Annotations[NvSnapRestoreFromAnnotation] = match.Hash + span.SetAttributes( + otelattr.String("nvsnap.decision", "stamped_restore_from_content_addressed"), + otelattr.String("nvsnap.hash", match.Hash), + otelattr.String("nvsnap.source_checkpoint_id", match.CheckpointID), + ) + log.WithFields(logrus.Fields{ + "functionVersionID": fvID, + "hash": match.Hash, + "sourceCheckpointID": match.CheckpointID, + "sourceNode": match.CapturedOnNode, + }).Info("nvsnap: stamped nvsnap.io/restore-from (content-addressed dedup across fvIDs)") + // Update our CFS to Warm with the dedup'd hash so subsequent + // pods for THIS fvID hit the fvID-keyed fast path above + // (avoids one lookup RPC per admission once we know). + if err := markCFSWarmFromLookup(ctx, c.dynClient, fvID, match.Hash); err != nil { + // Non-fatal — the annotation is already stamped on this + // pod; the next admission will just re-do the lookup. + log.WithError(err).Debug("nvsnap: failed to persist dedup'd hash into CFS; will re-lookup next admission") + } + return + } + + // True cold start. Hook B trigger — cache is Cold / Fetching / + // Failed / absent and no content-addressed match exists either. + pod.Annotations[NvSnapCheckpointOnWarmAnnotation] = "true" + span.SetAttributes(otelattr.String("nvsnap.decision", "stamped_checkpoint_on_warm")) +} + +// lookupContentAddressedMatch queries nvsnap-server for a checkpoint +// whose canonical content matches this pod's spec. Returns the +// freshest match if any, nil otherwise (or on any error — fail-open, +// the pod just cold-starts as before). +// +// Inputs derived from the pod: +// - imageRef: the "inference" container's image string (NVCF +// convention; falls back to pod.Spec.Containers[0] if absent) +// - modelID: pulled from container env (NIM_MODEL_NAME, +// HF_MODEL_ID, etc.) or --model args (vLLM/SGLang style) +// - engineFlags: container.Args minus --model* (server canonicalizes) +// - driverMajor: best-effort from node label; left 0 (= match any) +// when we can't resolve it cheaply, so we don't gate the dedup +// on perfect node info +func (c K8sComputeBackend) lookupContentAddressedMatch(ctx context.Context, pod *corev1.Pod, fvID string, log *logrus.Entry) *nvsnap.LookupMatch { + if c.nvsnapClient == nil { + return nil + } + target := inferenceContainer(pod) + if target == nil || target.Image == "" { + // No inference container, no canonical inputs — can't dedup. + return nil + } + + lookupCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + resp, err := c.nvsnapClient.LookupCheckpoints(lookupCtx, nvsnap.LookupRequest{ + ImageRef: target.Image, + ModelID: extractModelID(target), + EngineFlags: canonicalizeArgs(target.Args), + // DriverMajor left 0 → match any driver. Production wiring + // will populate this from a node-label cache once the node + // the pod will land on is known; admission-time we don't + // have that, and partial-match-better-than-no-match. + }) + if err != nil { + log.WithError(err).WithField("functionVersionID", fvID). + Debug("nvsnap: content-addressed lookup failed; pod will cold-start") + return nil + } + if len(resp.Matches) == 0 { + return nil + } + return &resp.Matches[0] +} + +// createInitialCFS materializes a NvSnapFunctionState at state=Cold for +// the given function-version. Used by Hook A's first-touch path: +// before this, Hook A skipped stamping checkpoint-on-warm when no CFS +// existed, which left the function in a chicken-and-egg state with +// Hook B (no annotation -> no Hook B fire -> no checkpoint -> no CFS +// status update). Mirrors the controller-side getOrCreateCFS in +// pkg/nvca/nvsnap/reconciler/state.go (deliberately duplicated rather +// than imported, to avoid an import cycle: pkg/nvca depends on +// pkg/nvca/nvsnap/* via nvsnap_controller_start.go, and reconciler +// importing pkg/nvca would close the cycle). +// +// AlreadyExists on race: re-read the existing object — another +// caller materialized it between our Get and Create, the state we +// want is already there. +func createInitialCFS(ctx context.Context, dc dynamic.Interface, fvID string) (*unstructured.Unstructured, error) { + fresh := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": fvID}, + "spec": map[string]any{ + "functionVersionID": fvID, + }, + "status": map[string]any{ + "localCacheState": string(nvsnapv1alpha1.LocalCacheStateCold), + }, + }, + } + created, err := dc.Resource(nvsnapFunctionStateGVR).Create(ctx, fresh, metav1.CreateOptions{}) + if err == nil { + return created, nil + } + if apierrors.IsAlreadyExists(err) { + return dc.Resource(nvsnapFunctionStateGVR).Get(ctx, fvID, metav1.GetOptions{}) + } + return nil, fmt.Errorf("create NvSnapFunctionState %s: %w", fvID, err) +} + +// inferenceContainer returns the canonical inference container in +// pod.Spec.Containers — defaults to the one named "inference" (NVCF +// convention), falls back to the first GPU-requesting container, +// then the first container. Returns nil only if Containers is empty. +func inferenceContainer(pod *corev1.Pod) *corev1.Container { + if pod == nil { + return nil + } + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == inferenceContainerName { + return &pod.Spec.Containers[i] + } + } + for i := range pod.Spec.Containers { + if _, ok := pod.Spec.Containers[i].Resources.Limits["nvidia.com/gpu"]; ok { + return &pod.Spec.Containers[i] + } + } + if len(pod.Spec.Containers) > 0 { + return &pod.Spec.Containers[0] + } + return nil +} + +// extractModelID inspects the container's env + args for the +// well-known model-identifier keys (NIM, vLLM, SGLang, TRT-LLM, HF). +// Mirrors the agent-side helper in nvsnap's internal/agent/catalog.go +// so both sides extract the same value from the same pod spec. +func extractModelID(c *corev1.Container) string { + envKeys := []string{ + "NIM_MODEL_NAME", + "MODEL_NAME", + "MODEL_REPO", + "HF_MODEL_ID", + "HUGGINGFACE_MODEL", + } + for _, e := range c.Env { + for _, key := range envKeys { + if e.Name == key && e.Value != "" { + return e.Value + } + } + } + for i, arg := range c.Args { + if arg == "--model" || arg == "--model-path" { + if i+1 < len(c.Args) { + return c.Args[i+1] + } + } + if strings.HasPrefix(arg, "--model=") { + return strings.TrimPrefix(arg, "--model=") + } + if strings.HasPrefix(arg, "--model-path=") { + return strings.TrimPrefix(arg, "--model-path=") + } + } + return "" +} + +// canonicalizeArgs strips --model* tokens (recorded separately as +// modelID) and sorts the remainder. Same logic the nvsnap agent and +// nvsnap-server use; the server canonicalizes again on its side, so +// any drift between the two implementations self-corrects to "no +// match". Mirroring keeps the round-trip free of false negatives. +func canonicalizeArgs(args []string) []string { + if len(args) == 0 { + return nil + } + out := make([]string, 0, len(args)) + skip := false + for _, a := range args { + if skip { + skip = false + continue + } + if a == "--model" || a == "--model-path" { + skip = true + continue + } + if strings.HasPrefix(a, "--model=") || strings.HasPrefix(a, "--model-path=") { + continue + } + out = append(out, a) + } + sort.Strings(out) + return out +} + +// markCFSWarmFromLookup persists a content-addressed dedup hit into +// NvSnapFunctionState.status. Result: subsequent pods for this fvID +// take the fvID-keyed fast path in stampNvSnapAnnotations and skip the +// lookup RPC. capturedHere stays false — we didn't capture, we +// borrowed someone else's checkpoint. +// +// Mirrors writeStatus semantics from pkg/nvca/nvsnap/reconciler/state.go +// but avoids importing reconciler to dodge the import cycle (same +// rationale as createInitialCFS). +// nvsnapArtifactExists reports whether nvsnap-server still has a catalog +// record for hash. Returns (false,nil) on a definitive 404, (true,nil) +// when present, and (false,err) on any other error so callers can +// fail-open (a transient nvsnap-server blip must not invalidate Warm). +func nvsnapArtifactExists(ctx context.Context, client *nvsnap.Client, hash string) (bool, error) { + if _, err := client.GetCheckpoint(ctx, hash); err != nil { + var apiErr *nvsnap.APIError + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound { + return false, nil + } + return false, err + } + return true, nil +} + +// resetCFSToCold flips a stale Warm NvSnapFunctionState back to Cold and +// clears the dangling checkpoint hash, so the next admission stamps +// checkpoint-on-warm and Hook B re-captures. Mirrors markCFSWarmFromLookup's +// UpdateStatus/Update fallback for CRDs without a /status subresource. +func resetCFSToCold(ctx context.Context, dc dynamic.Interface, fvID string) error { + cur, err := dc.Resource(nvsnapFunctionStateGVR).Get(ctx, fvID, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get NvSnapFunctionState %s: %w", fvID, err) + } + status, _, _ := unstructured.NestedMap(cur.Object, "status") + if status == nil { + status = map[string]any{} + } + status["localCacheState"] = string(nvsnapv1alpha1.LocalCacheStateCold) + status["checkpointHash"] = "" + status["capturedHere"] = false + status["lastError"] = "" + if err := unstructured.SetNestedField(cur.Object, status, "status"); err != nil { + return fmt.Errorf("set status: %w", err) + } + if _, err := dc.Resource(nvsnapFunctionStateGVR).UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err != nil { + if apierrors.IsNotFound(err) || apierrors.IsMethodNotSupported(err) { + if _, err := dc.Resource(nvsnapFunctionStateGVR).Update(ctx, cur, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("update NvSnapFunctionState %s: %w", fvID, err) + } + return nil + } + return fmt.Errorf("update status %s: %w", fvID, err) + } + return nil +} + +func markCFSWarmFromLookup(ctx context.Context, dc dynamic.Interface, fvID, hash string) error { + cur, err := dc.Resource(nvsnapFunctionStateGVR).Get(ctx, fvID, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get NvSnapFunctionState %s: %w", fvID, err) + } + status, _, _ := unstructured.NestedMap(cur.Object, "status") + if status == nil { + status = map[string]any{} + } + status["checkpointHash"] = hash + status["localCacheState"] = string(nvsnapv1alpha1.LocalCacheStateWarm) + status["capturedHere"] = false + status["capturedAt"] = time.Now().UTC().Format(time.RFC3339) + status["lastError"] = "" + if err := unstructured.SetNestedField(cur.Object, status, "status"); err != nil { + return fmt.Errorf("set status: %w", err) + } + if _, err := dc.Resource(nvsnapFunctionStateGVR).UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err != nil { + if apierrors.IsNotFound(err) || apierrors.IsMethodNotSupported(err) { + // CRD without /status subresource — fall back to full Update. + // Same known TOCTOU on spec as state.go writeStatus; same fix + // (registering subresource:status) resolves both. + if _, err := dc.Resource(nvsnapFunctionStateGVR).Update(ctx, cur, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("update NvSnapFunctionState %s: %w", fvID, err) + } + return nil + } + return fmt.Errorf("update status %s: %w", fvID, err) + } + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap_hook_lookup_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_hook_lookup_test.go new file mode 100644 index 000000000..34d72ec82 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_hook_lookup_test.go @@ -0,0 +1,445 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +// Tests for Hook A content-addressed lookup (nvnvsnap#59 NVCA side): +// the new lookupContentAddressedMatch + helper extractors. Stands up +// a fake nvsnap-server with httptest to exercise the full POST /lookup +// round-trip against a stub catalog. + +package nvca + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap" +) + +// fakeLookupServer stands up an httptest.Server that replies to +// POST /api/v1/checkpoints/lookup with the configured matches. +// recordedRequest captures the last received body so tests can +// assert NVCA built the right query. +type fakeLookupServer struct { + matches []nvsnap.LookupMatch + statusCode int + recordedRequest nvsnap.LookupRequest + requestCount int + // artifactMissing makes the GetCheckpoint existence probe (the + // Warm-path self-heal in stampNvSnapAnnotations) return 404. Default + // false → the probe returns 200 ("artifact exists") so the fvID + // fast path isn't misread as a deleted artifact. + artifactMissing bool +} + +func (f *fakeLookupServer) handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/checkpoints/lookup" { + // GET /api/v1/checkpoints/{id} — the Warm-path artifact + // existence check. Serve 200 (exists) unless artifactMissing. + if strings.HasPrefix(r.URL.Path, "/api/v1/checkpoints/") && !f.artifactMissing { + _, _ = w.Write([]byte(`{}`)) + return + } + http.NotFound(w, r) + return + } + f.requestCount++ + _ = json.NewDecoder(r.Body).Decode(&f.recordedRequest) + if f.statusCode != 0 { + http.Error(w, "boom", f.statusCode) + return + } + _ = json.NewEncoder(w).Encode(nvsnap.LookupResponse{Matches: f.matches}) + }) +} + +// withNvSnapClient wires a nvsnap.Client pointed at the test server into +// the K8sComputeBackend. +func (b K8sComputeBackend) withNvSnapClient(srv *httptest.Server) K8sComputeBackend { + b.nvsnapClient = nvsnap.NewClient(nvsnap.WithBaseURL(srv.URL)) + return b +} + +func inferencePodWithSpec(name, fvID, image, model string, args []string) *corev1.Pod { + return &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "inference", + Image: image, + Env: []corev1.EnvVar{{Name: "NIM_MODEL_NAME", Value: model}}, + Args: args, + }, + }, + }, + } +} + +// Day 2 deploy scenario: fvID=Y is brand new (no CFS), but the +// canonical workload identity matches an existing capture under +// fvID=X. Hook A should query the lookup endpoint, find the X +// checkpoint, stamp restore-from with its hash, and skip cold-start. +func TestStampContentAddressedDedupAcrossFVIDs(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + fake := &fakeLookupServer{matches: []nvsnap.LookupMatch{{ + Hash: "85ec4d75ee57c1be444dd19733f63cfd", + CheckpointID: "85ec4d75__20260531-174604", + CapturedOnNode: "node-1", + ImageRef: "nvcr.io/foo:1.2", + }}} + srv := httptest.NewServer(fake.handler()) + defer srv.Close() + + // No CFS for fv-Y exists; createInitialCFS will run first. + c := newHookTestBackend(t).withNvSnapClient(srv) + pod := inferencePodWithSpec("p1", "fv-Y", "nvcr.io/foo:1.2", "meta/llama", + []string{"--port", "8000"}) + + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-Y"), logrus.NewEntry(logrus.New())) + + if got := pod.Annotations[NvSnapRestoreFromAnnotation]; got != "85ec4d75ee57c1be444dd19733f63cfd" { + t.Errorf("restore-from = %q, want lookup match hash", got) + } + // checkpoint-on-warm should NOT be stamped — we're restoring, not + // triggering a fresh capture. + if _, ok := pod.Annotations[NvSnapCheckpointOnWarmAnnotation]; ok { + t.Error("checkpoint-on-warm should NOT be stamped on dedup hit") + } + // Server must have been queried with the canonical inputs. + if fake.recordedRequest.ImageRef != "nvcr.io/foo:1.2" { + t.Errorf("ImageRef sent = %q", fake.recordedRequest.ImageRef) + } + if fake.recordedRequest.ModelID != "meta/llama" { + t.Errorf("ModelID sent = %q", fake.recordedRequest.ModelID) + } + // CFS_Y should now be persisted at Warm with the dedup'd hash so + // the next admission for fv-Y takes the fvID-keyed fast path. + got, err := c.dynClient.Resource(nvsnapFunctionStateGVR). + Get(context.Background(), "fv-Y", metav1.GetOptions{}) + if err != nil { + t.Fatalf("CFS_Y should exist after dedup: %v", err) + } + state, _, _ := unstructured.NestedString(got.Object, "status", "localCacheState") + if state != string(nvsnapv1alpha1.LocalCacheStateWarm) { + t.Errorf("CFS_Y state = %q, want Warm", state) + } + hash, _, _ := unstructured.NestedString(got.Object, "status", "checkpointHash") + if hash != "85ec4d75ee57c1be444dd19733f63cfd" { + t.Errorf("CFS_Y hash = %q, want lookup match", hash) + } + capturedHere, _, _ := unstructured.NestedBool(got.Object, "status", "capturedHere") + if capturedHere { + t.Error("capturedHere should be false — we didn't capture, we borrowed") + } +} + +// Lookup returns no matches → fall through to cold-start (Hook B +// stamps checkpoint-on-warm), no restore-from annotation. +func TestStampNoLookupMatchFallsThroughToCold(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + fake := &fakeLookupServer{matches: nil} // empty matches + srv := httptest.NewServer(fake.handler()) + defer srv.Close() + + c := newHookTestBackend(t).withNvSnapClient(srv) + pod := inferencePodWithSpec("p1", "fv-cold", "nvcr.io/never-seen:1.0", "", nil) + + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-cold"), logrus.NewEntry(logrus.New())) + + if _, ok := pod.Annotations[NvSnapRestoreFromAnnotation]; ok { + t.Error("restore-from should NOT be stamped when lookup has no match") + } + if pod.Annotations[NvSnapCheckpointOnWarmAnnotation] != "true" { + t.Error("checkpoint-on-warm should be stamped on cold-start path") + } +} + +// Lookup endpoint errors → fail-open. Pod still gets checkpoint-on-warm +// stamped so it'll be captured by Hook B; NvSnap as an optimization is +// never allowed to block pod creation. +func TestStampLookupErrorFailsOpenToCold(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + fake := &fakeLookupServer{statusCode: http.StatusInternalServerError} + srv := httptest.NewServer(fake.handler()) + defer srv.Close() + + c := newHookTestBackend(t).withNvSnapClient(srv) + pod := inferencePodWithSpec("p1", "fv-err", "nvcr.io/foo:1.2", "meta/llama", nil) + + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-err"), logrus.NewEntry(logrus.New())) + + if _, ok := pod.Annotations[NvSnapRestoreFromAnnotation]; ok { + t.Error("restore-from should NOT be stamped on lookup error") + } + if pod.Annotations[NvSnapCheckpointOnWarmAnnotation] != "true" { + t.Error("checkpoint-on-warm should be stamped on fail-open path") + } +} + +// fvID-keyed fast path (CFS Warm + hash already set) must NOT +// query the lookup endpoint. Avoids redundant RPC + dedup races on +// the steady state. +func TestStampFvIDFastPathSkipsLookup(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + fake := &fakeLookupServer{matches: []nvsnap.LookupMatch{{Hash: "wrong"}}} + srv := httptest.NewServer(fake.handler()) + defer srv.Close() + + c := newHookTestBackend(t, + nvsnapFunctionStateUnstructured("fv-1", false, "fvid-keyed-hash", nvsnapv1alpha1.LocalCacheStateWarm), + ).withNvSnapClient(srv) + pod := inferencePodWithSpec("p1", "fv-1", "nvcr.io/foo:1.2", "meta/llama", nil) + + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + + if got := pod.Annotations[NvSnapRestoreFromAnnotation]; got != "fvid-keyed-hash" { + t.Errorf("restore-from = %q, want fvID-keyed hash (not lookup result)", got) + } + if fake.requestCount != 0 { + t.Errorf("lookup endpoint was called %d times; should be 0 on fvID fast path", fake.requestCount) + } +} + +// nil nvsnapClient (lookup unavailable): fall through to cold-start. +// Mirrors what happens if nvsnap.NewClient construction failed at +// K8sComputeBackend init. +func TestStampNilNvSnapClientFallsThrough(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + c := newHookTestBackend(t) // nvsnapClient stays nil + pod := inferencePodWithSpec("p1", "fv-no-client", "nvcr.io/foo:1.2", "", nil) + + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-no-client"), logrus.NewEntry(logrus.New())) + + if _, ok := pod.Annotations[NvSnapRestoreFromAnnotation]; ok { + t.Error("restore-from should NOT be stamped with no nvsnap client") + } + if pod.Annotations[NvSnapCheckpointOnWarmAnnotation] != "true" { + t.Error("checkpoint-on-warm should still be stamped (cold-start path)") + } +} + +// Pod with no inference container → skip lookup gracefully. Some +// non-standard pod shapes might land here (test fixtures, custom +// workloads); we shouldn't crash on them. +func TestStampNoInferenceContainerSkipsLookup(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + fake := &fakeLookupServer{} + srv := httptest.NewServer(fake.handler()) + defer srv.Close() + + c := newHookTestBackend(t).withNvSnapClient(srv) + pod := &corev1.Pod{} // no containers at all + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-bare"), logrus.NewEntry(logrus.New())) + + if fake.requestCount != 0 { + t.Errorf("should not call lookup for pod with no containers; calls=%d", fake.requestCount) + } +} + +func TestInferenceContainer_Selection(t *testing.T) { + cases := []struct { + name string + pod *corev1.Pod + want string // expected container name, "" means nil result + }{ + { + name: "named 'inference' wins", + pod: &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: "sidecar"}, + {Name: "inference"}, + }}}, + want: "inference", + }, + { + name: "falls back to first GPU-requesting container", + pod: &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: "cpu-only"}, + {Name: "gpu-worker", Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"nvidia.com/gpu": resource.MustParse("1")}, + }}, + }}}, + want: "gpu-worker", + }, + { + name: "falls back to first container if no match", + pod: &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: "first"}, + {Name: "second"}, + }}}, + want: "first", + }, + { + name: "nil pod → nil", + pod: nil, + want: "", + }, + { + name: "no containers → nil", + pod: &corev1.Pod{}, + want: "", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := inferenceContainer(tc.pod) + if tc.want == "" { + if got != nil { + t.Errorf("got %v, want nil", got.Name) + } + return + } + if got == nil || got.Name != tc.want { + t.Errorf("got %v, want %q", got, tc.want) + } + }) + } +} + +func TestExtractModelID_HookA(t *testing.T) { + cases := []struct { + name string + c corev1.Container + want string + }{ + {"NIM env var", corev1.Container{Env: []corev1.EnvVar{{Name: "NIM_MODEL_NAME", Value: "meta/llama"}}}, "meta/llama"}, + {"HF env var", corev1.Container{Env: []corev1.EnvVar{{Name: "HF_MODEL_ID", Value: "TinyLlama"}}}, "TinyLlama"}, + {"--model arg", corev1.Container{Args: []string{"--port", "8000", "--model", "meta-llama/Llama"}}, "meta-llama/Llama"}, + {"--model= arg", corev1.Container{Args: []string{"--model=foo/bar"}}, "foo/bar"}, + {"--model-path", corev1.Container{Args: []string{"--model-path", "/models/local"}}, "/models/local"}, + {"--model-path=", corev1.Container{Args: []string{"--model-path=/m"}}, "/m"}, + {"env beats args", corev1.Container{ + Env: []corev1.EnvVar{{Name: "MODEL_NAME", Value: "from-env"}}, + Args: []string{"--model", "from-args"}, + }, "from-env"}, + {"empty env falls through to args", corev1.Container{ + Env: []corev1.EnvVar{{Name: "NIM_MODEL_NAME", Value: ""}}, + Args: []string{"--model", "from-args"}, + }, "from-args"}, + {"--model dangling", corev1.Container{Args: []string{"--model"}}, ""}, + {"nothing", corev1.Container{}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := extractModelID(&tc.c); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } +} + +func TestCanonicalizeArgs_HookA(t *testing.T) { + cases := []struct { + name string + in []string + want []string + }{ + {"strips --model + sorts", []string{"--port", "8000", "--model", "x", "--tp", "1"}, []string{"--port", "--tp", "1", "8000"}}, + {"strips --model=", []string{"--model=x", "--port=8000"}, []string{"--port=8000"}}, + {"strips --model-path", []string{"--model-path", "/x", "--port", "8000"}, []string{"--port", "8000"}}, + {"sorts identical inputs identically", []string{"--b", "--a"}, []string{"--a", "--b"}}, + {"empty in → nil out", nil, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := canonicalizeArgs(tc.in) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("got %v, want %v", got, tc.want) + } + }) + } +} + +// Sanity: the request body NVCA POSTs to /lookup contains exactly the +// canonical inputs in the JSON shape the server expects. Round-trip +// via the recorded request lets us catch silent field renames. +func TestLookupRequestShape(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + fake := &fakeLookupServer{matches: nil} + srv := httptest.NewServer(fake.handler()) + defer srv.Close() + + c := newHookTestBackend(t).withNvSnapClient(srv) + pod := inferencePodWithSpec("p1", "fv-1", + "nvcr.io/foo:1.2", "meta/llama", + []string{"--port", "8000", "--model", "ignored-in-args", "--tp", "1"}) + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + + if fake.recordedRequest.ImageRef != "nvcr.io/foo:1.2" { + t.Errorf("ImageRef = %q", fake.recordedRequest.ImageRef) + } + if fake.recordedRequest.ModelID != "meta/llama" { + t.Errorf("ModelID = %q", fake.recordedRequest.ModelID) + } + // canonicalized: --model stripped, rest sorted + wantFlags := []string{"--port", "--tp", "1", "8000"} + if !reflect.DeepEqual(fake.recordedRequest.EngineFlags, wantFlags) { + t.Errorf("EngineFlags = %v, want %v", fake.recordedRequest.EngineFlags, wantFlags) + } + // DriverMajor stays 0 (we can't resolve at admission cheaply yet — + // future: cache node labels at NVCA agent startup) + if fake.recordedRequest.DriverMajor != 0 { + t.Errorf("DriverMajor = %d, want 0 (unresolved at admission)", fake.recordedRequest.DriverMajor) + } +} + +// Smoke: the prepared request body deserializes as nvsnap.LookupRequest +// on the server side. Failure mode this catches: JSON tag mismatch +// between NVCA's view and nvsnap-server's view of the lookup schema. +func TestLookupRequest_JSONRoundTrip(t *testing.T) { + src := nvsnap.LookupRequest{ + ImageRef: "nvcr.io/foo:1.2", + ModelID: "meta/llama", + EngineFlags: []string{"--port", "8000"}, + DriverMajor: 550, + Limit: 5, + } + b, err := json.Marshal(src) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(b), `"imageRef":"nvcr.io/foo:1.2"`) { + t.Errorf("imageRef field missing or renamed; body=%s", string(b)) + } + if !strings.Contains(string(b), `"engineFlags":["--port","8000"]`) { + t.Errorf("engineFlags field missing or renamed; body=%s", string(b)) + } + var dst nvsnap.LookupRequest + if err := json.Unmarshal(b, &dst); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !reflect.DeepEqual(src, dst) { + t.Errorf("round-trip mismatch:\n src=%+v\n dst=%+v", src, dst) + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/nvsnap_hook_test.go b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_hook_test.go new file mode 100644 index 000000000..269d4ae98 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/nvsnap_hook_test.go @@ -0,0 +1,344 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +*/ + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package nvca + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + fakedynamic "k8s.io/client-go/dynamic/fake" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvsnapv1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/nvsnap" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" +) + +// withFlagEnabled temporarily flips a feature flag on, returning a +// cleanup func. featureflag.CLIFlag.Set is the only exported way to +// mutate flags programmatically. +func withFlagEnabled(t *testing.T, flagKey string) func() { + t.Helper() + cli := &featureflag.CLIFlag{} + if err := cli.Set("+" + flagKey); err != nil { + t.Fatalf("enable %s: %v", flagKey, err) + } + return func() { + _ = cli.Set("-" + flagKey) + } +} + +// nvsnapFunctionStateUnstructured builds an unstructured NvSnapFunctionState +// for fake-dynamic seeding. +func nvsnapFunctionStateUnstructured(name string, optOut bool, hash string, state nvsnapv1alpha1.NvSnapFunctionStateLocalCacheState) *unstructured.Unstructured { + u := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": nvsnapv1alpha1.SchemeGroupVersion.String(), + "kind": "NvSnapFunctionState", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{ + "functionVersionID": name, + "optOut": optOut, + }, + "status": map[string]any{ + "checkpointHash": hash, + "localCacheState": string(state), + }, + }, + } + return u +} + +// newHookTestBackend wires a K8sComputeBackend with only the dynClient +// set — enough for stampNvSnapAnnotations, which doesn't read any other +// field. Seeds the fake dynamic client with the given NvSnapFunctionState +// objects. +func newHookTestBackend(t *testing.T, seed ...*unstructured.Unstructured) K8sComputeBackend { + t.Helper() + scheme := runtime.NewScheme() + // Register the NvSnapFunctionState gvk so fakedynamic can list/get it. + scheme.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: nvsnapFunctionStateGVR.Group, + Version: nvsnapFunctionStateGVR.Version, + Kind: "NvSnapFunctionState", + }, + &unstructured.Unstructured{}, + ) + scheme.AddKnownTypeWithName( + schema.GroupVersionKind{ + Group: nvsnapFunctionStateGVR.Group, + Version: nvsnapFunctionStateGVR.Version, + Kind: "NvSnapFunctionStateList", + }, + &unstructured.UnstructuredList{}, + ) + listKinds := map[schema.GroupVersionResource]string{ + nvsnapFunctionStateGVR: "NvSnapFunctionStateList", + } + objs := make([]runtime.Object, 0, len(seed)) + for _, o := range seed { + objs = append(objs, o) + } + dc := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, listKinds, objs...) + return K8sComputeBackend{dynClient: dc} +} + +func newReq(fvID string) *nvcav2beta1.ICMSRequest { + return &nvcav2beta1.ICMSRequest{ + Spec: nvcav2beta1.ICMSRequestSpec{ + FunctionDetails: function.Details{ + FunctionVersionID: fvID, + }, + }, + } +} + +func TestStampNoFlagDoesNothing(t *testing.T) { + c := newHookTestBackend(t, + nvsnapFunctionStateUnstructured("fv-1", false, "deadbeef", nvsnapv1alpha1.LocalCacheStateWarm), + ) + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + if _, ok := pod.Annotations[NvSnapRestoreFromAnnotation]; ok { + t.Error("restore-from should NOT be stamped when feature flag is off") + } + if _, ok := pod.Annotations[NvSnapCheckpointOnWarmAnnotation]; ok { + t.Error("checkpoint-on-warm should NOT be stamped when feature flag is off") + } +} + +func TestStampWarmHashStampsRestoreFrom(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + c := newHookTestBackend(t, + nvsnapFunctionStateUnstructured("fv-1", false, "deadbeef", nvsnapv1alpha1.LocalCacheStateWarm), + ) + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + if got := pod.Annotations[NvSnapRestoreFromAnnotation]; got != "deadbeef" { + t.Errorf("restore-from = %q, want deadbeef", got) + } + if got := pod.Annotations[NvSnapFunctionVersionIDAnnotation]; got != "fv-1" { + t.Errorf("function-version-id = %q, want fv-1 (Hook B reads FV id from pod)", got) + } + // Regression for Greptile P2 on MR !1698: when the cache is + // already Warm, the pod is being restored — NOT a candidate for + // a fresh checkpoint. Stamping checkpoint-on-warm here would + // cause N restored replicas to each POST a redundant + // CreateCheckpoint to nvsnap-server, overwriting each other's hash. + if _, ok := pod.Annotations[NvSnapCheckpointOnWarmAnnotation]; ok { + t.Error("checkpoint-on-warm must NOT be stamped on a restore (cache Warm) — would cause redundant re-checkpoint by Hook B") + } +} + +func TestStampColdSkipsRestoreFromKeepsOnWarm(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + c := newHookTestBackend(t, + nvsnapFunctionStateUnstructured("fv-1", false, "deadbeef", nvsnapv1alpha1.LocalCacheStateCold), + ) + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + if _, ok := pod.Annotations[NvSnapRestoreFromAnnotation]; ok { + t.Error("restore-from should NOT be stamped when cache state is Cold") + } + if got := pod.Annotations[NvSnapCheckpointOnWarmAnnotation]; got != "true" { + t.Errorf("checkpoint-on-warm should still be stamped to drive Hook B, got %q", got) + } +} + +func TestStampOptOutDoesNothing(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + c := newHookTestBackend(t, + nvsnapFunctionStateUnstructured("fv-1", true /*optOut*/, "deadbeef", nvsnapv1alpha1.LocalCacheStateWarm), + ) + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + if _, ok := pod.Annotations[NvSnapRestoreFromAnnotation]; ok { + t.Error("restore-from should NOT be stamped when function-version is opted out") + } + if _, ok := pod.Annotations[NvSnapCheckpointOnWarmAnnotation]; ok { + t.Error("checkpoint-on-warm should NOT be stamped when function-version is opted out") + } +} + +func TestStampMissingCFSCreatesItAndStampsCheckpointOnWarm(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + // No NvSnapFunctionState seeded — GET returns NotFound. This is the + // first-touch path: function-version has never been checkpointed. + // Hook A must materialize an empty CFS at state=Cold and stamp + // the Hook B trigger so the reconciler picks the pod up and + // captures the first checkpoint. Without this, no CFS would ever + // exist (chicken-and-egg with Hook B). + c := newHookTestBackend(t) + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-never-seen"), logrus.NewEntry(logrus.New())) + + if _, ok := pod.Annotations[NvSnapRestoreFromAnnotation]; ok { + t.Error("restore-from should NOT be stamped — no checkpoint exists yet") + } + if got := pod.Annotations[NvSnapCheckpointOnWarmAnnotation]; got != "true" { + t.Errorf("checkpoint-on-warm = %q, want true (first-touch fix)", got) + } + if got := pod.Annotations[NvSnapFunctionVersionIDAnnotation]; got != "fv-never-seen" { + t.Errorf("function-version-id = %q, want fv-never-seen", got) + } + + // CFS should have been created with state=Cold. + got, err := c.dynClient.Resource(nvsnapFunctionStateGVR).Get(context.Background(), "fv-never-seen", metav1.GetOptions{}) + if err != nil { + t.Fatalf("NvSnapFunctionState/fv-never-seen was NOT created: %v (chicken-and-egg regression)", err) + } + state, _, _ := unstructured.NestedString(got.Object, "status", "localCacheState") + if state != string(nvsnapv1alpha1.LocalCacheStateCold) { + t.Errorf("initial CFS state = %q, want Cold", state) + } + fv, _, _ := unstructured.NestedString(got.Object, "spec", "functionVersionID") + if fv != "fv-never-seen" { + t.Errorf("CFS spec.functionVersionID = %q, want fv-never-seen", fv) + } +} + +func TestStampEmptyFunctionVersionDoesNothing(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + c := newHookTestBackend(t) + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq(""), logrus.NewEntry(logrus.New())) + if len(pod.Annotations) != 0 { + t.Errorf("no FV id should leave pod unchanged, got %v", pod.Annotations) + } +} + +func TestStampWarmFetchingNotYetReady(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + + // Hash is set (NGC says it exists) but bytes are still in + // flight to this cluster. Must not stamp restore-from yet — + // stamping while Fetching means the webhook tries to resolve a + // manifest that may not exist locally yet. + c := newHookTestBackend(t, + nvsnapFunctionStateUnstructured("fv-1", false, "deadbeef", nvsnapv1alpha1.LocalCacheStateFetching), + ) + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + if _, ok := pod.Annotations[NvSnapRestoreFromAnnotation]; ok { + t.Error("restore-from should NOT be stamped when cache state is Fetching") + } + // checkpoint-on-warm IS stamped — this pod can still be a + // fresh-capture target if the FV has none, or a re-checkpoint + // target if NGC has a hash but we want to refresh. + if got := pod.Annotations[NvSnapCheckpointOnWarmAnnotation]; got != "true" { + t.Errorf("checkpoint-on-warm should still be stamped during Fetching, got %q", got) + } +} + +var _ = metav1.Now // silence unused import; left in scope for future tests that exercise CapturedAt + +// nvsnapServerStub returns a fake nvsnap-server that replies with `code` +// to GetCheckpoint (GET /api/v1/checkpoints/{id}). +func nvsnapServerStub(t *testing.T, code int) string { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(code) + if code == http.StatusOK { + _, _ = w.Write([]byte(`{"id":"deadbeef","status":"Completed","hash":"deadbeef"}`)) + } + })) + t.Cleanup(srv.Close) + return srv.URL +} + +// TestStampWarm_ArtifactMissing_ResetsAndRecaptures (nvca#190): when CFS +// says Warm but nvsnap-server has no artifact for the hash (deleted out from +// under NVCA), Hook A must NOT stamp restore-from — it must reset CFS to +// Cold and stamp checkpoint-on-warm so Hook B re-captures. +func TestStampWarm_ArtifactMissing_ResetsAndRecaptures(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + c := newHookTestBackend(t, + nvsnapFunctionStateUnstructured("fv-1", false, "deadbeef", nvsnapv1alpha1.LocalCacheStateWarm), + ) + c.nvsnapClient = nvsnap.NewClient(nvsnap.WithBaseURL(nvsnapServerStub(t, http.StatusNotFound))) + + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + + if got, ok := pod.Annotations[NvSnapRestoreFromAnnotation]; ok { + t.Errorf("restore-from must NOT be stamped for a deleted artifact, got %q", got) + } + if got := pod.Annotations[NvSnapCheckpointOnWarmAnnotation]; got != "true" { + t.Errorf("checkpoint-on-warm should be stamped to re-capture, got %q", got) + } + cfs, err := c.dynClient.Resource(nvsnapFunctionStateGVR).Get(context.Background(), "fv-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get CFS: %v", err) + } + state, _, _ := unstructured.NestedString(cfs.Object, "status", "localCacheState") + if state != string(nvsnapv1alpha1.LocalCacheStateCold) { + t.Errorf("CFS localCacheState = %q, want Cold (reset)", state) + } +} + +// TestStampWarm_ArtifactCheckErrorFailsOpen: a transient nvsnap-server error +// (non-404) must NOT invalidate a Warm CFS — Hook A fails open and still +// stamps restore-from so valid restores aren't broken by a blip. +func TestStampWarm_ArtifactCheckErrorFailsOpen(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + c := newHookTestBackend(t, + nvsnapFunctionStateUnstructured("fv-1", false, "deadbeef", nvsnapv1alpha1.LocalCacheStateWarm), + ) + c.nvsnapClient = nvsnap.NewClient(nvsnap.WithBaseURL(nvsnapServerStub(t, http.StatusInternalServerError))) + + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + + if got := pod.Annotations[NvSnapRestoreFromAnnotation]; got != "deadbeef" { + t.Errorf("fail-open: restore-from should still be stamped on a transient error, got %q", got) + } +} + +// TestStampWarm_ArtifactExists_StampsRestoreFrom: the verification passes +// (artifact present) → normal Warm fast path stamps restore-from. +func TestStampWarm_ArtifactExists_StampsRestoreFrom(t *testing.T) { + defer withFlagEnabled(t, "NvSnapCheckpointRestore")() + c := newHookTestBackend(t, + nvsnapFunctionStateUnstructured("fv-1", false, "deadbeef", nvsnapv1alpha1.LocalCacheStateWarm), + ) + c.nvsnapClient = nvsnap.NewClient(nvsnap.WithBaseURL(nvsnapServerStub(t, http.StatusOK))) + + pod := &corev1.Pod{} + c.stampNvSnapAnnotations(context.Background(), pod, newReq("fv-1"), logrus.NewEntry(logrus.New())) + + if got := pod.Annotations[NvSnapRestoreFromAnnotation]; got != "deadbeef" { + t.Errorf("restore-from = %q, want deadbeef (artifact exists)", got) + } + if _, ok := pod.Annotations[NvSnapCheckpointOnWarmAnnotation]; ok { + t.Error("checkpoint-on-warm must NOT be stamped when restoring") + } +} diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel index b961c0786..ab28ded7a 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel @@ -42,6 +42,7 @@ go_library( "manifests/otel_collector_config.yaml", "manifests/rbacTemplate.yaml", "manifests/vault_config_template.hcl", + "manifests/nvsnap.nvcf.nvidia.io_nvsnapfunctionstates_crd.yaml", ], importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/reconcile", visibility = ["//visibility:public"], diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile.go index 8a6548c75..8c6233863 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile.go @@ -35,13 +35,16 @@ var ( storageRequestsCRDData []byte //go:embed manifests/nvcf.nvidia.io_miniservices_crd.yaml miniserviceCRDData []byte + //go:embed manifests/nvsnap.nvcf.nvidia.io_nvsnapfunctionstates_crd.yaml + nvsnapFunctionStateCRDData []byte ) const ( - StorageRequestCRDName = "storagerequests.nvca.nvcf.nvidia.io" - MiniServicesCRDName = "miniservices.nvca.nvcf.nvidia.io" - ICMSRequestCRDName = "icmsrequests.nvca.nvcf.nvidia.io" - NVCFBackendCRDName = "nvcfbackends.nvcf.nvidia.io" + StorageRequestCRDName = "storagerequests.nvca.nvcf.nvidia.io" + MiniServicesCRDName = "miniservices.nvca.nvcf.nvidia.io" + ICMSRequestCRDName = "icmsrequests.nvca.nvcf.nvidia.io" + NVCFBackendCRDName = "nvcfbackends.nvcf.nvidia.io" + NvSnapFunctionStateCRDName = "nvsnapfunctionstates.nvsnap.nvcf.nvidia.io" ) func (c *BackendK8sCache) setupCRDs(ctx context.Context) error { @@ -55,6 +58,10 @@ func (c *BackendK8sCache) setupCRDs(ctx context.Context) error { if err != nil { return fmt.Errorf("make MiniService CRD: %v", err) } + nvsnapFunctionStateCRD, err := decodeCRD(nvsnapFunctionStateCRDData) + if err != nil { + return fmt.Errorf("make NvSnapFunctionState CRD: %v", err) + } // Get the NVCFBackend CRD to use as owner reference. // This ensures operator-managed CRDs are garbage collected when Helm uninstalls. @@ -87,6 +94,7 @@ func (c *BackendK8sCache) setupCRDs(ctx context.Context) error { storageReqCRD, miniserviceCRD, makeICMSRequestCRD(), + nvsnapFunctionStateCRD, } { // Set owner reference so CRDs are garbage collected when NVCFBackend CRD is deleted if ownerRef != nil { diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/nvsnap.nvcf.nvidia.io_nvsnapfunctionstates_crd.yaml b/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/nvsnap.nvcf.nvidia.io_nvsnapfunctionstates_crd.yaml new file mode 100644 index 000000000..f080d18d5 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/nvsnap.nvcf.nvidia.io_nvsnapfunctionstates_crd.yaml @@ -0,0 +1,52 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: nvsnapfunctionstates.nvsnap.nvcf.nvidia.io +spec: + group: nvsnap.nvcf.nvidia.io + names: + kind: NvSnapFunctionState + listKind: NvSnapFunctionStateList + plural: nvsnapfunctionstates + singular: nvsnapfunctionstate + shortNames: + - cfs + scope: Cluster + versions: + - name: v1alpha1 + additionalPrinterColumns: + - name: Function + jsonPath: .spec.functionVersionID + description: NGC function-version UUID + type: string + - name: Cache + jsonPath: .status.localCacheState + description: Per-cluster cache readiness (Cold | Fetching | Warm | Failed) + type: string + - name: Hash + jsonPath: .status.checkpointHash + description: Content-addressed checkpoint hash (first 12 chars displayed) + type: string + - name: Attempts + jsonPath: .status.attemptCount + description: Cumulative checkpoint attempts + type: integer + - name: Age + jsonPath: .metadata.creationTimestamp + type: date + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + served: true + storage: true + subresources: + status: {} diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go index 80c09c8f6..bfeee59c1 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go @@ -800,6 +800,19 @@ func (bc *BackendK8sCache) setupNVCARBAC(ctx context.Context, nb *nvidiaiov1.NVC ResourceNames: []string{nvcaoptypes.NVCAModuleName}, Verbs: []string{"get", "list", "watch"}, }, + // NvSnap integration (PR-3 + PR-5): NVCA agent reads and + // writes NvSnapFunctionState — cluster-scoped CRD that tracks + // per-function-version checkpoint state. Read in Hook A to + // decide whether to stamp nvsnap.io/restore-from at pod + // apply; written by Hook B's reconciler after a successful + // checkpoint. The integration is gated behind + // featureflag.NvSnapCheckpointRestore (default off), so this + // rule is dormant until that flag is enabled on a cluster. + { + APIGroups: []string{"nvsnap.nvcf.nvidia.io"}, + Resources: []string{"nvsnapfunctionstates", "nvsnapfunctionstates/status"}, + Verbs: crudVerbs, + }, }, } diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go index 504d97df2..1d0b717c4 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go @@ -1596,6 +1596,11 @@ func Test_setupNVCARBAC(t *testing.T) { ResourceNames: []string{nvcaoptypes.NVCAModuleName}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{"nvsnap.nvcf.nvidia.io"}, + Resources: []string{"nvsnapfunctionstates", "nvsnapfunctionstates/status"}, + Verbs: []string{"get", "list", "watch", "create", "update", "delete", "patch"}, + }, {}, // Node rule, added below { APIGroups: []string{""}, @@ -2011,6 +2016,11 @@ func Test_setupNVCARBAC_ValidationPolicy(t *testing.T) { ResourceNames: []string{nvcaoptypes.NVCAModuleName}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{"nvsnap.nvcf.nvidia.io"}, + Resources: []string{"nvsnapfunctionstates", "nvsnapfunctionstates/status"}, + Verbs: []string{"get", "list", "watch", "create", "update", "delete", "patch"}, + }, { APIGroups: []string{""}, Resources: []string{"nodes"}, @@ -2238,6 +2248,11 @@ func Test_NVLinkOptimized(t *testing.T) { ResourceNames: []string{nvcaoptypes.NVCAModuleName}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{"nvsnap.nvcf.nvidia.io"}, + Resources: []string{"nvsnapfunctionstates", "nvsnapfunctionstates/status"}, + Verbs: []string{"get", "list", "watch", "create", "update", "delete", "patch"}, + }, { APIGroups: []string{""}, Resources: []string{"nodes"}, diff --git a/src/compute-plane-services/nvca/scripts/codegen_update b/src/compute-plane-services/nvca/scripts/codegen_update index e26336b42..21915b7f5 100755 --- a/src/compute-plane-services/nvca/scripts/codegen_update +++ b/src/compute-plane-services/nvca/scripts/codegen_update @@ -79,6 +79,7 @@ main() { deepcopy "nvca/v1" deepcopy "nvca/v1alpha1" deepcopy "nvca/v2beta1" + deepcopy "nvsnap/v1alpha1" clientgen_all }