From 22c1e899ada3c6d2b1d3cc79debe2a4d7e67961e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 10:19:09 -0400 Subject: [PATCH 01/13] test(e2e): mark test-onboard-resume.sh as bridge-probe in inventory Begin migration of test/e2e/test-onboard-resume.sh into the typed Vitest E2E scenario framework as a free-standing live test under test/e2e-scenario/live/onboard-resume.test.ts. Refs: #4348, #5098 --- test/e2e-scenario/migration/legacy-inventory.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index e66981d726..18ebf60458 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -523,12 +523,12 @@ "legacyScript": "test/e2e/test-onboard-resume.sh", "domain": "smoke-onboarding", "ownerIssue": "#4348", - "status": "not-migrated", - "targetVitestScenarios": [], + "status": "bridge-probe", + "targetVitestScenarios": ["test/e2e-scenario/live/onboard-resume.test.ts"], "bridgeProbes": [], "retiredReason": "", "deletionReady": false, - "notes": "Initial completeness row; classify detailed coverage and deletion evidence in the owning migration issue before deleting." + "notes": "Migration in progress (epic #5098, owner #4348): typed live test at test/e2e-scenario/live/onboard-resume.test.ts. Bash script retained until typed coverage soaks per epic policy; deletion is a follow-up PR with #4357 approval." }, { "legacyScript": "test/e2e/test-openclaw-discord-pairing.sh", From 8705dbbce60ea76bc351553844cda66538eaf96d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 10:21:56 -0400 Subject: [PATCH 02/13] test(e2e): add SandboxClient.get and SandboxClient.exists helpers Mirrors `openshell sandbox get ` exit-code probe from test/e2e/test-onboard-resume.sh. Lives on SandboxClient (not host) per fix-location convention: openshell-area helpers belong with the existing sandbox.list / sandbox.status wrappers. `exists()` returns boolean from exit code so callers can branch on sandbox presence without bridging through assertExitZero. Refs: #4348, #5098 --- .../framework-tests/e2e-clients.test.ts | 41 +++++++++++++++++++ .../e2e-scenario/framework/clients/sandbox.ts | 13 ++++++ 2 files changed, 54 insertions(+) diff --git a/test/e2e-scenario/framework-tests/e2e-clients.test.ts b/test/e2e-scenario/framework-tests/e2e-clients.test.ts index 77e8824fe2..98b320b575 100644 --- a/test/e2e-scenario/framework-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-clients.test.ts @@ -175,6 +175,47 @@ describe("E2E fixture clients", () => { }); }); + it("sandbox client builds OpenShell sandbox get commands", async () => { + const runner = new FakeRunner(); + const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); + + await sandbox.get("e2e-resume"); + + expect(runner.calls[0]).toEqual({ + command: "openshell", + args: ["sandbox", "get", "e2e-resume"], + options: { + artifactName: "sandbox-get-e2e-resume", + }, + }); + }); + + it("sandbox client exists() returns true on exit 0", async () => { + const runner = new FakeRunner(); + runner.exitCode = 0; + const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); + + expect(await sandbox.exists("e2e-resume")).toBe(true); + expect(runner.calls[0].args).toEqual(["sandbox", "get", "e2e-resume"]); + }); + + it("sandbox client exists() returns false on non-zero exit", async () => { + const runner = new FakeRunner(); + runner.exitCode = 1; + runner.stderr = "sandbox not found"; + const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); + + expect(await sandbox.exists("missing")).toBe(false); + }); + + it("sandbox client get() rejects flag-shaped sandbox names before command construction", async () => { + const runner = new FakeRunner(); + const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); + + await expect(() => sandbox.get("--bad")).toThrow(/sandbox name is invalid/); + expect(runner.calls).toEqual([]); + }); + it("sandbox client rejects flag-shaped sandbox names before command construction", async () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); diff --git a/test/e2e-scenario/framework/clients/sandbox.ts b/test/e2e-scenario/framework/clients/sandbox.ts index 795060dad3..45f4eee7ec 100644 --- a/test/e2e-scenario/framework/clients/sandbox.ts +++ b/test/e2e-scenario/framework/clients/sandbox.ts @@ -44,6 +44,19 @@ export class SandboxClient { }); } + get(name: string, options: ShellProbeRunOptions = {}): Promise { + validateSandboxName(name); + return this.openshell(["sandbox", "get", name], { + artifactName: `sandbox-get-${name}`, + ...options, + }); + } + + async exists(name: string, options: ShellProbeRunOptions = {}): Promise { + const result = await this.get(name, options); + return result.exitCode === 0; + } + exec( name: string, command: string[], From 49755fa538b04fbfe0bf5250cf0488e0b5b27261 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 10:23:40 -0400 Subject: [PATCH 03/13] ci(e2e): add onboard-resume-vitest job to e2e-vitest-scenarios Discrete free-standing job for the typed migration of test/e2e/test-onboard-resume.sh, mirroring the openshell-version-pin-vitest precedent from #5107. Uses bash install.sh to bootstrap openshell CLI and node, then runs the live Vitest scenario with NVIDIA_API_KEY exposed. Bash script remains scheduled under nightly-e2e `onboard-resume-e2e`; this job runs only on workflow_dispatch until typed coverage soaks per epic #5098 policy. Refs: #4348, #5098 --- .github/workflows/e2e-vitest-scenarios.yaml | 55 +++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 26b60608ab..8121fac939 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -206,3 +206,58 @@ jobs: include-hidden-files: false if-no-files-found: ignore retention-days: 14 + + # Migrated from test/e2e/test-onboard-resume.sh — regression for #446. + # Disruption-recovery shape (interrupt onboard via NEMOCLAW_E2E_FAILURE_INJECTION, + # then verify --resume completes without rerunning preflight/gateway/sandbox). + # Free-standing per #5049/#5107 precedent: doesn't fit the steady-state + # registry-driven expected-state probe model. Bash script kept in nightly-e2e + # under onboard-resume-e2e until typed coverage soaks (epic #5098 policy). + onboard-resume-vitest: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/onboard-resume + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Install NemoClaw (provides openshell CLI + node bootstrap) + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash install.sh --non-interactive --yes-i-accept-third-party-software + + - name: Install root dev dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run onboard-resume live test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: | + set -euo pipefail + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/onboard-resume.test.ts \ + --silent=false --reporter=default + + - name: Upload onboard-resume artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-onboard-resume + path: e2e-artifacts/vitest/onboard-resume/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 From 75d5b2c6faa23f1cbb9fa34d2880befad436a985 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 10:26:19 -0400 Subject: [PATCH 04/13] test(e2e): add typed live test for onboard-resume regression (#446) Migrates the assertion contracts from test/e2e/test-onboard-resume.sh into a free-standing Vitest live test under test/e2e-scenario/live/. Drives the real `nemoclaw onboard` CLI via NEMOCLAW_E2E_FAILURE_INJECTION to deterministically fail at the policies step, then runs `nemoclaw onboard --resume --non-interactive` with NVIDIA_API_KEY stripped from the environment to prove credential hydration from the session file. 22 assertions, organized into Phase 1 (prereqs), Phase 2 (interrupted onboard), and Phase 3 (resume completion). Uses SandboxClient.exists (added in prior commit) for the `openshell sandbox get` probe; reads the onboard session JSON inline for state assertions; registers sandbox/session cleanup with the cleanup fixture. Bash script at test/e2e/test-onboard-resume.sh kept untouched per epic #5098 suite-separation rule until typed coverage soaks; it remains scheduled in nightly-e2e.yaml under onboard-resume-e2e. Refs: #446, #4348, #5098 --- test/e2e-scenario/live/onboard-resume.test.ts | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 test/e2e-scenario/live/onboard-resume.test.ts diff --git a/test/e2e-scenario/live/onboard-resume.test.ts b/test/e2e-scenario/live/onboard-resume.test.ts new file mode 100644 index 0000000000..6ff78479ee --- /dev/null +++ b/test/e2e-scenario/live/onboard-resume.test.ts @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { expect, test } from "../framework/e2e-test.ts"; + +// Migrated from test/e2e/test-onboard-resume.sh — regression for #446. +// +// Disruption-recovery shape: drives the real `nemoclaw onboard` CLI through +// the deterministic E2E failure-injection hook +// (NEMOCLAW_E2E_FAILURE_INJECTION + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP), then +// invokes `nemoclaw onboard --resume --non-interactive` with NVIDIA_API_KEY +// stripped from the environment to prove the credential is hydrated from the +// onboard session file. +// +// Free-standing per #5049/#5107 precedent: the steady-state expected-state +// probe model in expected-states.ts does not capture log-grep contracts +// ("[resume] Skipping preflight (cached)") or the JSON-shape of an interrupted +// onboard session. Asserts inline, helpers-not-bridges. +// +// The legacy bash workflow (`onboard-resume-e2e` in nightly-e2e.yaml) is kept +// untouched per epic #5098 suite-separation rule until typed coverage soaks. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json"); +const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-resume"; + +// 15 minutes per onboard run; matches NEMOCLAW_E2E_DEFAULT_TIMEOUT in the +// legacy bash test (`export NEMOCLAW_E2E_DEFAULT_TIMEOUT=600` is per-step; +// the full onboard sequence dominates). +const ONBOARD_TIMEOUT_MS = 15 * 60_000; + +interface SessionStateInterrupted { + status: "failed"; + lastCompletedStep: "openclaw"; + failure: { step: "policies" }; +} + +interface SessionStateComplete { + status: "complete"; + provider: "nvidia-prod"; + steps: Record< + "preflight" | "gateway" | "sandbox" | "provider_selection" | "inference" | "openclaw" | "policies", + { status: "complete" } + >; +} + +function readSession(file: string): T { + return JSON.parse(fs.readFileSync(file, "utf8")) as T; +} + +test("onboard-resume: interrupted onboard then --resume completes without redoing cached steps", async ({ + artifacts, + cleanup, + host, + sandbox, + secrets, +}) => { + // ────────────────────────────────────────────────────────────────── + // Phase 1: prerequisites (host-side, all faithful on ubuntu-latest) + // ────────────────────────────────────────────────────────────────── + + // Assertion: cli-built — `bin/nemoclaw.js` exists in the repo checkout. + expect( + fs.existsSync(CLI_ENTRYPOINT), + `bin/nemoclaw.js missing — ensure the workflow runs npm ci + npm run build:cli before this test`, + ).toBe(true); + + // Assertion: docker-running — `docker info` exits 0. + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info", + timeoutMs: 30_000, + }); + expect(dockerInfo.exitCode, dockerInfo.stderr).toBe(0); + + // Assertion: openshell-installed — openshell CLI is on PATH (installed by + // the workflow's `bash install.sh` step before this test runs). + const openshellVersion = await host.command("openshell", ["--version"], { + artifactName: "prereq-openshell-version", + timeoutMs: 30_000, + }); + expect(openshellVersion.exitCode, openshellVersion.stderr).toBe(0); + + // Assertion: nvidia-api-key-present — secrets.required(...) skips the test + // if NVIDIA_API_KEY is unset (correct behavior under workflow_dispatch + // without the secret wired in). + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey).toMatch(/^nvapi-/); + + // ────────────────────────────────────────────────────────────────── + // Phase 0 (deferred): pre-cleanup of leftover sandbox/session state. + // Done after the prereq gates pass so we don't mutate host state if + // the test would have skipped anyway. + // ────────────────────────────────────────────────────────────────── + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy", + timeoutMs: 60_000, + }); + await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-sandbox-delete", + timeoutMs: 60_000, + }); + await sandbox.openshell(["forward", "stop", "18789"], { + artifactName: "pre-cleanup-openshell-forward-stop", + timeoutMs: 30_000, + }); + await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "pre-cleanup-openshell-gateway-destroy", + timeoutMs: 60_000, + }); + fs.rmSync(SESSION_FILE, { force: true }); + + // Register cleanup for the sandbox we are about to create. The cleanup + // fixture runs these in LIFO at end-of-test regardless of pass/fail. + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy", + timeoutMs: 120_000, + }); + await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete", + timeoutMs: 60_000, + }); + await sandbox.openshell(["forward", "stop", "18789"], { + artifactName: "cleanup-openshell-forward-stop", + timeoutMs: 30_000, + }); + await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy", + timeoutMs: 60_000, + }); + fs.rmSync(SESSION_FILE, { force: true }); + }); + + // ────────────────────────────────────────────────────────────────── + // Phase 2: first onboard (forced failure at the policies step) + // ────────────────────────────────────────────────────────────────── + const firstRun = await host.command("node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { + artifactName: "phase-2-onboard-interrupted", + env: { + NVIDIA_API_KEY: apiKey, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_POLICY_MODE: "suggested", + NEMOCLAW_E2E_FAILURE_INJECTION: "1", + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", + }, + inheritEnv: true, + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + const firstText = `${firstRun.stdout}\n${firstRun.stderr}`; + + // Assertion: interrupted-exit-1. + expect(firstRun.exitCode, firstText).toBe(1); + + // Assertion: sandbox-created-log. + expect(firstText).toContain(`Sandbox '${SANDBOX_NAME}' created`); + + // Assertion: forced-failure-log — failure injection fired at the policies step. + expect(firstText).toContain("[e2e] Forced onboarding failure at step 'policies'."); + + // Assertion: sandbox-exists-after-interrupt — `openshell sandbox get` exits 0. + expect(await sandbox.exists(SANDBOX_NAME)).toBe(true); + + // Assertion: session-file-present. + expect(fs.existsSync(SESSION_FILE)).toBe(true); + + // Assertion: session-file-interrupted-state. + const interrupted = readSession(SESSION_FILE); + await artifacts.writeJson("phase-2-session-state.json", interrupted); + expect(interrupted.status).toBe("failed"); + expect(interrupted.lastCompletedStep).toBe("openclaw"); + expect(interrupted.failure?.step).toBe("policies"); + + // ────────────────────────────────────────────────────────────────── + // Phase 3: resume — NVIDIA_API_KEY removed from env so the resume run + // must hydrate the credential from the session file. + // ────────────────────────────────────────────────────────────────── + const resumeRun = await host.command( + "node", + [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"], + { + artifactName: "phase-3-onboard-resume", + env: { + // NVIDIA_API_KEY intentionally absent. + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_POLICY_MODE: "skip", + }, + inheritEnv: true, + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + const resumeText = `${resumeRun.stdout}\n${resumeRun.stderr}`; + + // Assertion: resume-exit-0. + expect(resumeRun.exitCode, resumeText).toBe(0); + + // Assertion: resume-skipped-{preflight,gateway,sandbox}-log. + expect(resumeText).toContain("[resume] Skipping preflight (cached)"); + expect(resumeText).toContain("[resume] Skipping gateway (running)"); + expect(resumeText).toContain(`[resume] Skipping sandbox (${SANDBOX_NAME})`); + + // Assertion: resume-no-{preflight,gateway,sandbox}-rerun. + expect(resumeText).not.toContain("[1/7] Preflight checks"); + expect(resumeText).not.toContain("[2/7] Starting OpenShell gateway"); + expect(resumeText).not.toContain("[5/7] Creating sandbox"); + + // Assertion: resume-inference-handled — first onboard completed through + // openclaw (step 7) before failing at policies (step 8). Inference was + // already configured during that run, so the resume path either re-runs + // it or detects readiness and skips. Both are valid. + const ranInference = resumeText.includes("[4/7] Setting up inference provider"); + const skippedInference = + resumeText.includes("[resume] Skipping inference") || + resumeText.includes("[reuse] Skipping inference"); + expect(ranInference || skippedInference, resumeText).toBe(true); + + // Assertion: sandbox-manageable-after-resume. + const sandboxStatus = await host.command( + "node", + [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], + { artifactName: "phase-3-nemoclaw-status", timeoutMs: 60_000 }, + ); + expect(sandboxStatus.exitCode, sandboxStatus.stderr).toBe(0); + + // Assertion: session-file-complete-state. + const complete = readSession(SESSION_FILE); + await artifacts.writeJson("phase-3-session-state.json", complete); + expect(complete.status).toBe("complete"); + expect(complete.provider).toBe("nvidia-prod"); + for (const step of [ + "preflight", + "gateway", + "sandbox", + "provider_selection", + "inference", + "openclaw", + "policies", + ] as const) { + expect(complete.steps[step]?.status, `step ${step}`).toBe("complete"); + } + + // Assertion: registry-has-sandbox. + expect(fs.existsSync(REGISTRY_FILE)).toBe(true); + expect(fs.readFileSync(REGISTRY_FILE, "utf8")).toContain(SANDBOX_NAME); +}); From debb4b2521574bbbf852c88d3defa2a054121b08 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 10:45:15 -0400 Subject: [PATCH 05/13] test(e2e): pass framework env to onboard-resume spawns to fix ENOENT Pass 1 of phase-5 convergence on dispatch 27283289318. First blocking assertion: openshell-installed (#3 in chain), failing with `spawn openshell ENOENT`. Root cause: shell-probe.ts:127-129 \u2014 when inheritEnv is unset, the child gets only options.env. My prereq probes passed no env, so spawn had no PATH to resolve `openshell` against. The onboard runs worked only because they used inheritEnv:true. Fix: use buildAvailabilityProbeEnv() everywhere (framework allowlist incl. PATH/HOME/CI; explicitly excludes NVIDIA_API_KEY). Layer the secret explicitly only on the first onboard; the resume run's env deliberately omits it to test credential hydration from the session file \u2014 this is the typed expression of the bash test's `env -u NVIDIA_API_KEY` invariant. Also drops inheritEnv:true on the onboard runs in favor of the same allowlist composition pattern, matching OnboardingPhaseFixture.commandEnv. Refs: #4348, #5098 --- test/e2e-scenario/live/onboard-resume.test.ts | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/test/e2e-scenario/live/onboard-resume.test.ts b/test/e2e-scenario/live/onboard-resume.test.ts index 6ff78479ee..ca11820842 100644 --- a/test/e2e-scenario/live/onboard-resume.test.ts +++ b/test/e2e-scenario/live/onboard-resume.test.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { buildAvailabilityProbeEnv } from "../framework/availability-env.ts"; import { expect, test } from "../framework/e2e-test.ts"; // Migrated from test/e2e/test-onboard-resume.sh — regression for #446. @@ -71,9 +72,15 @@ test("onboard-resume: interrupted onboard then --resume completes without redoin `bin/nemoclaw.js missing — ensure the workflow runs npm ci + npm run build:cli before this test`, ).toBe(true); - // Assertion: docker-running — `docker info` exits 0. + // Assertion: docker-running — `docker info` exits 0. Pass framework + // allowlist env (includes PATH, HOME, etc.) so spawn can locate `docker`. + // The shell-probe boundary defaults to no env inheritance; framework spawns + // must opt in via buildAvailabilityProbeEnv() to keep secret-passthrough + // explicit (NVIDIA_API_KEY is NOT in the allowlist; we layer it explicitly + // in Phase 2 below). const dockerInfo = await host.command("docker", ["info"], { artifactName: "prereq-docker-info", + env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); expect(dockerInfo.exitCode, dockerInfo.stderr).toBe(0); @@ -82,6 +89,7 @@ test("onboard-resume: interrupted onboard then --resume completes without redoin // the workflow's `bash install.sh` step before this test runs). const openshellVersion = await host.command("openshell", ["--version"], { artifactName: "prereq-openshell-version", + env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); expect(openshellVersion.exitCode, openshellVersion.stderr).toBe(0); @@ -97,20 +105,25 @@ test("onboard-resume: interrupted onboard then --resume completes without redoin // Done after the prereq gates pass so we don't mutate host state if // the test would have skipped anyway. // ────────────────────────────────────────────────────────────────── + const probeEnv = buildAvailabilityProbeEnv(); await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { artifactName: "pre-cleanup-nemoclaw-destroy", + env: probeEnv, timeoutMs: 60_000, }); await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "pre-cleanup-openshell-sandbox-delete", + env: probeEnv, timeoutMs: 60_000, }); await sandbox.openshell(["forward", "stop", "18789"], { artifactName: "pre-cleanup-openshell-forward-stop", + env: probeEnv, timeoutMs: 30_000, }); await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "pre-cleanup-openshell-gateway-destroy", + env: probeEnv, timeoutMs: 60_000, }); fs.rmSync(SESSION_FILE, { force: true }); @@ -118,20 +131,25 @@ test("onboard-resume: interrupted onboard then --resume completes without redoin // Register cleanup for the sandbox we are about to create. The cleanup // fixture runs these in LIFO at end-of-test regardless of pass/fail. cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + const cleanupEnv = buildAvailabilityProbeEnv(); await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { artifactName: "cleanup-nemoclaw-destroy", + env: cleanupEnv, timeoutMs: 120_000, }); await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "cleanup-openshell-sandbox-delete", + env: cleanupEnv, timeoutMs: 60_000, }); await sandbox.openshell(["forward", "stop", "18789"], { artifactName: "cleanup-openshell-forward-stop", + env: cleanupEnv, timeoutMs: 30_000, }); await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "cleanup-openshell-gateway-destroy", + env: cleanupEnv, timeoutMs: 60_000, }); fs.rmSync(SESSION_FILE, { force: true }); @@ -143,16 +161,14 @@ test("onboard-resume: interrupted onboard then --resume completes without redoin const firstRun = await host.command("node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { artifactName: "phase-2-onboard-interrupted", env: { + ...buildAvailabilityProbeEnv(), NVIDIA_API_KEY: apiKey, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, NEMOCLAW_RECREATE_SANDBOX: "1", NEMOCLAW_POLICY_MODE: "suggested", NEMOCLAW_E2E_FAILURE_INJECTION: "1", NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", }, - inheritEnv: true, redactionValues: [apiKey], timeoutMs: ONBOARD_TIMEOUT_MS, }); @@ -189,14 +205,16 @@ test("onboard-resume: interrupted onboard then --resume completes without redoin [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"], { artifactName: "phase-3-onboard-resume", + // buildAvailabilityProbeEnv() does NOT pass NVIDIA_API_KEY through — + // it's outside the framework allowlist. Resume must hydrate the + // credential from the session file. This is exactly the bash test's + // `env -u NVIDIA_API_KEY` invariant, expressed via the framework's + // explicit secret-passthrough rule. env: { - // NVIDIA_API_KEY intentionally absent. - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + ...buildAvailabilityProbeEnv(), NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, NEMOCLAW_POLICY_MODE: "skip", }, - inheritEnv: true, redactionValues: [apiKey], timeoutMs: ONBOARD_TIMEOUT_MS, }, @@ -230,7 +248,11 @@ test("onboard-resume: interrupted onboard then --resume completes without redoin const sandboxStatus = await host.command( "node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], - { artifactName: "phase-3-nemoclaw-status", timeoutMs: 60_000 }, + { + artifactName: "phase-3-nemoclaw-status", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, ); expect(sandboxStatus.exitCode, sandboxStatus.stderr).toBe(0); From 3827a96cb38b56d07be950c9f7923ee9b5df1d06 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 10:55:52 -0400 Subject: [PATCH 06/13] test(e2e): pass framework env to sandbox.exists() in onboard-resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass 2 of phase-5 convergence on dispatch 27284469157. Line moved forward (75ms → 127s) clearing all Phase 1 prereqs and the Phase 2 first onboard. New blocking assertion: sandbox-exists-after-interrupt (#9 in chain), failing again with `spawn openshell ENOENT` \u2014 same root cause class as pass 1, missed during the previous fix because the sandbox.exists() call passed no options so options.env was empty. Fix: thread `env: buildAvailabilityProbeEnv()` into the exists() call. The SandboxClient correctly does not auto-supply env (mirrors HostCliClient.expectNemoclawAvailable convention \u2014 callers stay explicit about the env boundary, helpers-not-bridges per #5049). Refs: #4348, #5098 --- test/e2e-scenario/live/onboard-resume.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario/live/onboard-resume.test.ts b/test/e2e-scenario/live/onboard-resume.test.ts index ca11820842..2325ca9588 100644 --- a/test/e2e-scenario/live/onboard-resume.test.ts +++ b/test/e2e-scenario/live/onboard-resume.test.ts @@ -184,7 +184,11 @@ test("onboard-resume: interrupted onboard then --resume completes without redoin expect(firstText).toContain("[e2e] Forced onboarding failure at step 'policies'."); // Assertion: sandbox-exists-after-interrupt — `openshell sandbox get` exits 0. - expect(await sandbox.exists(SANDBOX_NAME)).toBe(true); + // Pass framework env so the spawn can locate `openshell` on PATH; the + // SandboxClient threads options through to ShellProbe but does not + // auto-supply env (mirrors HostCliClient — callers stay explicit about the + // env boundary). + expect(await sandbox.exists(SANDBOX_NAME, { env: buildAvailabilityProbeEnv() })).toBe(true); // Assertion: session-file-present. expect(fs.existsSync(SESSION_FILE)).toBe(true); From 4577fc5a9aa2a4ab29c63fdaa6712ddff5d7649a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 11:19:19 -0400 Subject: [PATCH 07/13] test(e2e): mark test-onboard-resume.sh as covered after typed migration Convergence achieved on PR #5147 (epic #5098, owner #4348). Two consecutive PASS dispatches confirmed the typed live test (test/e2e-scenario/live/onboard-resume.test.ts) reproduces the bash script's 22 assertions end-to-end: RED (pass 1, openshell-installed): runs/27283289318 RED (pass 2, sandbox-exists-after-i.): runs/27284469157 GREEN (pass 3, all 22 asserts, 144.3s): runs/27285136703 GREEN (verification, 140.5s): runs/27285846914 Inventory transitions to covered. Bash guard at test/e2e/test-onboard-resume.sh remains scheduled in nightly-e2e.yaml under onboard-resume-e2e until typed coverage soaks per epic policy; deletion is a follow-up PR with #4357 approval. Refs: #446, #4348, #5098 --- test/e2e-scenario/migration/legacy-inventory.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index 18ebf60458..ad4ebd0825 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -523,12 +523,17 @@ "legacyScript": "test/e2e/test-onboard-resume.sh", "domain": "smoke-onboarding", "ownerIssue": "#4348", - "status": "bridge-probe", + "status": "covered", "targetVitestScenarios": ["test/e2e-scenario/live/onboard-resume.test.ts"], "bridgeProbes": [], "retiredReason": "", "deletionReady": false, - "notes": "Migration in progress (epic #5098, owner #4348): typed live test at test/e2e-scenario/live/onboard-resume.test.ts. Bash script retained until typed coverage soaks per epic policy; deletion is a follow-up PR with #4357 approval." + "notes": "Covered by free-standing live test (PR #5147). Disruption-recovery shape: drives `nemoclaw onboard` via NEMOCLAW_E2E_FAILURE_INJECTION to fail at policies, then `nemoclaw onboard --resume --non-interactive` with NVIDIA_API_KEY stripped from env to prove credential hydration from the session file. 22 assertions translated 1:1 from the bash script (regression for #446). Not registry-driven; dispatched as the discrete onboard-resume-vitest job in e2e-vitest-scenarios.yaml. Bash guard retained in nightly-e2e.yaml under onboard-resume-e2e; deletion is a follow-up PR with #4357 approval.", + "convergenceEvidence": { + "redRunUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/27283289318", + "greenRunUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/27285136703", + "verificationRunUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/27285846914" + } }, { "legacyScript": "test/e2e/test-openclaw-discord-pairing.sh", From 5cdbee0b1a2e1ccfe88e47a75f967d38a3394edb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 11:38:11 -0400 Subject: [PATCH 08/13] test(e2e): gate onboard-resume live test on NEMOCLAW_RUN_E2E_SCENARIOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two PR-caused CI failures resolved in one batch (Batch-Safe Fix Policy: same-class mechanical fixes, low risk, testable together): 1. cli-test-shards (1) / cli-tests / checks: the cli vitest project's glob 'test/**/*.test.{js,ts}' picks up test/e2e-scenario/live/** even though the e2e-scenarios-live project gates that directory on NEMOCLAW_RUN_E2E_SCENARIOS=1. Without the env var, this test ran in cli-test-shards (1) and failed at the openshell-installed prereq probe with spawn ENOENT — runner has no real openshell, no Docker, no NVIDIA_API_KEY. Add test.skipIf(!shouldRunLiveE2EScenarios()) so the test self-skips when the live gate isn't set, mirroring the project-level include glob. 2. static-checks (Files were modified by following hooks: biome format): biome reformatted the test body. Reformat is mechanical only, no semantic change. Verified locally: - cli project (gate unset): 1 skipped, 0 failed - e2e-scenarios-live project: test still discovered and runs - framework-tests e2e-clients: 23/23 pass - prek pre-push hooks: all pass ShellCheck SARIF check is also red but is ambient (recent PRs alternate success/failure with the same 'Requires authentication' GH API error on the SARIF upload step; permissions in code-scanning.yaml are correct). Not addressed in this commit. Refs: #4348, #5098 --- test/e2e-scenario/live/onboard-resume.test.ts | 390 +++++++++--------- 1 file changed, 199 insertions(+), 191 deletions(-) diff --git a/test/e2e-scenario/live/onboard-resume.test.ts b/test/e2e-scenario/live/onboard-resume.test.ts index 2325ca9588..666e9926f8 100644 --- a/test/e2e-scenario/live/onboard-resume.test.ts +++ b/test/e2e-scenario/live/onboard-resume.test.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { buildAvailabilityProbeEnv } from "../framework/availability-env.ts"; import { expect, test } from "../framework/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../framework/live-project-gate.ts"; // Migrated from test/e2e/test-onboard-resume.sh — regression for #446. // @@ -46,7 +47,13 @@ interface SessionStateComplete { status: "complete"; provider: "nvidia-prod"; steps: Record< - "preflight" | "gateway" | "sandbox" | "provider_selection" | "inference" | "openclaw" | "policies", + | "preflight" + | "gateway" + | "sandbox" + | "provider_selection" + | "inference" + | "openclaw" + | "policies", { status: "complete" } >; } @@ -55,229 +62,230 @@ function readSession(file: string): T { return JSON.parse(fs.readFileSync(file, "utf8")) as T; } -test("onboard-resume: interrupted onboard then --resume completes without redoing cached steps", async ({ - artifacts, - cleanup, - host, - sandbox, - secrets, -}) => { - // ────────────────────────────────────────────────────────────────── - // Phase 1: prerequisites (host-side, all faithful on ubuntu-latest) - // ────────────────────────────────────────────────────────────────── +// Gate the test on NEMOCLAW_RUN_E2E_SCENARIOS=1 so it only runs under the +// `e2e-scenarios-live` Vitest project (dispatched by the +// onboard-resume-vitest workflow job). The `cli` project's glob +// `test/**/*.test.{js,ts}` would otherwise pick this file up in cli-test-shards +// where there's no real `openshell` CLI, no Docker daemon, and no +// NVIDIA_API_KEY — producing a guaranteed ENOENT/skip noise. Mirrors the +// gate enforced by the `e2e-scenarios-live` project's `include:` glob in +// vitest.config.ts; live-only tests opt in to that gate explicitly. +test.skipIf(!shouldRunLiveE2EScenarios())( + "onboard-resume: interrupted onboard then --resume completes without redoing cached steps", + async ({ artifacts, cleanup, host, sandbox, secrets }) => { + // ────────────────────────────────────────────────────────────────── + // Phase 1: prerequisites (host-side, all faithful on ubuntu-latest) + // ────────────────────────────────────────────────────────────────── - // Assertion: cli-built — `bin/nemoclaw.js` exists in the repo checkout. - expect( - fs.existsSync(CLI_ENTRYPOINT), - `bin/nemoclaw.js missing — ensure the workflow runs npm ci + npm run build:cli before this test`, - ).toBe(true); + // Assertion: cli-built — `bin/nemoclaw.js` exists in the repo checkout. + expect( + fs.existsSync(CLI_ENTRYPOINT), + `bin/nemoclaw.js missing — ensure the workflow runs npm ci + npm run build:cli before this test`, + ).toBe(true); - // Assertion: docker-running — `docker info` exits 0. Pass framework - // allowlist env (includes PATH, HOME, etc.) so spawn can locate `docker`. - // The shell-probe boundary defaults to no env inheritance; framework spawns - // must opt in via buildAvailabilityProbeEnv() to keep secret-passthrough - // explicit (NVIDIA_API_KEY is NOT in the allowlist; we layer it explicitly - // in Phase 2 below). - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - expect(dockerInfo.exitCode, dockerInfo.stderr).toBe(0); - - // Assertion: openshell-installed — openshell CLI is on PATH (installed by - // the workflow's `bash install.sh` step before this test runs). - const openshellVersion = await host.command("openshell", ["--version"], { - artifactName: "prereq-openshell-version", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - expect(openshellVersion.exitCode, openshellVersion.stderr).toBe(0); + // Assertion: docker-running — `docker info` exits 0. Pass framework + // allowlist env (includes PATH, HOME, etc.) so spawn can locate `docker`. + // The shell-probe boundary defaults to no env inheritance; framework spawns + // must opt in via buildAvailabilityProbeEnv() to keep secret-passthrough + // explicit (NVIDIA_API_KEY is NOT in the allowlist; we layer it explicitly + // in Phase 2 below). + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(dockerInfo.exitCode, dockerInfo.stderr).toBe(0); - // Assertion: nvidia-api-key-present — secrets.required(...) skips the test - // if NVIDIA_API_KEY is unset (correct behavior under workflow_dispatch - // without the secret wired in). - const apiKey = secrets.required("NVIDIA_API_KEY"); - expect(apiKey).toMatch(/^nvapi-/); + // Assertion: openshell-installed — openshell CLI is on PATH (installed by + // the workflow's `bash install.sh` step before this test runs). + const openshellVersion = await host.command("openshell", ["--version"], { + artifactName: "prereq-openshell-version", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(openshellVersion.exitCode, openshellVersion.stderr).toBe(0); - // ────────────────────────────────────────────────────────────────── - // Phase 0 (deferred): pre-cleanup of leftover sandbox/session state. - // Done after the prereq gates pass so we don't mutate host state if - // the test would have skipped anyway. - // ────────────────────────────────────────────────────────────────── - const probeEnv = buildAvailabilityProbeEnv(); - await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "pre-cleanup-nemoclaw-destroy", - env: probeEnv, - timeoutMs: 60_000, - }); - await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "pre-cleanup-openshell-sandbox-delete", - env: probeEnv, - timeoutMs: 60_000, - }); - await sandbox.openshell(["forward", "stop", "18789"], { - artifactName: "pre-cleanup-openshell-forward-stop", - env: probeEnv, - timeoutMs: 30_000, - }); - await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "pre-cleanup-openshell-gateway-destroy", - env: probeEnv, - timeoutMs: 60_000, - }); - fs.rmSync(SESSION_FILE, { force: true }); + // Assertion: nvidia-api-key-present — secrets.required(...) skips the test + // if NVIDIA_API_KEY is unset (correct behavior under workflow_dispatch + // without the secret wired in). + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey).toMatch(/^nvapi-/); - // Register cleanup for the sandbox we are about to create. The cleanup - // fixture runs these in LIFO at end-of-test regardless of pass/fail. - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - const cleanupEnv = buildAvailabilityProbeEnv(); + // ────────────────────────────────────────────────────────────────── + // Phase 0 (deferred): pre-cleanup of leftover sandbox/session state. + // Done after the prereq gates pass so we don't mutate host state if + // the test would have skipped anyway. + // ────────────────────────────────────────────────────────────────── + const probeEnv = buildAvailabilityProbeEnv(); await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy", - env: cleanupEnv, - timeoutMs: 120_000, + artifactName: "pre-cleanup-nemoclaw-destroy", + env: probeEnv, + timeoutMs: 60_000, }); await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete", - env: cleanupEnv, + artifactName: "pre-cleanup-openshell-sandbox-delete", + env: probeEnv, timeoutMs: 60_000, }); await sandbox.openshell(["forward", "stop", "18789"], { - artifactName: "cleanup-openshell-forward-stop", - env: cleanupEnv, + artifactName: "pre-cleanup-openshell-forward-stop", + env: probeEnv, timeoutMs: 30_000, }); await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "cleanup-openshell-gateway-destroy", - env: cleanupEnv, + artifactName: "pre-cleanup-openshell-gateway-destroy", + env: probeEnv, timeoutMs: 60_000, }); fs.rmSync(SESSION_FILE, { force: true }); - }); - // ────────────────────────────────────────────────────────────────── - // Phase 2: first onboard (forced failure at the policies step) - // ────────────────────────────────────────────────────────────────── - const firstRun = await host.command("node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { - artifactName: "phase-2-onboard-interrupted", - env: { - ...buildAvailabilityProbeEnv(), - NVIDIA_API_KEY: apiKey, - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_RECREATE_SANDBOX: "1", - NEMOCLAW_POLICY_MODE: "suggested", - NEMOCLAW_E2E_FAILURE_INJECTION: "1", - NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", - }, - redactionValues: [apiKey], - timeoutMs: ONBOARD_TIMEOUT_MS, - }); - const firstText = `${firstRun.stdout}\n${firstRun.stderr}`; + // Register cleanup for the sandbox we are about to create. The cleanup + // fixture runs these in LIFO at end-of-test regardless of pass/fail. + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + const cleanupEnv = buildAvailabilityProbeEnv(); + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy", + env: cleanupEnv, + timeoutMs: 120_000, + }); + await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete", + env: cleanupEnv, + timeoutMs: 60_000, + }); + await sandbox.openshell(["forward", "stop", "18789"], { + artifactName: "cleanup-openshell-forward-stop", + env: cleanupEnv, + timeoutMs: 30_000, + }); + await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy", + env: cleanupEnv, + timeoutMs: 60_000, + }); + fs.rmSync(SESSION_FILE, { force: true }); + }); - // Assertion: interrupted-exit-1. - expect(firstRun.exitCode, firstText).toBe(1); + // ────────────────────────────────────────────────────────────────── + // Phase 2: first onboard (forced failure at the policies step) + // ────────────────────────────────────────────────────────────────── + const firstRun = await host.command("node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { + artifactName: "phase-2-onboard-interrupted", + env: { + ...buildAvailabilityProbeEnv(), + NVIDIA_API_KEY: apiKey, + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_POLICY_MODE: "suggested", + NEMOCLAW_E2E_FAILURE_INJECTION: "1", + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", + }, + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + const firstText = `${firstRun.stdout}\n${firstRun.stderr}`; - // Assertion: sandbox-created-log. - expect(firstText).toContain(`Sandbox '${SANDBOX_NAME}' created`); + // Assertion: interrupted-exit-1. + expect(firstRun.exitCode, firstText).toBe(1); - // Assertion: forced-failure-log — failure injection fired at the policies step. - expect(firstText).toContain("[e2e] Forced onboarding failure at step 'policies'."); + // Assertion: sandbox-created-log. + expect(firstText).toContain(`Sandbox '${SANDBOX_NAME}' created`); - // Assertion: sandbox-exists-after-interrupt — `openshell sandbox get` exits 0. - // Pass framework env so the spawn can locate `openshell` on PATH; the - // SandboxClient threads options through to ShellProbe but does not - // auto-supply env (mirrors HostCliClient — callers stay explicit about the - // env boundary). - expect(await sandbox.exists(SANDBOX_NAME, { env: buildAvailabilityProbeEnv() })).toBe(true); + // Assertion: forced-failure-log — failure injection fired at the policies step. + expect(firstText).toContain("[e2e] Forced onboarding failure at step 'policies'."); - // Assertion: session-file-present. - expect(fs.existsSync(SESSION_FILE)).toBe(true); + // Assertion: sandbox-exists-after-interrupt — `openshell sandbox get` exits 0. + // Pass framework env so the spawn can locate `openshell` on PATH; the + // SandboxClient threads options through to ShellProbe but does not + // auto-supply env (mirrors HostCliClient — callers stay explicit about the + // env boundary). + expect(await sandbox.exists(SANDBOX_NAME, { env: buildAvailabilityProbeEnv() })).toBe(true); - // Assertion: session-file-interrupted-state. - const interrupted = readSession(SESSION_FILE); - await artifacts.writeJson("phase-2-session-state.json", interrupted); - expect(interrupted.status).toBe("failed"); - expect(interrupted.lastCompletedStep).toBe("openclaw"); - expect(interrupted.failure?.step).toBe("policies"); + // Assertion: session-file-present. + expect(fs.existsSync(SESSION_FILE)).toBe(true); - // ────────────────────────────────────────────────────────────────── - // Phase 3: resume — NVIDIA_API_KEY removed from env so the resume run - // must hydrate the credential from the session file. - // ────────────────────────────────────────────────────────────────── - const resumeRun = await host.command( - "node", - [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"], - { - artifactName: "phase-3-onboard-resume", - // buildAvailabilityProbeEnv() does NOT pass NVIDIA_API_KEY through — - // it's outside the framework allowlist. Resume must hydrate the - // credential from the session file. This is exactly the bash test's - // `env -u NVIDIA_API_KEY` invariant, expressed via the framework's - // explicit secret-passthrough rule. - env: { - ...buildAvailabilityProbeEnv(), - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_POLICY_MODE: "skip", + // Assertion: session-file-interrupted-state. + const interrupted = readSession(SESSION_FILE); + await artifacts.writeJson("phase-2-session-state.json", interrupted); + expect(interrupted.status).toBe("failed"); + expect(interrupted.lastCompletedStep).toBe("openclaw"); + expect(interrupted.failure?.step).toBe("policies"); + + // ────────────────────────────────────────────────────────────────── + // Phase 3: resume — NVIDIA_API_KEY removed from env so the resume run + // must hydrate the credential from the session file. + // ────────────────────────────────────────────────────────────────── + const resumeRun = await host.command( + "node", + [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"], + { + artifactName: "phase-3-onboard-resume", + // buildAvailabilityProbeEnv() does NOT pass NVIDIA_API_KEY through — + // it's outside the framework allowlist. Resume must hydrate the + // credential from the session file. This is exactly the bash test's + // `env -u NVIDIA_API_KEY` invariant, expressed via the framework's + // explicit secret-passthrough rule. + env: { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_POLICY_MODE: "skip", + }, + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, }, - redactionValues: [apiKey], - timeoutMs: ONBOARD_TIMEOUT_MS, - }, - ); - const resumeText = `${resumeRun.stdout}\n${resumeRun.stderr}`; + ); + const resumeText = `${resumeRun.stdout}\n${resumeRun.stderr}`; - // Assertion: resume-exit-0. - expect(resumeRun.exitCode, resumeText).toBe(0); + // Assertion: resume-exit-0. + expect(resumeRun.exitCode, resumeText).toBe(0); - // Assertion: resume-skipped-{preflight,gateway,sandbox}-log. - expect(resumeText).toContain("[resume] Skipping preflight (cached)"); - expect(resumeText).toContain("[resume] Skipping gateway (running)"); - expect(resumeText).toContain(`[resume] Skipping sandbox (${SANDBOX_NAME})`); + // Assertion: resume-skipped-{preflight,gateway,sandbox}-log. + expect(resumeText).toContain("[resume] Skipping preflight (cached)"); + expect(resumeText).toContain("[resume] Skipping gateway (running)"); + expect(resumeText).toContain(`[resume] Skipping sandbox (${SANDBOX_NAME})`); - // Assertion: resume-no-{preflight,gateway,sandbox}-rerun. - expect(resumeText).not.toContain("[1/7] Preflight checks"); - expect(resumeText).not.toContain("[2/7] Starting OpenShell gateway"); - expect(resumeText).not.toContain("[5/7] Creating sandbox"); + // Assertion: resume-no-{preflight,gateway,sandbox}-rerun. + expect(resumeText).not.toContain("[1/7] Preflight checks"); + expect(resumeText).not.toContain("[2/7] Starting OpenShell gateway"); + expect(resumeText).not.toContain("[5/7] Creating sandbox"); - // Assertion: resume-inference-handled — first onboard completed through - // openclaw (step 7) before failing at policies (step 8). Inference was - // already configured during that run, so the resume path either re-runs - // it or detects readiness and skips. Both are valid. - const ranInference = resumeText.includes("[4/7] Setting up inference provider"); - const skippedInference = - resumeText.includes("[resume] Skipping inference") || - resumeText.includes("[reuse] Skipping inference"); - expect(ranInference || skippedInference, resumeText).toBe(true); + // Assertion: resume-inference-handled — first onboard completed through + // openclaw (step 7) before failing at policies (step 8). Inference was + // already configured during that run, so the resume path either re-runs + // it or detects readiness and skips. Both are valid. + const ranInference = resumeText.includes("[4/7] Setting up inference provider"); + const skippedInference = + resumeText.includes("[resume] Skipping inference") || + resumeText.includes("[reuse] Skipping inference"); + expect(ranInference || skippedInference, resumeText).toBe(true); - // Assertion: sandbox-manageable-after-resume. - const sandboxStatus = await host.command( - "node", - [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], - { + // Assertion: sandbox-manageable-after-resume. + const sandboxStatus = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { artifactName: "phase-3-nemoclaw-status", env: buildAvailabilityProbeEnv(), timeoutMs: 60_000, - }, - ); - expect(sandboxStatus.exitCode, sandboxStatus.stderr).toBe(0); + }); + expect(sandboxStatus.exitCode, sandboxStatus.stderr).toBe(0); - // Assertion: session-file-complete-state. - const complete = readSession(SESSION_FILE); - await artifacts.writeJson("phase-3-session-state.json", complete); - expect(complete.status).toBe("complete"); - expect(complete.provider).toBe("nvidia-prod"); - for (const step of [ - "preflight", - "gateway", - "sandbox", - "provider_selection", - "inference", - "openclaw", - "policies", - ] as const) { - expect(complete.steps[step]?.status, `step ${step}`).toBe("complete"); - } + // Assertion: session-file-complete-state. + const complete = readSession(SESSION_FILE); + await artifacts.writeJson("phase-3-session-state.json", complete); + expect(complete.status).toBe("complete"); + expect(complete.provider).toBe("nvidia-prod"); + for (const step of [ + "preflight", + "gateway", + "sandbox", + "provider_selection", + "inference", + "openclaw", + "policies", + ] as const) { + expect(complete.steps[step]?.status, `step ${step}`).toBe("complete"); + } - // Assertion: registry-has-sandbox. - expect(fs.existsSync(REGISTRY_FILE)).toBe(true); - expect(fs.readFileSync(REGISTRY_FILE, "utf8")).toContain(SANDBOX_NAME); -}); + // Assertion: registry-has-sandbox. + expect(fs.existsSync(REGISTRY_FILE)).toBe(true); + expect(fs.readFileSync(REGISTRY_FILE, "utf8")).toContain(SANDBOX_NAME); + }, +); From 2b4c6aed96a35ad418c8d61afbb98f94390da562 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 16:46:38 -0400 Subject: [PATCH 09/13] test(e2e): simplify onboard resume coverage Signed-off-by: Julie Yaunches --- .github/workflows/e2e-vitest-scenarios.yaml | 55 ---------- .../framework-tests/e2e-clients.test.ts | 41 ------- .../e2e-scenario/framework/clients/sandbox.ts | 13 --- test/e2e-scenario/live/onboard-resume.test.ts | 101 +++++++++++------- .../migration/legacy-inventory.json | 11 +- 5 files changed, 68 insertions(+), 153 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 8121fac939..26b60608ab 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -206,58 +206,3 @@ jobs: include-hidden-files: false if-no-files-found: ignore retention-days: 14 - - # Migrated from test/e2e/test-onboard-resume.sh — regression for #446. - # Disruption-recovery shape (interrupt onboard via NEMOCLAW_E2E_FAILURE_INJECTION, - # then verify --resume completes without rerunning preflight/gateway/sandbox). - # Free-standing per #5049/#5107 precedent: doesn't fit the steady-state - # registry-driven expected-state probe model. Bash script kept in nightly-e2e - # under onboard-resume-e2e until typed coverage soaks (epic #5098 policy). - onboard-resume-vitest: - runs-on: ubuntu-latest - timeout-minutes: 60 - env: - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/onboard-resume - NEMOCLAW_RUN_E2E_SCENARIOS: "1" - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - - name: Install NemoClaw (provides openshell CLI + node bootstrap) - env: - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - run: bash install.sh --non-interactive --yes-i-accept-third-party-software - - - name: Install root dev dependencies - run: npm ci --ignore-scripts - - - name: Build CLI - run: npm run build:cli - - - name: Run onboard-resume live test - env: - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - run: | - set -euo pipefail - [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true - export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" - npx vitest run --project e2e-scenarios-live \ - test/e2e-scenario/live/onboard-resume.test.ts \ - --silent=false --reporter=default - - - name: Upload onboard-resume artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: e2e-vitest-scenarios-onboard-resume - path: e2e-artifacts/vitest/onboard-resume/ - include-hidden-files: false - if-no-files-found: ignore - retention-days: 14 diff --git a/test/e2e-scenario/framework-tests/e2e-clients.test.ts b/test/e2e-scenario/framework-tests/e2e-clients.test.ts index 98b320b575..77e8824fe2 100644 --- a/test/e2e-scenario/framework-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-clients.test.ts @@ -175,47 +175,6 @@ describe("E2E fixture clients", () => { }); }); - it("sandbox client builds OpenShell sandbox get commands", async () => { - const runner = new FakeRunner(); - const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); - - await sandbox.get("e2e-resume"); - - expect(runner.calls[0]).toEqual({ - command: "openshell", - args: ["sandbox", "get", "e2e-resume"], - options: { - artifactName: "sandbox-get-e2e-resume", - }, - }); - }); - - it("sandbox client exists() returns true on exit 0", async () => { - const runner = new FakeRunner(); - runner.exitCode = 0; - const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); - - expect(await sandbox.exists("e2e-resume")).toBe(true); - expect(runner.calls[0].args).toEqual(["sandbox", "get", "e2e-resume"]); - }); - - it("sandbox client exists() returns false on non-zero exit", async () => { - const runner = new FakeRunner(); - runner.exitCode = 1; - runner.stderr = "sandbox not found"; - const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); - - expect(await sandbox.exists("missing")).toBe(false); - }); - - it("sandbox client get() rejects flag-shaped sandbox names before command construction", async () => { - const runner = new FakeRunner(); - const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); - - await expect(() => sandbox.get("--bad")).toThrow(/sandbox name is invalid/); - expect(runner.calls).toEqual([]); - }); - it("sandbox client rejects flag-shaped sandbox names before command construction", async () => { const runner = new FakeRunner(); const sandbox = new SandboxClient(runner, { openshellPath: "openshell" }); diff --git a/test/e2e-scenario/framework/clients/sandbox.ts b/test/e2e-scenario/framework/clients/sandbox.ts index 45f4eee7ec..795060dad3 100644 --- a/test/e2e-scenario/framework/clients/sandbox.ts +++ b/test/e2e-scenario/framework/clients/sandbox.ts @@ -44,19 +44,6 @@ export class SandboxClient { }); } - get(name: string, options: ShellProbeRunOptions = {}): Promise { - validateSandboxName(name); - return this.openshell(["sandbox", "get", name], { - artifactName: `sandbox-get-${name}`, - ...options, - }); - } - - async exists(name: string, options: ShellProbeRunOptions = {}): Promise { - const result = await this.get(name, options); - return result.exitCode === 0; - } - exec( name: string, command: string[], diff --git a/test/e2e-scenario/live/onboard-resume.test.ts b/test/e2e-scenario/live/onboard-resume.test.ts index 666e9926f8..6a827e2d9d 100644 --- a/test/e2e-scenario/live/onboard-resume.test.ts +++ b/test/e2e-scenario/live/onboard-resume.test.ts @@ -9,22 +9,20 @@ import { buildAvailabilityProbeEnv } from "../framework/availability-env.ts"; import { expect, test } from "../framework/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../framework/live-project-gate.ts"; -// Migrated from test/e2e/test-onboard-resume.sh — regression for #446. +// Adds focused Vitest live coverage for test/e2e/test-onboard-resume.sh's +// disruption-recovery contract — regression for #446. // -// Disruption-recovery shape: drives the real `nemoclaw onboard` CLI through -// the deterministic E2E failure-injection hook -// (NEMOCLAW_E2E_FAILURE_INJECTION + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP), then -// invokes `nemoclaw onboard --resume --non-interactive` with NVIDIA_API_KEY -// stripped from the environment to prove the credential is hydrated from the -// onboard session file. +// Shape: drive the real `nemoclaw onboard` CLI through the deterministic E2E +// failure-injection hook (NEMOCLAW_E2E_FAILURE_INJECTION + +// NEMOCLAW_E2E_FORCE_FAIL_AT_STEP), then invoke +// `nemoclaw onboard --resume --non-interactive` with NVIDIA_API_KEY stripped +// from the environment to prove the credential is hydrated from the onboard +// session file. // -// Free-standing per #5049/#5107 precedent: the steady-state expected-state -// probe model in expected-states.ts does not capture log-grep contracts -// ("[resume] Skipping preflight (cached)") or the JSON-shape of an interrupted -// onboard session. Asserts inline, helpers-not-bridges. -// -// The legacy bash workflow (`onboard-resume-e2e` in nightly-e2e.yaml) is kept -// untouched per epic #5098 suite-separation rule until typed coverage soaks. +// This stays as a simple live Vitest test: assertions are inline, no registry, +// no migration ledger, no new shared helper, and no workflow replacement. The +// legacy bash workflow (`onboard-resume-e2e` in nightly-e2e.yaml) remains until +// a later retirement PR deletes/replaces that lane explicitly. const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); @@ -62,14 +60,28 @@ function readSession(file: string): T { return JSON.parse(fs.readFileSync(file, "utf8")) as T; } -// Gate the test on NEMOCLAW_RUN_E2E_SCENARIOS=1 so it only runs under the -// `e2e-scenarios-live` Vitest project (dispatched by the -// onboard-resume-vitest workflow job). The `cli` project's glob -// `test/**/*.test.{js,ts}` would otherwise pick this file up in cli-test-shards -// where there's no real `openshell` CLI, no Docker daemon, and no -// NVIDIA_API_KEY — producing a guaranteed ENOENT/skip noise. Mirrors the -// gate enforced by the `e2e-scenarios-live` project's `include:` glob in -// vitest.config.ts; live-only tests opt in to that gate explicitly. +function interruptedSessionSummary(session: SessionStateInterrupted): Record { + return { + status: session.status, + lastCompletedStep: session.lastCompletedStep, + failureStep: session.failure?.step, + }; +} + +function completeSessionSummary(session: SessionStateComplete): Record { + return { + status: session.status, + provider: session.provider, + stepStatuses: Object.fromEntries( + Object.entries(session.steps).map(([step, value]) => [step, value.status]), + ), + }; +} + +// Gate the test on NEMOCLAW_RUN_E2E_SCENARIOS=1 so accidental cli-test-shard +// discovery does not run it without real `openshell`, Docker, or NVIDIA_API_KEY. +// Live-only tests opt in to the same gate used by the `e2e-scenarios-live` +// project include glob in vitest.config.ts. test.skipIf(!shouldRunLiveE2EScenarios())( "onboard-resume: interrupted onboard then --resume completes without redoing cached steps", async ({ artifacts, cleanup, host, sandbox, secrets }) => { @@ -83,9 +95,9 @@ test.skipIf(!shouldRunLiveE2EScenarios())( `bin/nemoclaw.js missing — ensure the workflow runs npm ci + npm run build:cli before this test`, ).toBe(true); - // Assertion: docker-running — `docker info` exits 0. Pass framework - // allowlist env (includes PATH, HOME, etc.) so spawn can locate `docker`. - // The shell-probe boundary defaults to no env inheritance; framework spawns + // Assertion: docker-running — `docker info` exits 0. Pass fixture allowlist + // env (includes PATH, HOME, etc.) so spawn can locate `docker`. + // The shell-probe boundary defaults to no env inheritance; fixture spawns // must opt in via buildAvailabilityProbeEnv() to keep secret-passthrough // explicit (NVIDIA_API_KEY is NOT in the allowlist; we layer it explicitly // in Phase 2 below). @@ -97,7 +109,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( expect(dockerInfo.exitCode, dockerInfo.stderr).toBe(0); // Assertion: openshell-installed — openshell CLI is on PATH (installed by - // the workflow's `bash install.sh` step before this test runs). + // the live validation setup before this test runs). const openshellVersion = await host.command("openshell", ["--version"], { artifactName: "prereq-openshell-version", env: buildAvailabilityProbeEnv(), @@ -164,6 +176,17 @@ test.skipIf(!shouldRunLiveE2EScenarios())( timeoutMs: 60_000, }); fs.rmSync(SESSION_FILE, { force: true }); + + const sandboxAfterCleanup = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-get-after-delete", + env: cleanupEnv, + timeoutMs: 30_000, + }); + expect( + sandboxAfterCleanup.exitCode, + `sandbox ${SANDBOX_NAME} still exists after cleanup`, + ).not.toBe(0); + expect(fs.existsSync(SESSION_FILE), `${SESSION_FILE} still exists after cleanup`).toBe(false); }); // ────────────────────────────────────────────────────────────────── @@ -195,18 +218,24 @@ test.skipIf(!shouldRunLiveE2EScenarios())( expect(firstText).toContain("[e2e] Forced onboarding failure at step 'policies'."); // Assertion: sandbox-exists-after-interrupt — `openshell sandbox get` exits 0. - // Pass framework env so the spawn can locate `openshell` on PATH; the - // SandboxClient threads options through to ShellProbe but does not - // auto-supply env (mirrors HostCliClient — callers stay explicit about the - // env boundary). - expect(await sandbox.exists(SANDBOX_NAME, { env: buildAvailabilityProbeEnv() })).toBe(true); + // Keep this check local to the test instead of adding a shared helper for a + // single assertion. + const sandboxAfterInterrupt = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "phase-2-openshell-sandbox-get", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(sandboxAfterInterrupt.exitCode, sandboxAfterInterrupt.stderr).toBe(0); // Assertion: session-file-present. expect(fs.existsSync(SESSION_FILE)).toBe(true); // Assertion: session-file-interrupted-state. const interrupted = readSession(SESSION_FILE); - await artifacts.writeJson("phase-2-session-state.json", interrupted); + await artifacts.writeJson( + "phase-2-session-summary.json", + interruptedSessionSummary(interrupted), + ); expect(interrupted.status).toBe("failed"); expect(interrupted.lastCompletedStep).toBe("openclaw"); expect(interrupted.failure?.step).toBe("policies"); @@ -221,10 +250,10 @@ test.skipIf(!shouldRunLiveE2EScenarios())( { artifactName: "phase-3-onboard-resume", // buildAvailabilityProbeEnv() does NOT pass NVIDIA_API_KEY through — - // it's outside the framework allowlist. Resume must hydrate the + // it's outside the fixture env allowlist. Resume must hydrate the // credential from the session file. This is exactly the bash test's - // `env -u NVIDIA_API_KEY` invariant, expressed via the framework's - // explicit secret-passthrough rule. + // `env -u NVIDIA_API_KEY` invariant, expressed via explicit + // secret-passthrough. env: { ...buildAvailabilityProbeEnv(), NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, @@ -269,7 +298,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( // Assertion: session-file-complete-state. const complete = readSession(SESSION_FILE); - await artifacts.writeJson("phase-3-session-state.json", complete); + await artifacts.writeJson("phase-3-session-summary.json", completeSessionSummary(complete)); expect(complete.status).toBe("complete"); expect(complete.provider).toBe("nvidia-prod"); for (const step of [ diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index ad4ebd0825..e66981d726 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -523,17 +523,12 @@ "legacyScript": "test/e2e/test-onboard-resume.sh", "domain": "smoke-onboarding", "ownerIssue": "#4348", - "status": "covered", - "targetVitestScenarios": ["test/e2e-scenario/live/onboard-resume.test.ts"], + "status": "not-migrated", + "targetVitestScenarios": [], "bridgeProbes": [], "retiredReason": "", "deletionReady": false, - "notes": "Covered by free-standing live test (PR #5147). Disruption-recovery shape: drives `nemoclaw onboard` via NEMOCLAW_E2E_FAILURE_INJECTION to fail at policies, then `nemoclaw onboard --resume --non-interactive` with NVIDIA_API_KEY stripped from env to prove credential hydration from the session file. 22 assertions translated 1:1 from the bash script (regression for #446). Not registry-driven; dispatched as the discrete onboard-resume-vitest job in e2e-vitest-scenarios.yaml. Bash guard retained in nightly-e2e.yaml under onboard-resume-e2e; deletion is a follow-up PR with #4357 approval.", - "convergenceEvidence": { - "redRunUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/27283289318", - "greenRunUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/27285136703", - "verificationRunUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/27285846914" - } + "notes": "Initial completeness row; classify detailed coverage and deletion evidence in the owning migration issue before deleting." }, { "legacyScript": "test/e2e/test-openclaw-discord-pairing.sh", From 37f06680741c0876f1f4e82b642c2ae77d00083d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 16:58:08 -0400 Subject: [PATCH 10/13] test(e2e): add onboard resume live coverage Signed-off-by: Julie Yaunches --- test/e2e-scenario/live/onboard-resume.test.ts | 320 ++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 test/e2e-scenario/live/onboard-resume.test.ts diff --git a/test/e2e-scenario/live/onboard-resume.test.ts b/test/e2e-scenario/live/onboard-resume.test.ts new file mode 100644 index 0000000000..635c05c375 --- /dev/null +++ b/test/e2e-scenario/live/onboard-resume.test.ts @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; + +// Adds focused Vitest live coverage for test/e2e/test-onboard-resume.sh's +// disruption-recovery contract — regression for #446. +// +// Shape: drive the real `nemoclaw onboard` CLI through the deterministic E2E +// failure-injection hook (NEMOCLAW_E2E_FAILURE_INJECTION + +// NEMOCLAW_E2E_FORCE_FAIL_AT_STEP), then invoke +// `nemoclaw onboard --resume --non-interactive` with NVIDIA_API_KEY stripped +// from the environment to prove the credential is hydrated from the onboard +// session file. +// +// This stays as a simple live Vitest test: assertions are inline, no registry, +// no migration ledger, no new shared helper, and no workflow replacement. The +// legacy bash workflow (`onboard-resume-e2e` in nightly-e2e.yaml) remains until +// a later retirement PR deletes/replaces that lane explicitly. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json"); +const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-resume"; + +// 15 minutes per onboard run; matches NEMOCLAW_E2E_DEFAULT_TIMEOUT in the +// legacy bash test (`export NEMOCLAW_E2E_DEFAULT_TIMEOUT=600` is per-step; +// the full onboard sequence dominates). +const ONBOARD_TIMEOUT_MS = 15 * 60_000; + +interface SessionStateInterrupted { + status: "failed"; + lastCompletedStep: "openclaw"; + failure: { step: "policies" }; +} + +interface SessionStateComplete { + status: "complete"; + provider: "nvidia-prod"; + steps: Record< + | "preflight" + | "gateway" + | "sandbox" + | "provider_selection" + | "inference" + | "openclaw" + | "policies", + { status: "complete" } + >; +} + +function readSession(file: string): T { + return JSON.parse(fs.readFileSync(file, "utf8")) as T; +} + +function interruptedSessionSummary(session: SessionStateInterrupted): Record { + return { + status: session.status, + lastCompletedStep: session.lastCompletedStep, + failureStep: session.failure?.step, + }; +} + +function completeSessionSummary(session: SessionStateComplete): Record { + return { + status: session.status, + provider: session.provider, + stepStatuses: Object.fromEntries( + Object.entries(session.steps).map(([step, value]) => [step, value.status]), + ), + }; +} + +// Gate the test on NEMOCLAW_RUN_E2E_SCENARIOS=1 so accidental cli-test-shard +// discovery does not run it without real `openshell`, Docker, or NVIDIA_API_KEY. +// Live-only tests opt in to the same gate used by the `e2e-scenarios-live` +// project include glob in vitest.config.ts. +test.skipIf(!shouldRunLiveE2EScenarios())( + "onboard-resume: interrupted onboard then --resume completes without redoing cached steps", + async ({ artifacts, cleanup, host, sandbox, secrets }) => { + // ────────────────────────────────────────────────────────────────── + // Phase 1: prerequisites (host-side, all faithful on ubuntu-latest) + // ────────────────────────────────────────────────────────────────── + + // Assertion: cli-built — `bin/nemoclaw.js` exists in the repo checkout. + expect( + fs.existsSync(CLI_ENTRYPOINT), + `bin/nemoclaw.js missing — ensure the workflow runs npm ci + npm run build:cli before this test`, + ).toBe(true); + + // Assertion: docker-running — `docker info` exits 0. Pass fixture allowlist + // env (includes PATH, HOME, etc.) so spawn can locate `docker`. + // The shell-probe boundary defaults to no env inheritance; fixture spawns + // must opt in via buildAvailabilityProbeEnv() to keep secret-passthrough + // explicit (NVIDIA_API_KEY is NOT in the allowlist; we layer it explicitly + // in Phase 2 below). + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(dockerInfo.exitCode, dockerInfo.stderr).toBe(0); + + // Assertion: openshell-installed — openshell CLI is on PATH (installed by + // the live validation setup before this test runs). + const openshellVersion = await host.command("openshell", ["--version"], { + artifactName: "prereq-openshell-version", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(openshellVersion.exitCode, openshellVersion.stderr).toBe(0); + + // Assertion: nvidia-api-key-present — secrets.required(...) skips the test + // if NVIDIA_API_KEY is unset (correct behavior under workflow_dispatch + // without the secret wired in). + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey).toMatch(/^nvapi-/); + + // ────────────────────────────────────────────────────────────────── + // Phase 0 (deferred): pre-cleanup of leftover sandbox/session state. + // Done after the prereq gates pass so we don't mutate host state if + // the test would have skipped anyway. + // ────────────────────────────────────────────────────────────────── + const probeEnv = buildAvailabilityProbeEnv(); + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy", + env: probeEnv, + timeoutMs: 60_000, + }); + await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-sandbox-delete", + env: probeEnv, + timeoutMs: 60_000, + }); + await sandbox.openshell(["forward", "stop", "18789"], { + artifactName: "pre-cleanup-openshell-forward-stop", + env: probeEnv, + timeoutMs: 30_000, + }); + await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "pre-cleanup-openshell-gateway-destroy", + env: probeEnv, + timeoutMs: 60_000, + }); + fs.rmSync(SESSION_FILE, { force: true }); + + // Register cleanup for the sandbox we are about to create. The cleanup + // fixture runs these in LIFO at end-of-test regardless of pass/fail. + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + const cleanupEnv = buildAvailabilityProbeEnv(); + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy", + env: cleanupEnv, + timeoutMs: 120_000, + }); + await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete", + env: cleanupEnv, + timeoutMs: 60_000, + }); + await sandbox.openshell(["forward", "stop", "18789"], { + artifactName: "cleanup-openshell-forward-stop", + env: cleanupEnv, + timeoutMs: 30_000, + }); + await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy", + env: cleanupEnv, + timeoutMs: 60_000, + }); + fs.rmSync(SESSION_FILE, { force: true }); + + const sandboxAfterCleanup = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-get-after-delete", + env: cleanupEnv, + timeoutMs: 30_000, + }); + expect( + sandboxAfterCleanup.exitCode, + `sandbox ${SANDBOX_NAME} still exists after cleanup`, + ).not.toBe(0); + expect(fs.existsSync(SESSION_FILE), `${SESSION_FILE} still exists after cleanup`).toBe(false); + }); + + // ────────────────────────────────────────────────────────────────── + // Phase 2: first onboard (forced failure at the policies step) + // ────────────────────────────────────────────────────────────────── + const firstRun = await host.command("node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { + artifactName: "phase-2-onboard-interrupted", + env: { + ...buildAvailabilityProbeEnv(), + NVIDIA_API_KEY: apiKey, + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_POLICY_MODE: "suggested", + NEMOCLAW_E2E_FAILURE_INJECTION: "1", + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", + }, + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + const firstText = `${firstRun.stdout}\n${firstRun.stderr}`; + + // Assertion: interrupted-exit-1. + expect(firstRun.exitCode, firstText).toBe(1); + + // Assertion: sandbox-created-log. + expect(firstText).toContain(`Sandbox '${SANDBOX_NAME}' created`); + + // Assertion: forced-failure-log — failure injection fired at the policies step. + expect(firstText).toContain("[e2e] Forced onboarding failure at step 'policies'."); + + // Assertion: sandbox-exists-after-interrupt — `openshell sandbox get` exits 0. + // Keep this check local to the test instead of adding a shared helper for a + // single assertion. + const sandboxAfterInterrupt = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "phase-2-openshell-sandbox-get", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(sandboxAfterInterrupt.exitCode, sandboxAfterInterrupt.stderr).toBe(0); + + // Assertion: session-file-present. + expect(fs.existsSync(SESSION_FILE)).toBe(true); + + // Assertion: session-file-interrupted-state. + const interrupted = readSession(SESSION_FILE); + await artifacts.writeJson( + "phase-2-session-summary.json", + interruptedSessionSummary(interrupted), + ); + expect(interrupted.status).toBe("failed"); + expect(interrupted.lastCompletedStep).toBe("openclaw"); + expect(interrupted.failure?.step).toBe("policies"); + + // ────────────────────────────────────────────────────────────────── + // Phase 3: resume — NVIDIA_API_KEY removed from env so the resume run + // must hydrate the credential from the session file. + // ────────────────────────────────────────────────────────────────── + const resumeRun = await host.command( + "node", + [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"], + { + artifactName: "phase-3-onboard-resume", + // buildAvailabilityProbeEnv() does NOT pass NVIDIA_API_KEY through — + // it's outside the fixture env allowlist. Resume must hydrate the + // credential from the session file. This is exactly the bash test's + // `env -u NVIDIA_API_KEY` invariant, expressed via explicit + // secret-passthrough. + env: { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_POLICY_MODE: "skip", + }, + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + const resumeText = `${resumeRun.stdout}\n${resumeRun.stderr}`; + + // Assertion: resume-exit-0. + expect(resumeRun.exitCode, resumeText).toBe(0); + + // Assertion: resume-skipped-{preflight,gateway,sandbox}-log. + expect(resumeText).toContain("[resume] Skipping preflight (cached)"); + expect(resumeText).toContain("[resume] Skipping gateway (running)"); + expect(resumeText).toContain(`[resume] Skipping sandbox (${SANDBOX_NAME})`); + + // Assertion: resume-no-{preflight,gateway,sandbox}-rerun. + expect(resumeText).not.toContain("[1/7] Preflight checks"); + expect(resumeText).not.toContain("[2/7] Starting OpenShell gateway"); + expect(resumeText).not.toContain("[5/7] Creating sandbox"); + + // Assertion: resume-inference-handled — first onboard completed through + // openclaw (step 7) before failing at policies (step 8). Inference was + // already configured during that run, so the resume path either re-runs + // it or detects readiness and skips. Both are valid. + const ranInference = resumeText.includes("[4/7] Setting up inference provider"); + const skippedInference = + resumeText.includes("[resume] Skipping inference") || + resumeText.includes("[reuse] Skipping inference"); + expect(ranInference || skippedInference, resumeText).toBe(true); + + // Assertion: sandbox-manageable-after-resume. + const sandboxStatus = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { + artifactName: "phase-3-nemoclaw-status", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expect(sandboxStatus.exitCode, sandboxStatus.stderr).toBe(0); + + // Assertion: session-file-complete-state. + const complete = readSession(SESSION_FILE); + await artifacts.writeJson("phase-3-session-summary.json", completeSessionSummary(complete)); + expect(complete.status).toBe("complete"); + expect(complete.provider).toBe("nvidia-prod"); + for (const step of [ + "preflight", + "gateway", + "sandbox", + "provider_selection", + "inference", + "openclaw", + "policies", + ] as const) { + expect(complete.steps[step]?.status, `step ${step}`).toBe("complete"); + } + + // Assertion: registry-has-sandbox. + expect(fs.existsSync(REGISTRY_FILE)).toBe(true); + expect(fs.readFileSync(REGISTRY_FILE, "utf8")).toContain(SANDBOX_NAME); + }, +); From 67808f33459ffd31673e9739913d3f132feeabf1 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 17:18:41 -0400 Subject: [PATCH 11/13] test(e2e): harden onboard resume live test Signed-off-by: Julie Yaunches --- test/e2e-scenario/live/onboard-resume.test.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario/live/onboard-resume.test.ts b/test/e2e-scenario/live/onboard-resume.test.ts index 635c05c375..105fbfe28a 100644 --- a/test/e2e-scenario/live/onboard-resume.test.ts +++ b/test/e2e-scenario/live/onboard-resume.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; @@ -29,6 +30,7 @@ const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json"); const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-resume"; +validateSandboxName(SANDBOX_NAME); // 15 minutes per onboard run; matches NEMOCLAW_E2E_DEFAULT_TIMEOUT in the // legacy bash test (`export NEMOCLAW_E2E_DEFAULT_TIMEOUT=600` is per-step; @@ -78,6 +80,17 @@ function completeSessionSummary(session: SessionStateComplete): Record containsExactJsonToken(item, token)); + if (value && typeof value === "object") { + return Object.entries(value).some( + ([key, item]) => key === token || containsExactJsonToken(item, token), + ); + } + return false; +} + // Gate the test on NEMOCLAW_RUN_E2E_SCENARIOS=1 so accidental cli-test-shard // discovery does not run it without real `openshell`, Docker, or NVIDIA_API_KEY. // Live-only tests opt in to the same gate used by the `e2e-scenarios-live` @@ -200,6 +213,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, NEMOCLAW_RECREATE_SANDBOX: "1", NEMOCLAW_POLICY_MODE: "suggested", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_E2E_FAILURE_INJECTION: "1", NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", }, @@ -258,6 +272,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( ...buildAvailabilityProbeEnv(), NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", }, redactionValues: [apiKey], timeoutMs: ONBOARD_TIMEOUT_MS, @@ -315,6 +330,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( // Assertion: registry-has-sandbox. expect(fs.existsSync(REGISTRY_FILE)).toBe(true); - expect(fs.readFileSync(REGISTRY_FILE, "utf8")).toContain(SANDBOX_NAME); + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as unknown; + expect(containsExactJsonToken(registry, SANDBOX_NAME)).toBe(true); }, ); From 7a39cdcd5c5801bd1cc06a6cb9c7cdbadb019bea Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 17:34:05 -0400 Subject: [PATCH 12/13] test(e2e): avoid printing malformed api key Signed-off-by: Julie Yaunches --- test/e2e-scenario/live/onboard-resume.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-scenario/live/onboard-resume.test.ts b/test/e2e-scenario/live/onboard-resume.test.ts index 105fbfe28a..fe5d6b382e 100644 --- a/test/e2e-scenario/live/onboard-resume.test.ts +++ b/test/e2e-scenario/live/onboard-resume.test.ts @@ -134,7 +134,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( // if NVIDIA_API_KEY is unset (correct behavior under workflow_dispatch // without the secret wired in). const apiKey = secrets.required("NVIDIA_API_KEY"); - expect(apiKey).toMatch(/^nvapi-/); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); // ────────────────────────────────────────────────────────────────── // Phase 0 (deferred): pre-cleanup of leftover sandbox/session state. From 0c5774864f51fdf983e2c6a6004f2ddb9e7eabe1 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 17:43:48 -0400 Subject: [PATCH 13/13] test(e2e): fix onboard resume step labels Signed-off-by: Julie Yaunches --- test/e2e-scenario/live/onboard-resume.test.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/e2e-scenario/live/onboard-resume.test.ts b/test/e2e-scenario/live/onboard-resume.test.ts index fe5d6b382e..e411f75962 100644 --- a/test/e2e-scenario/live/onboard-resume.test.ts +++ b/test/e2e-scenario/live/onboard-resume.test.ts @@ -53,7 +53,8 @@ interface SessionStateComplete { | "provider_selection" | "inference" | "openclaw" - | "policies", + | "policies" + | "agent_setup", { status: "complete" } >; } @@ -289,15 +290,15 @@ test.skipIf(!shouldRunLiveE2EScenarios())( expect(resumeText).toContain(`[resume] Skipping sandbox (${SANDBOX_NAME})`); // Assertion: resume-no-{preflight,gateway,sandbox}-rerun. - expect(resumeText).not.toContain("[1/7] Preflight checks"); - expect(resumeText).not.toContain("[2/7] Starting OpenShell gateway"); - expect(resumeText).not.toContain("[5/7] Creating sandbox"); + expect(resumeText).not.toContain("[1/8] Preflight checks"); + expect(resumeText).not.toContain("[2/8] Starting OpenShell gateway"); + expect(resumeText).not.toContain("[6/8] Creating sandbox"); // Assertion: resume-inference-handled — first onboard completed through - // openclaw (step 7) before failing at policies (step 8). Inference was - // already configured during that run, so the resume path either re-runs - // it or detects readiness and skips. Both are valid. - const ranInference = resumeText.includes("[4/7] Setting up inference provider"); + // openclaw before failing at policies. Inference was already configured + // during that run, so the resume path either re-runs it or detects + // readiness and skips. Both are valid. + const ranInference = resumeText.includes("[4/8] Setting up inference provider"); const skippedInference = resumeText.includes("[resume] Skipping inference") || resumeText.includes("[reuse] Skipping inference"); @@ -324,6 +325,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( "inference", "openclaw", "policies", + "agent_setup", ] as const) { expect(complete.steps[step]?.status, `step ${step}`).toBe("complete"); }