From 224b169e6d223bc09264d56322c1d58cd2a82bd3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 10:49:19 -0400 Subject: [PATCH 1/7] test(e2e): bridge onboard negative paths --- .../e2e-expected-state.test.ts | 16 ++ .../e2e-phase-onboarding.test.ts | 141 ++++++++++- .../e2e-phase-state-validation.test.ts | 51 +++- .../framework/phases/onboarding.ts | 222 +++++++++++++++++- .../framework/phases/state-validation.ts | 55 ++++- .../migration/legacy-inventory.json | 6 +- .../e2e-scenario/scenarios/expected-states.ts | 8 + .../e2e-scenario/scenarios/runtime-support.ts | 7 +- test/e2e-scenario/scenarios/types.ts | 10 + 9 files changed, 493 insertions(+), 23 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts index 8d255b484c..3b1cc2efe2 100644 --- a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts @@ -96,6 +96,22 @@ describe("probesForState maps typed expected-state into probe ids", () => { expect(probesForState(state)).toEqual([]); }); + it("custom-policy state records requested registry invariants without extra probes", () => { + const state = requireExpectedState("cloud-openclaw-custom-policies-ready"); + + expect(state.registry).toEqual({ + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + policyPresets: ["npm", "pypi"], + }); + expect(probesForState(state)).toEqual([ + "cli-installed", + "local-registry-fields-match", + "gateway-healthy", + "sandbox-running", + ]); + }); + it("post-reboot-recovery-ready locks down host-side invariants only", () => { // The post-reboot scenario locks the user-visible regression // surface: registry preservation and Docker container diff --git a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts index 7add981d6c..90f69e97ce 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts @@ -1,16 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, expectTypeOf, it } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { describe, expect, expectTypeOf, it } from "vitest"; import { ArtifactSink } from "../framework/artifacts.ts"; -import { HostCliClient, type CommandRunner } from "../framework/clients/index.ts"; +import { type CommandRunner, HostCliClient } from "../framework/clients/index.ts"; import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; -import { OnboardingPhaseFixture, type OnboardingSecrets } from "../framework/phases/index.ts"; import type { EnvironmentReady } from "../framework/phases/index.ts"; +import { OnboardingPhaseFixture, type OnboardingSecrets } from "../framework/phases/index.ts"; import type { ShellProbeResult, ShellProbeRunOptions, @@ -366,6 +366,135 @@ describe("onboarding phase fixture", () => { }); }); + it("runs custom policy onboarding with requested model and presets", async () => { + const runner = new FakeRunner(); + runner.enqueue( + shellResult(0, "Using NVIDIA Endpoints with model: nvidia/nemotron-3-super-120b-a12b\n"), + ); + const secrets = new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets); + + const instance = await onboard.from(ready({ onboarding: "cloud-openclaw-custom-policies" }), { + sandboxName: "e2e-custom-policies", + }); + + expect(instance).toMatchObject({ + onboarding: "cloud-openclaw-custom-policies", + sandboxName: "e2e-custom-policies", + }); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw-custom-policies", + timeoutMs: 900_000, + }, + }); + expect(runner.calls[0]?.options?.env).toMatchObject({ + NVIDIA_API_KEY: "secret-token", + NEMOCLAW_MODEL: "nvidia/nemotron-3-super-120b-a12b", + NEMOCLAW_POLICY_MODE: "custom", + NEMOCLAW_POLICY_PRESETS: "npm,pypi", + }); + }); + + it("accepts the invalid NVIDIA key negative path without stack traces", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Invalid NVIDIA API key. Must start with nvapi-")); + const cleanup = new FakeCleanup(); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets(), + cleanup, + ); + + const instance = await onboard.from( + ready({ onboarding: "cloud-openclaw-invalid-nvidia-key" }), + { sandboxName: "e2e-invalid-key" }, + ); + + expect(instance.expectedFailure).toEqual({ + phase: "onboarding", + errorClass: "invalid-nvidia-api-key", + }); + expect(runner.calls[0]).toMatchObject({ + options: { + artifactName: "onboard-cloud-openclaw-invalid-nvidia-key", + redactionValues: ["not-a-nvidia-key"], + }, + }); + expect(runner.calls[0]?.options?.env).toMatchObject({ + NVIDIA_API_KEY: "not-a-nvidia-key", + NEMOCLAW_POLICY_MODE: "skip", + }); + expect(cleanup.calls[0]?.name).toBe("destroy NemoClaw sandbox e2e-invalid-key"); + }); + + it("rejects invalid NVIDIA key output that contains a stack trace", async () => { + const runner = new FakeRunner(); + runner.enqueue( + shellResult(1, "Invalid NVIDIA API key. Must start with nvapi-\n at fail (/tmp/x.js:1:1)"), + ); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets()); + + await expect( + onboard.from(ready({ onboarding: "cloud-openclaw-invalid-nvidia-key" })), + ).rejects.toThrow(/printed a stack trace/); + }); + + it("rejects invalid NVIDIA key output without the expected user-facing signature", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "provider rejected credential")); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets()); + + await expect( + onboard.from(ready({ onboarding: "cloud-openclaw-invalid-nvidia-key" })), + ).rejects.toThrow(/without invalid-nvidia-api-key signature/); + }); + + it("accepts the gateway port conflict negative path without stack traces", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "!! Port 18080 is not available.")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + const instance = await onboard.from( + ready({ onboarding: "cloud-openclaw-gateway-port-conflict" }), + { sandboxName: "e2e-port-conflict" }, + ); + + expect(instance.expectedFailure).toEqual({ + phase: "onboarding", + errorClass: "gateway-port-conflict", + }); + expect(runner.calls[0]).toMatchObject({ + options: { + artifactName: "onboard-cloud-openclaw-gateway-port-conflict", + redactionValues: ["secret-token"], + }, + }); + expect(runner.calls[0]?.options?.env).toMatchObject({ + NVIDIA_API_KEY: "secret-token", + NEMOCLAW_GATEWAY_PORT: "18080", + NEMOCLAW_POLICY_MODE: "skip", + }); + }); + + it("rejects gateway port conflict output without the expected user-facing signature", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "gateway failed")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + await expect( + onboard.from(ready({ onboarding: "cloud-openclaw-gateway-port-conflict" })), + ).rejects.toThrow(/without gateway-port-conflict signature/); + }); + it("requires the docker-missing runtime expectation for the no-Docker negative path", async () => { const runner = new FakeRunner(); const onboard = new OnboardingPhaseFixture( @@ -477,9 +606,9 @@ describe("onboarding phase fixture", () => { new FakeSecrets(), ); - await expect(onboard.from(ready({ onboarding: "cloud-hermes" }))).rejects.toThrow( - /Unsupported onboarding profile 'cloud-hermes'/, - ); + await expect( + onboard.from(ready({ onboarding: "cloud-nvidia-openclaw-brave" })), + ).rejects.toThrow(/Unsupported onboarding profile 'cloud-nvidia-openclaw-brave'/); }); it("writes an onboarding phase result artifact on success", async () => { diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index ca69fc221b..5a2573aa8f 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -9,13 +9,13 @@ import { describe, expect, expectTypeOf, it } from "vitest"; import { ArtifactSink } from "../framework/artifacts.ts"; import { + type CommandRunner, GatewayClient, HostCliClient, SandboxClient, - type CommandRunner, } from "../framework/clients/index.ts"; import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; -import { StateValidationPhaseFixture, type NemoClawInstance } from "../framework/phases/index.ts"; +import { type NemoClawInstance, StateValidationPhaseFixture } from "../framework/phases/index.ts"; import type { ShellProbeResult, ShellProbeRunOptions, @@ -462,6 +462,53 @@ describe("state-validation phase fixture", () => { expect(runner.calls.map((call) => call.args)).toEqual([["--version"]]); }); + it("validates requested custom policy registry fields", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "200")); + runner.enqueue(shellResult(0, "NAME\ne2e-ubuntu-repo-cloud-openclaw\n")); + const fx = fixture(runner, { + readRegistry: () => ({ + entries: { + "e2e-ubuntu-repo-cloud-openclaw": { + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + policies: ["npm", "pypi"], + }, + }, + }), + }); + + const result = await fx.from("cloud-openclaw-custom-policies-ready", instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual([ + "cli-installed", + "local-registry-fields-match", + "gateway-healthy", + "sandbox-running", + ]); + }); + + it("fails custom policy registry validation when a preset is missing", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + const fx = fixture(runner, { + readRegistry: () => ({ + entries: { + "e2e-ubuntu-repo-cloud-openclaw": { + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + policies: ["npm"], + }, + }, + }), + }); + + await expect(fx.from("cloud-openclaw-custom-policies-ready", instance())).rejects.toThrow( + /expected registry policy preset pypi/, + ); + }); + it("rejects unknown expected-state IDs", async () => { const runner = new FakeRunner(); diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts index 817ff311f2..b13ad39843 100644 --- a/test/e2e-scenario/framework/phases/onboarding.ts +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -2,16 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import http from "node:http"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; - -import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import type { ArtifactSink } from "../artifacts.ts"; +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import { artifactLabel, assertExitZero } from "../clients/command.ts"; import type { HostCliClient } from "../clients/host.ts"; import { validateSandboxName } from "../clients/sandbox.ts"; -import type { ShellProbeResult } from "../shell-probe.ts"; import { redactString } from "../redaction.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; import type { EnvironmentReady } from "./environment.ts"; const ONBOARD_ARGS = [ @@ -32,6 +32,9 @@ const DOCKER_MISSING_PATTERNS = [ /Docker is not reachable/i, /could not talk to the Docker daemon/i, ]; +const INVALID_NVIDIA_KEY_PATTERNS = [/Invalid NVIDIA API key/i, /Must start with nvapi-/i]; +const GATEWAY_PORT_CONFLICT_PATTERNS = [/Port \d+ is not available/i]; +const STACK_TRACE_PATTERNS = [/(^|\s)(TypeError|ReferenceError|SyntaxError):/m, /^\s+at /m]; const MISSING_SANDBOX_DELETE_PATTERNS = [ /\bNotFound\b/i, /\bNot Found\b/i, @@ -57,8 +60,8 @@ export interface OnboardingOptions { } export interface OnboardingExpectedFailure { - phase: "preflight"; - errorClass: "docker-missing"; + phase: "preflight" | "onboarding"; + errorClass: "docker-missing" | "invalid-nvidia-api-key" | "gateway-port-conflict"; } export interface NemoClawInstance { @@ -135,6 +138,58 @@ function hasMissingSandboxDeleteSignature(result: ShellProbeResult): boolean { return MISSING_SANDBOX_DELETE_PATTERNS.some((pattern) => pattern.test(text)); } +function hasInvalidNvidiaKeySignature(result: ShellProbeResult): boolean { + const text = resultText(result); + return INVALID_NVIDIA_KEY_PATTERNS.every((pattern) => pattern.test(text)); +} + +function hasGatewayPortConflictSignature(result: ShellProbeResult): boolean { + const text = resultText(result); + return GATEWAY_PORT_CONFLICT_PATTERNS.some((pattern) => pattern.test(text)); +} + +async function listenOnLoopback(port: number): Promise { + const server = http.createServer((_request, response) => { + response.writeHead(204); + response.end(); + }); + await new Promise((resolve, reject) => { + const onError = (error: Error) => { + server.off("listening", onListening); + reject(error); + }; + const onListening = () => { + server.off("error", onError); + resolve(); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen(port, "127.0.0.1"); + }); + return server; +} + +function isAddressInUse(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + "code" in error && + (error as NodeJS.ErrnoException).code === "EADDRINUSE" + ); +} + +async function closeServer(server: http.Server): Promise { + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); +} + export class OnboardingPhaseFixture { constructor( private readonly host: HostCliClient, @@ -153,6 +208,15 @@ export class OnboardingPhaseFixture { case "cloud-openclaw": result = await this.cloudOpenClaw(environment, options); break; + case "cloud-openclaw-custom-policies": + result = await this.cloudOpenClawCustomPolicies(environment, options); + break; + case "cloud-openclaw-invalid-nvidia-key": + result = await this.cloudOpenClawInvalidNvidiaKey(environment, options); + break; + case "cloud-openclaw-gateway-port-conflict": + result = await this.cloudOpenClawGatewayPortConflict(environment, options); + break; case "cloud-openclaw-no-docker": result = await this.cloudOpenClawNoDocker(environment, options); break; @@ -195,6 +259,146 @@ export class OnboardingPhaseFixture { }; } + async cloudOpenClawCustomPolicies( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + if (!environment.docker.available) { + throw new Error( + "cloud-openclaw-custom-policies onboarding requires an available Docker runtime.", + ); + } + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + this.registerSandboxCleanup(sandboxName); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-openclaw-custom-policies", + env: commandEnv(sandboxName, { + NVIDIA_API_KEY: apiKey, + NEMOCLAW_MODEL: "nvidia/nemotron-3-super-120b-a12b", + NEMOCLAW_POLICY_MODE: "custom", + NEMOCLAW_POLICY_PRESETS: "npm,pypi", + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(result, "cloud-openclaw-custom-policies onboarding"); + return { + onboarding: environment.onboarding, + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + }; + } + + async cloudOpenClawInvalidNvidiaKey( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + if (!environment.docker.available) { + throw new Error( + "cloud-openclaw-invalid-nvidia-key onboarding requires an available Docker runtime.", + ); + } + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + this.registerSandboxCleanup(sandboxName); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-openclaw-invalid-nvidia-key", + env: commandEnv(sandboxName, { + NVIDIA_API_KEY: "not-a-nvidia-key", + NEMOCLAW_POLICY_MODE: "skip", + }), + redactionValues: ["not-a-nvidia-key"], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + if (result.exitCode === 0) { + throw new Error("cloud-openclaw-invalid-nvidia-key onboarding unexpectedly succeeded."); + } + this.assertNoStackTrace(result, "cloud-openclaw-invalid-nvidia-key"); + if (!hasInvalidNvidiaKeySignature(result)) { + throw new Error( + `cloud-openclaw-invalid-nvidia-key onboarding failed without invalid-nvidia-api-key signature: ${resultText(result)}`, + ); + } + return { + onboarding: environment.onboarding, + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + expectedFailure: { + phase: "onboarding", + errorClass: "invalid-nvidia-api-key", + }, + }; + } + + async cloudOpenClawGatewayPortConflict( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + if (!environment.docker.available) { + throw new Error( + "cloud-openclaw-gateway-port-conflict onboarding requires an available Docker runtime.", + ); + } + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + this.registerSandboxCleanup(sandboxName); + const port = 18080; + let server: http.Server | undefined; + try { + server = await listenOnLoopback(port); + } catch (error) { + if (!isAddressInUse(error)) { + throw error; + } + } + try { + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-openclaw-gateway-port-conflict", + env: commandEnv(sandboxName, { + NVIDIA_API_KEY: apiKey, + NEMOCLAW_GATEWAY_PORT: String(port), + NEMOCLAW_POLICY_MODE: "skip", + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + if (result.exitCode === 0) { + throw new Error("cloud-openclaw-gateway-port-conflict onboarding unexpectedly succeeded."); + } + this.assertNoStackTrace(result, "cloud-openclaw-gateway-port-conflict"); + if (!hasGatewayPortConflictSignature(result)) { + throw new Error( + `cloud-openclaw-gateway-port-conflict onboarding failed without gateway-port-conflict signature: ${resultText(result)}`, + ); + } + return { + onboarding: environment.onboarding, + sandboxName, + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + expectedFailure: { + phase: "onboarding", + errorClass: "gateway-port-conflict", + }, + }; + } finally { + if (server) { + await closeServer(server); + } + } + } + async cloudOpenClawNoDocker( environment: EnvironmentReady, options: OnboardingOptions = {}, @@ -247,6 +451,14 @@ export class OnboardingPhaseFixture { } } + private assertNoStackTrace(result: ShellProbeResult, label: string): void { + const text = resultText(result); + const pattern = STACK_TRACE_PATTERNS.find((candidate) => candidate.test(text)); + if (pattern) { + throw new Error(`${label} onboarding printed a stack trace matching ${pattern}: ${text}`); + } + } + private registerSandboxCleanup(sandboxName: string): void { if (!this.cleanup) return; this.cleanup.add(`destroy NemoClaw sandbox ${sandboxName}`, async () => { diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index ea09977afd..848a9337ce 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -4,18 +4,17 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; - -import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { probesForState, requireExpectedState } from "../../scenarios/expected-states.ts"; +import type { ExpectedState, StateProbeId } from "../../scenarios/types.ts"; import type { ArtifactSink } from "../artifacts.ts"; +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import { - trustedProviderEndpoint, type GatewayClient, type HostCliClient, type SandboxClient, + trustedProviderEndpoint, } from "../clients/index.ts"; import type { ShellProbeResult } from "../shell-probe.ts"; -import { probesForState, requireExpectedState } from "../../scenarios/expected-states.ts"; -import type { ExpectedState, StateProbeId } from "../../scenarios/types.ts"; import type { NemoClawInstance } from "./onboarding.ts"; // Mirror of `src/lib/state/registry.ts::REGISTRY_FILE`. The fixture @@ -130,7 +129,7 @@ export class StateValidationPhaseFixture { typeof expectedState === "string" ? requireExpectedState(expectedState) : expectedState; const probes: StateValidationProbeResult[] = []; for (const probe of probesForState(state)) { - probes.push(await this.runProbe(probe, instance)); + probes.push(await this.runProbe(probe, state, instance)); } const result = { state, probes }; await this.writeResult("passed", state, result); @@ -159,6 +158,7 @@ export class StateValidationPhaseFixture { private async runProbe( probe: StateProbeId, + state: ExpectedState, instance: NemoClawInstance | undefined, ): Promise { switch (probe) { @@ -174,6 +174,8 @@ export class StateValidationPhaseFixture { return await this.expectSandboxAbsent(requireInstance(probe, instance)); case "local-registry-entry-present": return this.expectLocalRegistryEntryPresent(requireInstance(probe, instance)); + case "local-registry-fields-match": + return this.expectRegistryFieldsMatch(state, requireInstance(probe, instance)); case "docker-sandbox-container-present": return await this.expectDockerSandboxContainerPresent(requireInstance(probe, instance)); default: { @@ -322,6 +324,47 @@ export class StateValidationPhaseFixture { return { id: "local-registry-entry-present", status: "passed", results: [] }; } + private expectRegistryFieldsMatch( + state: ExpectedState, + instance: NemoClawInstance, + ): StateValidationProbeResult { + if (!state.registry) { + throw new Error( + "state-validation local-registry-fields-match probe requires registry expectations.", + ); + } + const reader = this.io.readRegistry ?? defaultReadRegistry; + const registry = reader(); + const entry = registry?.entries[instance.sandboxName]; + if (!entry || typeof entry !== "object") { + throw new Error( + `state-validation expected registry fields for '${instance.sandboxName}', but no registry entry was found.`, + ); + } + const record = entry as Record; + if (state.registry.provider !== undefined && record.provider !== state.registry.provider) { + throw new Error( + `state-validation expected registry provider ${state.registry.provider}, got ${String(record.provider)}.`, + ); + } + if (state.registry.model !== undefined && record.model !== state.registry.model) { + throw new Error( + `state-validation expected registry model ${state.registry.model}, got ${String(record.model)}.`, + ); + } + if (state.registry.policyPresets) { + const actual = Array.isArray(record.policies) ? record.policies : []; + for (const preset of state.registry.policyPresets) { + if (!actual.includes(preset)) { + throw new Error( + `state-validation expected registry policy preset ${preset}; policies=${JSON.stringify(actual)}.`, + ); + } + } + } + return { id: "local-registry-fields-match", status: "passed", results: [] }; + } + private async expectDockerSandboxContainerPresent( instance: NemoClawInstance, ): Promise { diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index e66981d726..0efdf5b3d3 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -501,12 +501,12 @@ "legacyScript": "test/e2e/test-onboard-negative-paths.sh", "domain": "smoke-onboarding", "ownerIssue": "#4348", - "status": "not-migrated", + "status": "bridge-probe", "targetVitestScenarios": [], - "bridgeProbes": [], + "bridgeProbes": ["test/e2e-scenario/live/registry-scenarios.test.ts"], "retiredReason": "", "deletionReady": false, - "notes": "Initial completeness row; classify detailed coverage and deletion evidence in the owning migration issue before deleting." + "notes": "Bridge-probe migration for issue #2573 negative/edge onboarding paths: policy-mode fallback remains covered by focused unit tests; live Vitest registry scenarios now exercise invalid NVIDIA key, gateway port conflict, and custom policy/model onboarding paths." }, { "legacyScript": "test/e2e/test-onboard-repair.sh", diff --git a/test/e2e-scenario/scenarios/expected-states.ts b/test/e2e-scenario/scenarios/expected-states.ts index 13c5399a79..cc430f35b4 100644 --- a/test/e2e-scenario/scenarios/expected-states.ts +++ b/test/e2e-scenario/scenarios/expected-states.ts @@ -20,6 +20,11 @@ const cloudOpenclawReady: ExpectedState = { const cloudOpenclawCustomPoliciesReady: ExpectedState = { ...cloudOpenclawReady, id: "cloud-openclaw-custom-policies-ready", + registry: { + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + policyPresets: ["npm", "pypi"], + }, }; const cloudHermesReady: ExpectedState = { @@ -156,6 +161,9 @@ export function probesForState(state: ExpectedState): readonly StateProbeId[] { if (state.localRegistry?.expected === "present") { probes.push("local-registry-entry-present"); } + if (state.registry) { + probes.push("local-registry-fields-match"); + } if (state.dockerSandboxContainer?.expected === "present") { probes.push("docker-sandbox-container-present"); } diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts index 6a9f874264..97bd1e31ea 100644 --- a/test/e2e-scenario/scenarios/runtime-support.ts +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -6,7 +6,12 @@ import type { ScenarioDefinition } from "./types.ts"; const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]); const SUPPORTED_INSTALLS = new Set(["repo-current"]); const SUPPORTED_RUNTIMES = new Set(["docker-running"]); -const SUPPORTED_ONBOARDING = new Set(["cloud-openclaw"]); +const SUPPORTED_ONBOARDING = new Set([ + "cloud-openclaw", + "cloud-openclaw-custom-policies", + "cloud-openclaw-invalid-nvidia-key", + "cloud-openclaw-gateway-port-conflict", +]); // Lifecycle profiles wired into the live Vitest driver. A profile is // supported only after both (a) `LifecyclePhaseFixture.simulate(profile)` // dispatches it, and (b) at least one expected-state declares the post- diff --git a/test/e2e-scenario/scenarios/types.ts b/test/e2e-scenario/scenarios/types.ts index 0310c34404..6cd2c8ac5b 100644 --- a/test/e2e-scenario/scenarios/types.ts +++ b/test/e2e-scenario/scenarios/types.ts @@ -30,6 +30,7 @@ export type StateProbeId = | "sandbox-running" | "sandbox-absent" | "local-registry-entry-present" + | "local-registry-fields-match" | "docker-sandbox-container-present"; // User-facing phase the negative-scenario contract advertises. Wider @@ -85,6 +86,15 @@ export interface ExpectedState { // Used to assert that recovery information remains available even // when the live OpenShell gateway returns NotFound. dockerSandboxContainer?: { expected: ExpectedPresence }; + // Local NemoClaw registry fields the onboard flow records for the + // scenario sandbox. These are host-side, deterministic invariants + // for non-interactive onboard configuration: provider/model plus + // any custom policy presets selected from env. + registry?: { + provider?: string; + model?: string; + policyPresets?: readonly string[]; + }; } export type TransientClassifier = From 18c80e8c40774fc81a189a55caf33e4a1cf937fe Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 11:02:38 -0400 Subject: [PATCH 2/7] test(e2e): tolerate missing sandbox cleanup --- .../e2e-phase-onboarding.test.ts | 23 +++++++++++++++++++ .../framework/phases/onboarding.ts | 1 + 2 files changed, 24 insertions(+) diff --git a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts index 90f69e97ce..1545469e95 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts @@ -235,6 +235,29 @@ describe("onboarding phase fixture", () => { expect(cleanup.calls).toEqual([]); }); + it("tolerates current missing-sandbox cleanup wording", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(42, "provider rejected credential")); + const cleanup = new FakeCleanup(); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + cleanup, + ); + + await expect(onboard.from(ready(), { sandboxName: "e2e-missing" })).rejects.toThrow( + /cloud-openclaw onboarding failed/, + ); + + runner.enqueue( + shellResult( + 1, + "Sandbox 'e2e-missing' does not exist.\nRun 'nemoclaw onboard' to create one.", + ), + ); + await expect(cleanup.calls[0]?.run()).resolves.toBeUndefined(); + }); + it("registers sandbox cleanup after successful cloud OpenClaw onboarding", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "onboarded\n")); diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts index b13ad39843..92ddd96004 100644 --- a/test/e2e-scenario/framework/phases/onboarding.ts +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -41,6 +41,7 @@ const MISSING_SANDBOX_DELETE_PATTERNS = [ /sandbox not found/i, /sandbox .* not found/i, /sandbox .* not present/i, + /sandbox .* does not exist/i, /sandbox does not exist/i, /no such sandbox/i, ]; From e30ec36b8dd8bc6d230fe2ab57a5e6f0a2ff1acc Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 11:26:00 -0400 Subject: [PATCH 3/7] test(e2e): mark onboard negatives covered --- test/e2e-scenario/migration/legacy-inventory.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index 0efdf5b3d3..721181cea3 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -501,12 +501,12 @@ "legacyScript": "test/e2e/test-onboard-negative-paths.sh", "domain": "smoke-onboarding", "ownerIssue": "#4348", - "status": "bridge-probe", - "targetVitestScenarios": [], - "bridgeProbes": ["test/e2e-scenario/live/registry-scenarios.test.ts"], + "status": "covered", + "targetVitestScenarios": ["test/e2e-scenario/live/registry-scenarios.test.ts"], + "bridgeProbes": ["test/policy-tiers-onboard.test.ts", "src/lib/validation.test.ts"], "retiredReason": "", "deletionReady": false, - "notes": "Bridge-probe migration for issue #2573 negative/edge onboarding paths: policy-mode fallback remains covered by focused unit tests; live Vitest registry scenarios now exercise invalid NVIDIA key, gateway port conflict, and custom policy/model onboarding paths." + "notes": "Covered by live registry scenarios ubuntu-invalid-nvidia-key-negative, ubuntu-gateway-port-conflict-negative, and ubuntu-repo-cloud-openclaw-custom-policies. Policy-mode fallback and provider-aware validation remain covered by focused Vitest tests; deletion remains false until #4357/#5098 retirement approval." }, { "legacyScript": "test/e2e/test-onboard-repair.sh", From c3491c4001f16b1d9136a0fa1ee7d088dc89e5af Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 11:37:18 -0400 Subject: [PATCH 4/7] test(e2e): update live matrix expectations --- .../framework-tests/e2e-scenario-matrix.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts index b46c1e88d3..e1297b71f0 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts @@ -77,10 +77,14 @@ describe("live Vitest scenario matrix", () => { it("builds the default live Vitest matrix from fixture-supported scenarios only", () => { expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([ + "ubuntu-gateway-port-conflict-negative", + "ubuntu-invalid-nvidia-key-negative", "ubuntu-repo-cloud-openclaw", + "ubuntu-repo-cloud-openclaw-custom-policies", "ubuntu-repo-docker-post-reboot-recovery", ]); - expect(buildLiveScenarioMatrix()[0]).toMatchObject({ + const matrix = buildLiveScenarioMatrix(); + expect(matrix.find((entry) => entry.id === "ubuntu-repo-cloud-openclaw")).toMatchObject({ id: "ubuntu-repo-cloud-openclaw", runner: "ubuntu-latest", platform: "ubuntu-local", @@ -97,7 +101,9 @@ describe("live Vitest scenario matrix", () => { // confirm the lifecycle whitelist + post-reboot-recovery scenario // are wired together; the actual RED/GREEN behavior is exercised // by the live runner (gates on the fix landing in src/lib/). - expect(buildLiveScenarioMatrix()[1]).toMatchObject({ + expect( + matrix.find((entry) => entry.id === "ubuntu-repo-docker-post-reboot-recovery"), + ).toMatchObject({ id: "ubuntu-repo-docker-post-reboot-recovery", runner: "ubuntu-latest", platform: "ubuntu-local", @@ -128,7 +134,10 @@ describe("live Vitest scenario matrix", () => { expect(lines.length, "live matrix output must be a single line").toBe(1); const parsed = JSON.parse(lines[0]); expect(parsed.map((entry: { id: string }) => entry.id)).toEqual([ + "ubuntu-gateway-port-conflict-negative", + "ubuntu-invalid-nvidia-key-negative", "ubuntu-repo-cloud-openclaw", + "ubuntu-repo-cloud-openclaw-custom-policies", "ubuntu-repo-docker-post-reboot-recovery", ]); }); From 9f404b0fd17eebcbe1bb7bc82461df8e1ebbf0fc Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 17:11:51 -0400 Subject: [PATCH 5/7] test(e2e): add direct onboard negative Vitest --- .github/workflows/e2e-vitest-scenarios.yaml | 43 +++++ .../live/onboard-negative-paths.test.ts | 150 ++++++++++++++++++ .../e2e-scenarios-workflow.test.ts | 47 ++++++ tools/e2e-scenarios/workflow-boundary.mts | 92 +++++++++++ 4 files changed, 332 insertions(+) create mode 100644 test/e2e-scenario/live/onboard-negative-paths.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 26b60608ab..b7be29d072 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -206,3 +206,46 @@ jobs: include-hidden-files: false if-no-files-found: ignore retention-days: 14 + + onboard-negative-paths-vitest: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/onboard-negative-paths + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run onboard negative-paths live test + # Direct Vitest coverage for test/e2e/test-onboard-negative-paths.sh's + # invalid-key contract. This intentionally bypasses typed registry and + # state-validation machinery because the behavior is CLI exit/output. + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/onboard-negative-paths.test.ts \ + --silent=false --reporter=default + + - name: Upload onboard negative-paths artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-onboard-negative-paths + path: e2e-artifacts/vitest/onboard-negative-paths/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 diff --git a/test/e2e-scenario/live/onboard-negative-paths.test.ts b/test/e2e-scenario/live/onboard-negative-paths.test.ts new file mode 100644 index 0000000000..67653d9f3c --- /dev/null +++ b/test/e2e-scenario/live/onboard-negative-paths.test.ts @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; + +// Focused Vitest replacement coverage for the first contract from +// test/e2e/test-onboard-negative-paths.sh. Keep this free-standing: the +// behavior under test is the real CLI/non-interactive onboard boundary, not the +// typed registry/state-validation scenario model. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); +const SESSION_FILE = path.join(process.env.HOME ?? "/tmp", ".nemoclaw", "onboard-session.json"); +const INVALID_NVIDIA_API_KEY = "not-a-nvidia-key"; +const STACK_TRACE_PATTERNS = [/(^|\s)(TypeError|ReferenceError|SyntaxError):/m, /^\s+at /m]; + +process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); + +function resultText(result: { stdout: string; stderr: string }): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function hasStackTrace(text: string): boolean { + return STACK_TRACE_PATTERNS.some((pattern) => pattern.test(text)); +} + +function onboardEnv(extra: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + }; +} + +async function ignoreCleanupError(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup is best-effort because this negative path can fail before + // OpenShell exists on PATH or before any sandbox/gateway state is created. + } +} + +async function cleanupInvalidKeyState(host: HostCliClient, sandboxName: string): Promise { + await ignoreCleanupError(() => + host.nemoclaw([sandboxName, "destroy", "--yes"], { + artifactName: `cleanup-nemoclaw-destroy-${sandboxName}`, + env: onboardEnv({}), + timeoutMs: 60_000, + }), + ); + await ignoreCleanupError(() => + host.command("openshell", ["sandbox", "delete", sandboxName], { + artifactName: `cleanup-openshell-sandbox-delete-${sandboxName}`, + env: onboardEnv({}), + timeoutMs: 60_000, + }), + ); + await ignoreCleanupError(() => + host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy-nemoclaw", + env: onboardEnv({}), + timeoutMs: 60_000, + }), + ); + fs.rmSync(SESSION_FILE, { force: true }); +} + +test("onboard invalid NVIDIA key exits cleanly without a stack trace", async ({ + artifacts, + cleanup, + host, + skip, +}) => { + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-onboard-invalid-key", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `Docker is required to reach the live onboard invalid-key validation path: ${resultText(docker)}`, + ); + } + skip("Docker is required to reach the live onboard invalid-key validation path"); + } + + expect( + fs.existsSync(CLI_DIST_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + + const sandboxName = `e2e-invalid-key-${process.pid}`; + cleanup.add(`remove invalid-key onboard residue for ${sandboxName}`, async () => { + await cleanupInvalidKeyState(host, sandboxName); + }); + await cleanupInvalidKeyState(host, sandboxName); + + await artifacts.writeJson("scenario.json", { + id: "onboard-invalid-nvidia-key", + runner: "vitest", + boundary: "direct-cli-onboard", + legacySource: "test/e2e/test-onboard-negative-paths.sh", + contract: [ + "invalid NVIDIA key exits non-zero", + "invalid NVIDIA key message is explicit", + "invalid NVIDIA key path does not print a JavaScript stack trace", + ], + }); + + const result = await host.nemoclaw( + ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + { + artifactName: "onboard-invalid-nvidia-key", + env: onboardEnv({ + NEMOCLAW_SANDBOX_NAME: sandboxName, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_POLICY_MODE: "skip", + NVIDIA_API_KEY: INVALID_NVIDIA_API_KEY, + }), + redactionValues: [INVALID_NVIDIA_API_KEY], + timeoutMs: 5 * 60_000, + }, + ); + const text = resultText(result); + + expect(result.exitCode, text).not.toBe(0); + expect(text).toContain("Invalid NVIDIA API key"); + expect(text).toContain("Must start with nvapi-"); + expect(hasStackTrace(text), text).toBe(false); + + await artifacts.writeJson("scenario-result.json", { + id: "onboard-invalid-nvidia-key", + exitCode: result.exitCode, + assertions: { + nonZeroExit: result.exitCode !== 0, + explicitMessage: + text.includes("Invalid NVIDIA API key") && text.includes("Must start with nvapi-"), + noStackTrace: !hasStackTrace(text), + }, + }); +}); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 57780b41e2..4f4c4844f2 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -83,6 +83,35 @@ jobs: path: .e2e/openshell-version-pin/ include-hidden-files: true if-no-files-found: error + onboard-negative-paths-vitest: + runs-on: ubuntu-latest + needs: generate-matrix + if: \${{ inputs.scenarios != '' }} + env: + E2E_ARTIFACT_DIR: \${{ github.workspace }}/.e2e/onboard-negative-paths + NEMOCLAW_RUN_E2E_SCENARIOS: "0" + NVIDIA_API_KEY: \${{ secrets.NVIDIA_API_KEY }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: true + - name: Set up Node + uses: actions/setup-node@v4 + env: + NVIDIA_API_KEY: \${{ secrets.NVIDIA_API_KEY }} + - name: Install root dependencies + run: npm install + - name: Run onboard negative-paths live test + env: + NVIDIA_API_KEY: \${{ secrets.NVIDIA_API_KEY }} + run: npx vitest run --project e2e-scenarios-live "\${{ inputs.test_filter }}" + - name: Upload onboard negative-paths artifacts + uses: actions/upload-artifact@v4 + with: + name: onboard-negative-paths + path: .e2e/onboard-negative-paths/ + include-hidden-files: true + if-no-files-found: error `, ); @@ -143,6 +172,24 @@ jobs: "openshell-version-pin-vitest artifact upload must set include-hidden-files: false", "openshell-version-pin-vitest artifact upload must ignore missing fixture artifacts", "openshell-version-pin-vitest artifact upload retention-days must be 14", + "onboard-negative-paths-vitest job must run independently of generate-matrix", + "onboard-negative-paths-vitest job must run independently of workflow dispatch scenario filters", + "onboard-negative-paths-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1", + "onboard-negative-paths-vitest job must write artifacts under e2e-artifacts/vitest/onboard-negative-paths", + "onboard-negative-paths-vitest job env must not include NVIDIA_API_KEY", + "onboard-negative-paths-vitest checkout action must be pinned to a full commit SHA", + "onboard-negative-paths-vitest checkout step must set persist-credentials=false", + "onboard-negative-paths-vitest step 'Set up Node' env must not include NVIDIA_API_KEY", + "onboard-negative-paths-vitest setup-node action must be pinned to a full commit SHA", + "onboard-negative-paths-vitest step 'Run onboard negative-paths live test' env must not include NVIDIA_API_KEY", + "step 'Run onboard negative-paths live test' run script must not interpolate dispatch inputs directly", + "step 'Run onboard negative-paths live test' run script must include test/e2e-scenario/live/onboard-negative-paths.test.ts", + "onboard-negative-paths-vitest upload-artifact action must be pinned to a full commit SHA", + "onboard-negative-paths-vitest artifact upload name must be stable", + "artifact upload path must include e2e-artifacts/vitest/onboard-negative-paths/", + "onboard-negative-paths-vitest artifact upload must set include-hidden-files: false", + "onboard-negative-paths-vitest artifact upload must ignore missing fixture artifacts", + "onboard-negative-paths-vitest artifact upload retention-days must be 14", ]), ); } finally { diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index b8c8c85320..9b06e6d219 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -209,6 +209,97 @@ function validateOpenShellVersionPinVitestJob(errors: string[], jobs: WorkflowRe } } + +function validateOnboardNegativePathsVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "onboard-negative-paths-vitest"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing onboard-negative-paths-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("onboard-negative-paths-vitest job must run on ubuntu-latest"); + } + if (Object.hasOwn(job, "needs")) { + errors.push("onboard-negative-paths-vitest job must run independently of generate-matrix"); + } + if (Object.hasOwn(job, "if")) { + errors.push( + "onboard-negative-paths-vitest job must run independently of workflow dispatch scenario filters", + ); + } + + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("onboard-negative-paths-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/vitest/onboard-negative-paths" + ) { + errors.push( + "onboard-negative-paths-vitest job must write artifacts under e2e-artifacts/vitest/onboard-negative-paths", + ); + } + requireEnvDoesNotExposeSecret(errors, "onboard-negative-paths-vitest job", jobEnv, "NVIDIA_API_KEY"); + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + requireEnvDoesNotExposeSecret( + errors, + `onboard-negative-paths-vitest step '${step.name ?? step.uses ?? ""}'`, + asRecord(step.env), + "NVIDIA_API_KEY", + ); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("onboard-negative-paths-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "onboard-negative-paths-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("onboard-negative-paths-vitest checkout step must set persist-credentials=false"); + } + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("onboard-negative-paths-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "onboard-negative-paths-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run onboard negative-paths live test"); + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/onboard-negative-paths.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload onboard negative-paths artifacts"); + requireFullShaAction(errors, upload, "onboard-negative-paths-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-onboard-negative-paths") { + errors.push("onboard-negative-paths-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/onboard-negative-paths/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("onboard-negative-paths-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("onboard-negative-paths-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("onboard-negative-paths-vitest artifact upload retention-days must be 14"); + } +} + export function validateE2eVitestScenariosWorkflowBoundary( workflowPath = DEFAULT_VITEST_WORKFLOW_PATH, ): string[] { @@ -390,6 +481,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( } validateOpenShellVersionPinVitestJob(errors, jobs); + validateOnboardNegativePathsVitestJob(errors, jobs); return errors; } From 9eae02fcf214bf1df320f5880ca7990cdea09bb3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 17:24:01 -0400 Subject: [PATCH 6/7] fix(e2e): skip live onboard test by default --- .../live/onboard-negative-paths.test.ts | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/test/e2e-scenario/live/onboard-negative-paths.test.ts b/test/e2e-scenario/live/onboard-negative-paths.test.ts index 67653d9f3c..dc957e2ff5 100644 --- a/test/e2e-scenario/live/onboard-negative-paths.test.ts +++ b/test/e2e-scenario/live/onboard-negative-paths.test.ts @@ -21,6 +21,8 @@ const STACK_TRACE_PATTERNS = [/(^|\s)(TypeError|ReferenceError|SyntaxError):/m, process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const liveTest = process.env.NEMOCLAW_RUN_E2E_SCENARIOS === "1" ? test : test.skip; + function resultText(result: { stdout: string; stderr: string }): string { return [result.stdout, result.stderr].filter(Boolean).join("\n"); } @@ -72,79 +74,77 @@ async function cleanupInvalidKeyState(host: HostCliClient, sandboxName: string): fs.rmSync(SESSION_FILE, { force: true }); } -test("onboard invalid NVIDIA key exits cleanly without a stack trace", async ({ - artifacts, - cleanup, - host, - skip, -}) => { - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-onboard-invalid-key", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required to reach the live onboard invalid-key validation path: ${resultText(docker)}`, - ); +liveTest( + "onboard invalid NVIDIA key exits cleanly without a stack trace", + async ({ artifacts, cleanup, host, skip }) => { + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-onboard-invalid-key", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `Docker is required to reach the live onboard invalid-key validation path: ${resultText(docker)}`, + ); + } + skip("Docker is required to reach the live onboard invalid-key validation path"); } - skip("Docker is required to reach the live onboard invalid-key validation path"); - } - expect( - fs.existsSync(CLI_DIST_ENTRYPOINT), - "run `npm run build:cli` before live repo CLI scenarios", - ).toBe(true); + expect( + fs.existsSync(CLI_DIST_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); - const sandboxName = `e2e-invalid-key-${process.pid}`; - cleanup.add(`remove invalid-key onboard residue for ${sandboxName}`, async () => { + const sandboxName = `e2e-invalid-key-${process.pid}`; + cleanup.add(`remove invalid-key onboard residue for ${sandboxName}`, async () => { + await cleanupInvalidKeyState(host, sandboxName); + }); await cleanupInvalidKeyState(host, sandboxName); - }); - await cleanupInvalidKeyState(host, sandboxName); - - await artifacts.writeJson("scenario.json", { - id: "onboard-invalid-nvidia-key", - runner: "vitest", - boundary: "direct-cli-onboard", - legacySource: "test/e2e/test-onboard-negative-paths.sh", - contract: [ - "invalid NVIDIA key exits non-zero", - "invalid NVIDIA key message is explicit", - "invalid NVIDIA key path does not print a JavaScript stack trace", - ], - }); - - const result = await host.nemoclaw( - ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], - { - artifactName: "onboard-invalid-nvidia-key", - env: onboardEnv({ - NEMOCLAW_SANDBOX_NAME: sandboxName, - NEMOCLAW_RECREATE_SANDBOX: "1", - NEMOCLAW_PROVIDER: "cloud", - NEMOCLAW_POLICY_MODE: "skip", - NVIDIA_API_KEY: INVALID_NVIDIA_API_KEY, - }), - redactionValues: [INVALID_NVIDIA_API_KEY], - timeoutMs: 5 * 60_000, - }, - ); - const text = resultText(result); - - expect(result.exitCode, text).not.toBe(0); - expect(text).toContain("Invalid NVIDIA API key"); - expect(text).toContain("Must start with nvapi-"); - expect(hasStackTrace(text), text).toBe(false); - - await artifacts.writeJson("scenario-result.json", { - id: "onboard-invalid-nvidia-key", - exitCode: result.exitCode, - assertions: { - nonZeroExit: result.exitCode !== 0, - explicitMessage: - text.includes("Invalid NVIDIA API key") && text.includes("Must start with nvapi-"), - noStackTrace: !hasStackTrace(text), - }, - }); -}); + + await artifacts.writeJson("scenario.json", { + id: "onboard-invalid-nvidia-key", + runner: "vitest", + boundary: "direct-cli-onboard", + legacySource: "test/e2e/test-onboard-negative-paths.sh", + contract: [ + "invalid NVIDIA key exits non-zero", + "invalid NVIDIA key message is explicit", + "invalid NVIDIA key path does not print a JavaScript stack trace", + ], + }); + + const result = await host.nemoclaw( + ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + { + artifactName: "onboard-invalid-nvidia-key", + env: onboardEnv({ + NEMOCLAW_SANDBOX_NAME: sandboxName, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_POLICY_MODE: "skip", + NVIDIA_API_KEY: INVALID_NVIDIA_API_KEY, + }), + redactionValues: [INVALID_NVIDIA_API_KEY], + timeoutMs: 5 * 60_000, + }, + ); + const text = resultText(result); + + expect(result.exitCode, text).not.toBe(0); + expect(text).toContain("Invalid NVIDIA API key"); + expect(text).toContain("Must start with nvapi-"); + expect(hasStackTrace(text), text).toBe(false); + + await artifacts.writeJson("scenario-result.json", { + id: "onboard-invalid-nvidia-key", + exitCode: result.exitCode, + assertions: { + nonZeroExit: result.exitCode !== 0, + explicitMessage: + text.includes("Invalid NVIDIA API key") && text.includes("Must start with nvapi-"), + noStackTrace: !hasStackTrace(text), + }, + }); + }, +); From e025a802abd0cefe14a9124f0c111b377b60260f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 17:38:38 -0400 Subject: [PATCH 7/7] test(e2e): lock onboard job build step --- test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 4f4c4844f2..bacbbfd369 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -181,6 +181,7 @@ jobs: "onboard-negative-paths-vitest checkout step must set persist-credentials=false", "onboard-negative-paths-vitest step 'Set up Node' env must not include NVIDIA_API_KEY", "onboard-negative-paths-vitest setup-node action must be pinned to a full commit SHA", + "onboard-negative-paths-vitest job missing step: Build CLI", "onboard-negative-paths-vitest step 'Run onboard negative-paths live test' env must not include NVIDIA_API_KEY", "step 'Run onboard negative-paths live test' run script must not interpolate dispatch inputs directly", "step 'Run onboard negative-paths live test' run script must include test/e2e-scenario/live/onboard-negative-paths.test.ts",