diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 21f01e974e..07bb8c54f5 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -207,6 +207,49 @@ jobs: if-no-files-found: ignore retention-days: 14 + onboard-negative-paths-vitest: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/onboard-negative-paths + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run onboard negative-paths live test + # Direct Vitest coverage for test/e2e/test-onboard-negative-paths.sh's + # invalid-key contract. This intentionally bypasses typed registry and + # state-validation machinery because the behavior is CLI exit/output. + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/onboard-negative-paths.test.ts \ + --silent=false --reporter=default + + - name: Upload onboard negative-paths artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-onboard-negative-paths + path: e2e-artifacts/vitest/onboard-negative-paths/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. @@ -266,7 +309,9 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} run: | set -euo pipefail - npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/openclaw-tui-chat-correlation.test.ts --silent=false --reporter=default + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/openclaw-tui-chat-correlation.test.ts \ + --silent=false --reporter=default - name: Upload OpenClaw TUI chat correlation artifacts if: always() diff --git a/test/e2e-scenario/live/onboard-negative-paths.test.ts b/test/e2e-scenario/live/onboard-negative-paths.test.ts new file mode 100644 index 0000000000..dc957e2ff5 --- /dev/null +++ b/test/e2e-scenario/live/onboard-negative-paths.test.ts @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; + +// Focused Vitest replacement coverage for the first contract from +// test/e2e/test-onboard-negative-paths.sh. Keep this free-standing: the +// behavior under test is the real CLI/non-interactive onboard boundary, not the +// typed registry/state-validation scenario model. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); +const SESSION_FILE = path.join(process.env.HOME ?? "/tmp", ".nemoclaw", "onboard-session.json"); +const INVALID_NVIDIA_API_KEY = "not-a-nvidia-key"; +const STACK_TRACE_PATTERNS = [/(^|\s)(TypeError|ReferenceError|SyntaxError):/m, /^\s+at /m]; + +process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); + +const liveTest = process.env.NEMOCLAW_RUN_E2E_SCENARIOS === "1" ? test : test.skip; + +function resultText(result: { stdout: string; stderr: string }): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function hasStackTrace(text: string): boolean { + return STACK_TRACE_PATTERNS.some((pattern) => pattern.test(text)); +} + +function onboardEnv(extra: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + }; +} + +async function ignoreCleanupError(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup is best-effort because this negative path can fail before + // OpenShell exists on PATH or before any sandbox/gateway state is created. + } +} + +async function cleanupInvalidKeyState(host: HostCliClient, sandboxName: string): Promise { + await ignoreCleanupError(() => + host.nemoclaw([sandboxName, "destroy", "--yes"], { + artifactName: `cleanup-nemoclaw-destroy-${sandboxName}`, + env: onboardEnv({}), + timeoutMs: 60_000, + }), + ); + await ignoreCleanupError(() => + host.command("openshell", ["sandbox", "delete", sandboxName], { + artifactName: `cleanup-openshell-sandbox-delete-${sandboxName}`, + env: onboardEnv({}), + timeoutMs: 60_000, + }), + ); + await ignoreCleanupError(() => + host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy-nemoclaw", + env: onboardEnv({}), + timeoutMs: 60_000, + }), + ); + fs.rmSync(SESSION_FILE, { force: true }); +} + +liveTest( + "onboard invalid NVIDIA key exits cleanly without a stack trace", + async ({ artifacts, cleanup, host, skip }) => { + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-onboard-invalid-key", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `Docker is required to reach the live onboard invalid-key validation path: ${resultText(docker)}`, + ); + } + skip("Docker is required to reach the live onboard invalid-key validation path"); + } + + expect( + fs.existsSync(CLI_DIST_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + + const sandboxName = `e2e-invalid-key-${process.pid}`; + cleanup.add(`remove invalid-key onboard residue for ${sandboxName}`, async () => { + await cleanupInvalidKeyState(host, sandboxName); + }); + await cleanupInvalidKeyState(host, sandboxName); + + await artifacts.writeJson("scenario.json", { + id: "onboard-invalid-nvidia-key", + runner: "vitest", + boundary: "direct-cli-onboard", + legacySource: "test/e2e/test-onboard-negative-paths.sh", + contract: [ + "invalid NVIDIA key exits non-zero", + "invalid NVIDIA key message is explicit", + "invalid NVIDIA key path does not print a JavaScript stack trace", + ], + }); + + const result = await host.nemoclaw( + ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + { + artifactName: "onboard-invalid-nvidia-key", + env: onboardEnv({ + NEMOCLAW_SANDBOX_NAME: sandboxName, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_POLICY_MODE: "skip", + NVIDIA_API_KEY: INVALID_NVIDIA_API_KEY, + }), + redactionValues: [INVALID_NVIDIA_API_KEY], + timeoutMs: 5 * 60_000, + }, + ); + const text = resultText(result); + + expect(result.exitCode, text).not.toBe(0); + expect(text).toContain("Invalid NVIDIA API key"); + expect(text).toContain("Must start with nvapi-"); + expect(hasStackTrace(text), text).toBe(false); + + await artifacts.writeJson("scenario-result.json", { + id: "onboard-invalid-nvidia-key", + exitCode: result.exitCode, + assertions: { + nonZeroExit: result.exitCode !== 0, + explicitMessage: + text.includes("Invalid NVIDIA API key") && text.includes("Must start with nvapi-"), + noStackTrace: !hasStackTrace(text), + }, + }); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 57780b41e2..bacbbfd369 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -83,6 +83,35 @@ jobs: path: .e2e/openshell-version-pin/ include-hidden-files: true if-no-files-found: error + onboard-negative-paths-vitest: + runs-on: ubuntu-latest + needs: generate-matrix + if: \${{ inputs.scenarios != '' }} + env: + E2E_ARTIFACT_DIR: \${{ github.workspace }}/.e2e/onboard-negative-paths + NEMOCLAW_RUN_E2E_SCENARIOS: "0" + NVIDIA_API_KEY: \${{ secrets.NVIDIA_API_KEY }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: true + - name: Set up Node + uses: actions/setup-node@v4 + env: + NVIDIA_API_KEY: \${{ secrets.NVIDIA_API_KEY }} + - name: Install root dependencies + run: npm install + - name: Run onboard negative-paths live test + env: + NVIDIA_API_KEY: \${{ secrets.NVIDIA_API_KEY }} + run: npx vitest run --project e2e-scenarios-live "\${{ inputs.test_filter }}" + - name: Upload onboard negative-paths artifacts + uses: actions/upload-artifact@v4 + with: + name: onboard-negative-paths + path: .e2e/onboard-negative-paths/ + include-hidden-files: true + if-no-files-found: error `, ); @@ -143,6 +172,25 @@ jobs: "openshell-version-pin-vitest artifact upload must set include-hidden-files: false", "openshell-version-pin-vitest artifact upload must ignore missing fixture artifacts", "openshell-version-pin-vitest artifact upload retention-days must be 14", + "onboard-negative-paths-vitest job must run independently of generate-matrix", + "onboard-negative-paths-vitest job must run independently of workflow dispatch scenario filters", + "onboard-negative-paths-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1", + "onboard-negative-paths-vitest job must write artifacts under e2e-artifacts/vitest/onboard-negative-paths", + "onboard-negative-paths-vitest job env must not include NVIDIA_API_KEY", + "onboard-negative-paths-vitest checkout action must be pinned to a full commit SHA", + "onboard-negative-paths-vitest checkout step must set persist-credentials=false", + "onboard-negative-paths-vitest step 'Set up Node' env must not include NVIDIA_API_KEY", + "onboard-negative-paths-vitest setup-node action must be pinned to a full commit SHA", + "onboard-negative-paths-vitest job missing step: Build CLI", + "onboard-negative-paths-vitest step 'Run onboard negative-paths live test' env must not include NVIDIA_API_KEY", + "step 'Run onboard negative-paths live test' run script must not interpolate dispatch inputs directly", + "step 'Run onboard negative-paths live test' run script must include test/e2e-scenario/live/onboard-negative-paths.test.ts", + "onboard-negative-paths-vitest upload-artifact action must be pinned to a full commit SHA", + "onboard-negative-paths-vitest artifact upload name must be stable", + "artifact upload path must include e2e-artifacts/vitest/onboard-negative-paths/", + "onboard-negative-paths-vitest artifact upload must set include-hidden-files: false", + "onboard-negative-paths-vitest artifact upload must ignore missing fixture artifacts", + "onboard-negative-paths-vitest artifact upload retention-days must be 14", ]), ); } finally { diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index b8c8c85320..9b06e6d219 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -209,6 +209,97 @@ function validateOpenShellVersionPinVitestJob(errors: string[], jobs: WorkflowRe } } + +function validateOnboardNegativePathsVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "onboard-negative-paths-vitest"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing onboard-negative-paths-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("onboard-negative-paths-vitest job must run on ubuntu-latest"); + } + if (Object.hasOwn(job, "needs")) { + errors.push("onboard-negative-paths-vitest job must run independently of generate-matrix"); + } + if (Object.hasOwn(job, "if")) { + errors.push( + "onboard-negative-paths-vitest job must run independently of workflow dispatch scenario filters", + ); + } + + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("onboard-negative-paths-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/vitest/onboard-negative-paths" + ) { + errors.push( + "onboard-negative-paths-vitest job must write artifacts under e2e-artifacts/vitest/onboard-negative-paths", + ); + } + requireEnvDoesNotExposeSecret(errors, "onboard-negative-paths-vitest job", jobEnv, "NVIDIA_API_KEY"); + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + requireEnvDoesNotExposeSecret( + errors, + `onboard-negative-paths-vitest step '${step.name ?? step.uses ?? ""}'`, + asRecord(step.env), + "NVIDIA_API_KEY", + ); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("onboard-negative-paths-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "onboard-negative-paths-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("onboard-negative-paths-vitest checkout step must set persist-credentials=false"); + } + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("onboard-negative-paths-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "onboard-negative-paths-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run onboard negative-paths live test"); + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/onboard-negative-paths.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload onboard negative-paths artifacts"); + requireFullShaAction(errors, upload, "onboard-negative-paths-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-onboard-negative-paths") { + errors.push("onboard-negative-paths-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/onboard-negative-paths/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("onboard-negative-paths-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("onboard-negative-paths-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("onboard-negative-paths-vitest artifact upload retention-days must be 14"); + } +} + export function validateE2eVitestScenariosWorkflowBoundary( workflowPath = DEFAULT_VITEST_WORKFLOW_PATH, ): string[] { @@ -390,6 +481,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( } validateOpenShellVersionPinVitestJob(errors, jobs); + validateOnboardNegativePathsVitestJob(errors, jobs); return errors; }