diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 4b5d9b2d3f..d96e68f0ef 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -2269,6 +2269,59 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + diagnostics-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',diagnostics-vitest,') || contains(format(',{0},', inputs.scenarios), ',diagnostics,') }} + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "diagnostics" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/diagnostics + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-diag" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run diagnostics live test + # Migrated from test/e2e/test-diagnostics.sh. This preserves the + # ubuntu-latest + Docker/OpenShell + NVIDIA_API_KEY lane by running + # debug archives, install.sh/onboard, sandbox exec/status, and + # gateway-backed credentials list/reset from Vitest. + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/diagnostics.test.ts --silent=false --reporter=default + + - name: Upload diagnostics artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-diagnostics + path: e2e-artifacts/vitest/diagnostics/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + gateway-drift-preflight-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight-vitest,') || contains(format(',{0},', inputs.scenarios), ',gateway-drift-preflight,') }} @@ -2305,9 +2358,7 @@ jobs: # Docker/OpenShell mutation. run: | set -euo pipefail - npx vitest run --project cli \ - test/gateway-drift-preflight.test.ts \ - --silent=false --reporter=default + npx vitest run --project cli test/gateway-drift-preflight.test.ts --silent=false --reporter=default - name: Upload gateway drift preflight artifacts if: always() @@ -2988,6 +3039,7 @@ jobs: double-onboard-vitest, model-router-provider-routed-inference-vitest, sandbox-survival-vitest, + diagnostics-vitest, gateway-drift-preflight-vitest, openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, diff --git a/test/e2e-scenario/live/diagnostics.test.ts b/test/e2e-scenario/live/diagnostics.test.ts new file mode 100644 index 0000000000..cba9762398 --- /dev/null +++ b/test/e2e-scenario/live/diagnostics.test.ts @@ -0,0 +1,437 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-diagnostics.sh. + * + * Preserves the legacy real boundaries: repo CLI/version, debug archive + * creation/extraction, credential redaction checks, install.sh/onboard, + * Docker/OpenShell sandbox registration, sandbox exec for openclaw.json, host + * status output, and gateway-backed credentials list/reset behavior. + */ + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-diag-${process.pid}`; +const DIAGNOSTICS_MODEL = process.env.NEMOCLAW_MODEL ?? "openai/gpt-oss-120b"; +const DEBUG_QUICK_TIMEOUT_MS = 30_000; +const INSTALL_TIMEOUT_MS = 35 * 60_000; +const TEST_TIMEOUT_MS = 55 * 60_000; +validateSandboxName(SANDBOX_NAME); + +type RawCommandResult = { + status: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; + error?: Error; +}; + +function resultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function rawResultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function redactForAssertion(text: string, apiKey: string): string { + return text + .split(apiKey) + .join("[REDACTED]") + .replace(/nvapi-[A-Za-z0-9_-]{10,}/g, ""); +} + +function runRawNodeCliForLeakAssertion( + args: string[], + env: NodeJS.ProcessEnv, + timeoutMs: number, +): RawCommandResult { + const result = spawnSync("node", [CLI_ENTRYPOINT, ...args], { + cwd: REPO_ROOT, + encoding: "utf8", + env, + killSignal: "SIGKILL", + timeout: timeoutMs, + }); + return { + status: result.status, + signal: result.signal, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + error: result.error, + }; +} + +function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + const base = buildAvailabilityProbeEnv(); + return { + ...base, + HOME: home, + PATH: [path.join(home, ".local", "bin"), base.PATH].filter(Boolean).join(":"), + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_MODEL: DIAGNOSTICS_MODEL, + NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT: "1", + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup probes are intentionally best-effort so they do not mask the + // primary diagnostics assertion. + } +} + +function assertNoSecretInExtractedArchive(extractDir: string, apiKey: string): void { + const files: string[] = []; + const visit = (dir: string): void => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) visit(fullPath); + else if (entry.isFile()) files.push(fullPath); + } + }; + visit(extractDir); + + const leakedFiles: string[] = []; + const patternLeaks: string[] = []; + const apiKeyBytes = Buffer.from(apiKey, "utf8"); + for (const file of files) { + const content = fs.readFileSync(file); + const text = content.toString("utf8"); + if (content.includes(apiKeyBytes)) leakedFiles.push(path.relative(extractDir, file)); + if (/nvapi-[A-Za-z0-9_-]{10,}/.test(text)) patternLeaks.push(path.relative(extractDir, file)); + } + + expect(leakedFiles, "debug archive must not contain the exact NVIDIA_API_KEY").toEqual([]); + expect(patternLeaks, "debug archive must not contain nvapi-shaped credentials").toEqual([]); +} + +const runDiagnosticsTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +runDiagnosticsTest( + "diagnostics CLI creates sanitized archives and validates sandbox/credential diagnostics", + { timeout: TEST_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + expect( + fs.existsSync(CLI_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + + await artifacts.writeJson("scenario.json", { + id: "diagnostics", + runner: "vitest", + boundary: "debug-archive-install-sh-docker-openshell-sandbox-exec-credentials", + legacySource: "test/e2e/test-diagnostics.sh", + sandboxName: SANDBOX_NAME, + contracts: [ + "nemoclaw --version exits zero and prints semver", + "nemoclaw debug --quick creates a non-empty archive within the quick timeout", + "nemoclaw debug --output creates an extractable archive without NVIDIA credential values", + "debug --sandbox accepts a registered sandbox and rejects an unknown sandbox without a partial archive", + "sandbox openclaw.json is readable through real OpenShell sandbox exec and host status includes model data", + "credentials list hides secret values and credentials reset removes the provider credential from the gateway", + ], + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-diagnostics", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for diagnostics live E2E: ${resultText(docker)}`); + } + skip("Docker is required for diagnostics live E2E"); + } + + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-diagnostics-home-")); + cleanup.add(`remove diagnostics state for ${SANDBOX_NAME}`, async () => { + const env = testEnv(home); + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-diagnostics", + env, + redactionValues: [apiKey], + timeoutMs: 120_000, + }), + ); + await bestEffort(() => + host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete-diagnostics", + env, + timeoutMs: 60_000, + }), + ); + await bestEffort(() => + host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy-diagnostics", + env, + timeoutMs: 120_000, + }), + ); + fs.rmSync(home, { recursive: true, force: true }); + }); + + const env = testEnv(home, { NVIDIA_API_KEY: apiKey }); + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy-diagnostics", + env, + redactionValues: [apiKey], + timeoutMs: 120_000, + }), + ); + await bestEffort(() => + host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-sandbox-delete-diagnostics", + env, + timeoutMs: 60_000, + }), + ); + + const version = await host.command("node", [CLI_ENTRYPOINT, "--version"], { + artifactName: "diagnostics-nemoclaw-version", + env: testEnv(home), + timeoutMs: 30_000, + }); + expect(version.exitCode, resultText(version)).toBe(0); + expect(resultText(version)).toMatch(/\d+\.\d+\.\d+/); + + const quickDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-debug-quick-")); + const quickArchive = path.join(quickDir, "quick-debug.tar.gz"); + const quickStartedAt = Date.now(); + const quick = await host.command( + "node", + [CLI_ENTRYPOINT, "debug", "--quick", "--output", quickArchive], + { + artifactName: "diagnostics-debug-quick", + env: testEnv(home, { NEMOCLAW_SANDBOX_NAME: "" }), + timeoutMs: DEBUG_QUICK_TIMEOUT_MS, + }, + ); + const quickElapsedMs = Date.now() - quickStartedAt; + expect(quick.exitCode, resultText(quick)).toBe(0); + expect(fs.existsSync(quickArchive), "debug --quick must create an archive").toBe(true); + expect( + fs.statSync(quickArchive).size, + "debug --quick archive must be non-empty", + ).toBeGreaterThan(0); + expect( + quickElapsedMs, + "debug --quick must complete within the legacy 30s process timeout plus harness scheduling grace", + ).toBeLessThanOrEqual(DEBUG_QUICK_TIMEOUT_MS + 5_000); + + const install = await host.command( + "bash", + ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], + { + artifactName: "install-and-onboard-diagnostics", + cwd: REPO_ROOT, + env, + redactionValues: [apiKey], + timeoutMs: INSTALL_TIMEOUT_MS, + }, + ); + expect(install.exitCode, resultText(install)).toBe(0); + + const fullDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-debug-full-")); + const fullArchive = path.join(fullDir, "debug-full.tar.gz"); + const extractDir = path.join(fullDir, "extracted"); + fs.mkdirSync(extractDir, { recursive: true }); + const fullDebug = await host.command( + "node", + [CLI_ENTRYPOINT, "debug", "--output", fullArchive], + { + artifactName: "diagnostics-debug-full", + env, + redactionValues: [apiKey], + timeoutMs: 180_000, + }, + ); + expect(fullDebug.exitCode, resultText(fullDebug)).toBe(0); + expect(fs.existsSync(fullArchive), "debug --output must create an archive").toBe(true); + expect( + fs.statSync(fullArchive).size, + "debug --output archive must be non-empty", + ).toBeGreaterThan(0); + const extract = await host.command("tar", ["xzf", fullArchive, "-C", extractDir], { + artifactName: "diagnostics-debug-full-extract", + env: testEnv(home), + timeoutMs: 60_000, + }); + expect(extract.exitCode, resultText(extract)).toBe(0); + assertNoSecretInExtractedArchive(extractDir, apiKey); + + const knownArchive = path.join(fullDir, "known-sandbox.tar.gz"); + const knownSandboxDebug = await host.command( + "node", + [CLI_ENTRYPOINT, "debug", "--quick", "--sandbox", SANDBOX_NAME, "--output", knownArchive], + { + artifactName: "diagnostics-debug-known-sandbox", + env, + redactionValues: [apiKey], + timeoutMs: DEBUG_QUICK_TIMEOUT_MS, + }, + ); + expect(knownSandboxDebug.exitCode, resultText(knownSandboxDebug)).toBe(0); + expect(fs.existsSync(knownArchive), "registered --sandbox must create an archive").toBe(true); + expect( + fs.statSync(knownArchive).size, + "registered --sandbox archive must be non-empty", + ).toBeGreaterThan(0); + + const missingName = `nemoclaw-e2e-missing-${process.pid}-${Date.now()}`; + const missingArchive = path.join(fullDir, "unknown-sandbox.tar.gz"); + const unknownSandboxDebug = await host.command( + "node", + [CLI_ENTRYPOINT, "debug", "--quick", "--sandbox", missingName, "--output", missingArchive], + { + artifactName: "diagnostics-debug-unknown-sandbox", + env, + redactionValues: [apiKey], + timeoutMs: DEBUG_QUICK_TIMEOUT_MS, + }, + ); + const unknownText = resultText(unknownSandboxDebug); + expect(unknownSandboxDebug.exitCode, unknownText).not.toBe(0); + expect(unknownText).toContain(missingName); + expect(unknownText).toMatch(/not registered/i); + expect( + fs.existsSync(missingArchive), + "unknown --sandbox must not leave a partial archive", + ).toBe(false); + + const config = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", "cat /sandbox/.openclaw/openclaw.json"], + { + artifactName: "diagnostics-sandbox-openclaw-config", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }, + ); + expect(config.exitCode, resultText(config)).toBe(0); + expect(config.stdout.trim(), "openclaw.json must be readable inside sandbox").not.toBe(""); + + const status = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { + artifactName: "diagnostics-nemoclaw-status", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }); + expect(status.exitCode, resultText(status)).toBe(0); + expect(resultText(status)).toMatch(/Model/i); + + const rawCredentialsList = runRawNodeCliForLeakAssertion(["credentials", "list"], env, 60_000); + const credentialsText = rawResultText(rawCredentialsList); + expect(rawCredentialsList.status, redactForAssertion(credentialsText, apiKey)).toBe(0); + expect( + credentialsText.includes(apiKey), + "credentials list must not expose the exact NVIDIA_API_KEY", + ).toBe(false); + expect( + /nvapi-[A-Za-z0-9_-]{10,}/.test(credentialsText), + "credentials list must not expose nvapi-shaped values", + ).toBe(false); + expect(/nvidia-prod|No provider credentials registered/i.test(credentialsText)).toBe(true); + + await host.command("node", [CLI_ENTRYPOINT, "credentials", "list"], { + artifactName: "diagnostics-credentials-list", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }); + + let credentialsResetExercised = false; + let postResetCredentialsListRedacted = false; + let providerCredentialAbsentBeforeReset = false; + if (credentialsText.includes("nvidia-prod")) { + credentialsResetExercised = true; + const reset = await host.command( + "node", + [CLI_ENTRYPOINT, "credentials", "reset", "nvidia-prod", "--yes"], + { + artifactName: "diagnostics-credentials-reset", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }, + ); + expect(reset.exitCode, resultText(reset)).toBe(0); + expect(resultText(reset)).toContain("Removed provider 'nvidia-prod'"); + + const rawPostResetList = runRawNodeCliForLeakAssertion(["credentials", "list"], env, 60_000); + const postResetText = rawResultText(rawPostResetList); + expect(rawPostResetList.status, redactForAssertion(postResetText, apiKey)).toBe(0); + expect(postResetText.includes("nvidia-prod")).toBe(false); + expect( + postResetText.includes(apiKey), + "post-reset credentials list must not expose the exact NVIDIA_API_KEY", + ).toBe(false); + expect( + /nvapi-[A-Za-z0-9_-]{10,}/.test(postResetText), + "post-reset credentials list must not expose nvapi-shaped values", + ).toBe(false); + postResetCredentialsListRedacted = !postResetText.includes(apiKey); + + await host.command("node", [CLI_ENTRYPOINT, "credentials", "list"], { + artifactName: "diagnostics-credentials-list-after-reset", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }); + } else { + providerCredentialAbsentBeforeReset = true; + await artifacts.writeJson("credentials-reset.skip.json", { + provider: "nvidia-prod", + reason: + "credentials list reported no nvidia-prod provider credential after install/onboard", + acceptedNoProviderStore: /No provider credentials registered/i.test(credentialsText), + }); + } + + await artifacts.writeJson("scenario-result.json", { + id: "diagnostics", + sandboxName: SANDBOX_NAME, + model: DIAGNOSTICS_MODEL, + assertions: { + versionPrintedSemver: /\d+\.\d+\.\d+/.test(resultText(version)), + quickDebugArchiveCreated: fs.existsSync(quickArchive) && fs.statSync(quickArchive).size > 0, + fullDebugArchiveCreated: fs.existsSync(fullArchive) && fs.statSync(fullArchive).size > 0, + fullDebugArchiveSanitized: true, + registeredSandboxDebugAccepted: knownSandboxDebug.exitCode === 0, + unknownSandboxDebugRejected: unknownSandboxDebug.exitCode !== 0, + sandboxConfigReadable: config.exitCode === 0 && config.stdout.trim().length > 0, + statusShowsModel: /Model/i.test(resultText(status)), + credentialsListRedacted: !credentialsText.includes(apiKey), + credentialsResetExercised, + providerCredentialAbsentBeforeReset, + postResetCredentialsListRedacted, + }, + }); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index e13343d998..0adda2f072 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -407,6 +407,20 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["model-router-provider-routed-inference-vitest"], registryScenarios: [], }); + expect(evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "diagnostics" })).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["diagnostics-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "diagnostics-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["diagnostics-vitest"], + registryScenarios: [], + }); expect( evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "gateway-drift-preflight", @@ -864,6 +878,7 @@ jobs: "step 'Run double-onboard live Vitest test' run script must not interpolate dispatch inputs directly", "workflow missing hermes-e2e-vitest job", "workflow missing skill-agent-vitest job", + "workflow missing diagnostics-vitest job", "workflow missing model-router-provider-routed-inference-vitest job", "report-to-pr job must wait for live-scenarios", "report-to-pr step must pass jobs through JOBS env", @@ -1056,6 +1071,67 @@ jobs: } }); + it("rejects diagnostics workflow-boundary drift for secret and Docker auth handling", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record< + string, + { env?: Record; steps: Array> } + >; + }; + const job = workflow.jobs["diagnostics-vitest"]; + expect(job).toBeDefined(); + expect(job.steps).toEqual(expect.any(Array)); + job.env = { + ...job.env, + DOCKER_CONFIG: "${{ github.workspace }}/.docker-config-diagnostics", + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + GITHUB_TOKEN: "${{ github.token }}", + }; + const setupNodeIndex = job.steps.findIndex((step) => step.name === "Set up Node"); + expect(setupNodeIndex).toBeGreaterThan(0); + job.steps.splice(setupNodeIndex, 0, { + name: "Authenticate to Docker Hub", + env: { + DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}", + DOCKERHUB_TOKEN: "${{ secrets.DOCKERHUB_TOKEN }}", + }, + run: 'docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin', + }); + const runStep = job.steps.find((step) => step.name === "Run diagnostics live test"); + expect(runStep).toBeDefined(); + runStep!.run = `${runStep!.run}\necho "\${{ inputs.jobs }}"`; + const uploadStep = job.steps.find((step) => step.name === "Upload diagnostics artifacts"); + expect(uploadStep).toBeDefined(); + uploadStep!.with = { + ...((uploadStep!.with as Record) ?? {}), + "include-hidden-files": true, + "retention-days": 1, + }; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); + expect(errors).toEqual( + expect.arrayContaining([ + "diagnostics-vitest job must not expose Docker auth to branch-controlled steps", + "diagnostics-vitest job env must not include NVIDIA_API_KEY", + "diagnostics-vitest job env must not include GITHUB_TOKEN", + "diagnostics-vitest job must not authenticate to Docker Hub before branch-controlled test code runs", + "diagnostics-vitest step 'Authenticate to Docker Hub' env must not include DOCKERHUB_USERNAME", + "diagnostics-vitest step 'Authenticate to Docker Hub' env must not include DOCKERHUB_TOKEN", + "diagnostics-vitest step 'Authenticate to Docker Hub' run script must not use docker login or inline secret interpolation", + "step 'Run diagnostics live test' run script must not interpolate dispatch inputs directly", + "diagnostics-vitest artifact upload must set include-hidden-files: false", + "diagnostics-vitest artifact upload retention-days must be 14", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("rejects Docker Hub auth in the Hermes root-entrypoint smoke job", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 1afd0be5d2..ad38bdb1c0 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -2457,6 +2457,117 @@ function validateHermesRootEntrypointSmokeVitestJob(errors: string[], jobs: Work } } +function validateDiagnosticsVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "diagnostics-vitest"; + const scenarioName = "diagnostics"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing diagnostics-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("diagnostics-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + if (job["timeout-minutes"] !== 60) { + errors.push("diagnostics-vitest job must keep the 60 minute timeout"); + } + + const jobEnv = asRecord(job.env); + if ("DOCKER_CONFIG" in jobEnv) { + errors.push("diagnostics-vitest job must not expose Docker auth to branch-controlled steps"); + } + if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/diagnostics") { + errors.push("diagnostics-vitest job must write artifacts under e2e-artifacts/vitest/diagnostics"); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("diagnostics-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("diagnostics-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { + errors.push("diagnostics-vitest job must set NEMOCLAW_NON_INTERACTIVE=1"); + } + if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { + errors.push("diagnostics-vitest job must set NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1"); + } + if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-diag") { + errors.push("diagnostics-vitest job must use the stable e2e-diag sandbox name"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("diagnostics-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); + } + for (const secret of ["NVIDIA_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + requireEnvDoesNotExposeSecret(errors, "diagnostics-vitest job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `diagnostics-vitest step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run diagnostics live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + if (namedStep(steps, "Authenticate to Docker Hub")) { + errors.push( + "diagnostics-vitest job must not authenticate to Docker Hub before branch-controlled test code runs", + ); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("diagnostics-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "diagnostics-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("diagnostics-vitest checkout step must set persist-credentials=false"); + } + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("diagnostics-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "diagnostics-vitest setup-node"); + + const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run diagnostics live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { + errors.push("diagnostics-vitest Vitest step must receive NVIDIA_API_KEY from secrets"); + } + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/diagnostics.test.ts"); + requireRunDoesNotContain(errors, runVitest, "${{ inputs."); + + const upload = requireJobStep(errors, jobName, steps, "Upload diagnostics artifacts"); + requireFullShaAction(errors, upload, "diagnostics-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-diagnostics") { + errors.push("diagnostics-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/diagnostics/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("diagnostics-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("diagnostics-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("diagnostics-vitest artifact upload retention-days must be 14"); + } +} + function validateModelRouterProviderRoutedInferenceVitestJob( errors: string[], jobs: WorkflowRecord, @@ -3511,6 +3622,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( "issue-4434-tui-unreachable-inference-vitest", "issue-4434-tui-unreachable-inference", ); + validateDiagnosticsVitestJob(errors, jobs); validateModelRouterProviderRoutedInferenceVitestJob(errors, jobs); validateFreeStandingJobSelector( errors,