From 5c68e7408bd58c8723c44cf354417b89e8d5401a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 14:16:30 -0400 Subject: [PATCH 1/5] test(e2e): migrate test-diagnostics.sh to vitest Reserve Phase 4 E2E migration work for test-diagnostics.sh. Refs #5098 From 67ff7b5647cb926a6306f9bca7759e8b20c5b927 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 15:22:56 -0400 Subject: [PATCH 2/5] test(e2e): migrate test-diagnostics.sh to vitest --- .github/workflows/e2e-vitest-scenarios.yaml | 90 ++++ test/e2e-scenario/live/diagnostics.test.ts | 421 ++++++++++++++++++ .../e2e-scenarios-workflow.test.ts | 14 + tools/e2e-scenarios/free-standing-jobs.env | 6 +- 4 files changed, 528 insertions(+), 3 deletions(-) create mode 100644 test/e2e-scenario/live/diagnostics.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index f199050c83..6e28e63858 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -1692,6 +1692,95 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + diagnostics-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',diagnostics-vitest,') || contains(format(',{0},', inputs.scenarios), ',diagnostics,') }} + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-diagnostics + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/diagnostics + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-diag" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run diagnostics live test + # Migrated from test/e2e/test-diagnostics.sh. This preserves the + # ubuntu-latest + Docker/OpenShell + NVIDIA_API_KEY lane by running + # debug archives, install.sh/onboard, sandbox exec/status, and + # gateway-backed credentials list/reset from Vitest. + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/diagnostics.test.ts \ + --silent=false --reporter=default + + - name: Upload diagnostics artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-diagnostics + path: e2e-artifacts/vitest/diagnostics/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. @@ -1890,6 +1979,7 @@ jobs: double-onboard-vitest, model-router-provider-routed-inference-vitest, sandbox-survival-vitest, + diagnostics-vitest, openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, issue-4434-tui-unreachable-inference-vitest, diff --git a/test/e2e-scenario/live/diagnostics.test.ts b/test/e2e-scenario/live/diagnostics.test.ts new file mode 100644 index 0000000000..191c1524eb --- /dev/null +++ b/test/e2e-scenario/live/diagnostics.test.ts @@ -0,0 +1,421 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-diagnostics.sh. + * + * Preserves the legacy real boundaries: repo CLI/version, debug archive + * creation/extraction, credential redaction checks, install.sh/onboard, + * Docker/OpenShell sandbox registration, sandbox exec for openclaw.json, host + * status output, and gateway-backed credentials list/reset behavior. + */ + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-diag-${process.pid}`; +const DIAGNOSTICS_MODEL = process.env.NEMOCLAW_MODEL ?? "openai/gpt-oss-120b"; +const DEBUG_QUICK_TIMEOUT_MS = 30_000; +const INSTALL_TIMEOUT_MS = 35 * 60_000; +const TEST_TIMEOUT_MS = 55 * 60_000; +validateSandboxName(SANDBOX_NAME); + +type RawCommandResult = { + status: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; + error?: Error; +}; + +function resultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function rawResultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function redactForAssertion(text: string, apiKey: string): string { + return text + .split(apiKey) + .join("[REDACTED]") + .replace(/nvapi-[A-Za-z0-9_-]{10,}/g, ""); +} + +function runRawNodeCliForLeakAssertion( + args: string[], + env: NodeJS.ProcessEnv, + timeoutMs: number, +): RawCommandResult { + const result = spawnSync("node", [CLI_ENTRYPOINT, ...args], { + cwd: REPO_ROOT, + encoding: "utf8", + env, + killSignal: "SIGKILL", + timeout: timeoutMs, + }); + return { + status: result.status, + signal: result.signal, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + error: result.error, + }; +} + +function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + const base = buildAvailabilityProbeEnv(); + return { + ...base, + HOME: home, + PATH: [path.join(home, ".local", "bin"), base.PATH].filter(Boolean).join(":"), + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_MODEL: DIAGNOSTICS_MODEL, + NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT: "1", + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup probes are intentionally best-effort so they do not mask the + // primary diagnostics assertion. + } +} + +function assertNoSecretInExtractedArchive(extractDir: string, apiKey: string): void { + const files: string[] = []; + const visit = (dir: string): void => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) visit(fullPath); + else if (entry.isFile()) files.push(fullPath); + } + }; + visit(extractDir); + + const leakedFiles: string[] = []; + const patternLeaks: string[] = []; + const apiKeyBytes = Buffer.from(apiKey, "utf8"); + for (const file of files) { + const content = fs.readFileSync(file); + const text = content.toString("utf8"); + if (content.includes(apiKeyBytes)) leakedFiles.push(path.relative(extractDir, file)); + if (/nvapi-[A-Za-z0-9_-]{10,}/.test(text)) patternLeaks.push(path.relative(extractDir, file)); + } + + expect(leakedFiles, "debug archive must not contain the exact NVIDIA_API_KEY").toEqual([]); + expect(patternLeaks, "debug archive must not contain nvapi-shaped credentials").toEqual([]); +} + +const runDiagnosticsTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +runDiagnosticsTest( + "diagnostics CLI creates sanitized archives and validates sandbox/credential diagnostics", + { timeout: TEST_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + expect( + fs.existsSync(CLI_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + + await artifacts.writeJson("scenario.json", { + id: "diagnostics", + runner: "vitest", + boundary: "debug-archive-install-sh-docker-openshell-sandbox-exec-credentials", + legacySource: "test/e2e/test-diagnostics.sh", + sandboxName: SANDBOX_NAME, + contracts: [ + "nemoclaw --version exits zero and prints semver", + "nemoclaw debug --quick creates a non-empty archive within the quick timeout", + "nemoclaw debug --output creates an extractable archive without NVIDIA credential values", + "debug --sandbox accepts a registered sandbox and rejects an unknown sandbox without a partial archive", + "sandbox openclaw.json is readable through real OpenShell sandbox exec and host status includes model data", + "credentials list hides secret values and credentials reset removes the provider credential from the gateway", + ], + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-diagnostics", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for diagnostics live E2E: ${resultText(docker)}`); + } + skip("Docker is required for diagnostics live E2E"); + } + + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-diagnostics-home-")); + cleanup.add(`remove diagnostics state for ${SANDBOX_NAME}`, async () => { + const env = testEnv(home); + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-diagnostics", + env, + redactionValues: [apiKey], + timeoutMs: 120_000, + }), + ); + await bestEffort(() => + host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete-diagnostics", + env, + timeoutMs: 60_000, + }), + ); + await bestEffort(() => + host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy-diagnostics", + env, + timeoutMs: 120_000, + }), + ); + fs.rmSync(home, { recursive: true, force: true }); + }); + + const env = testEnv(home, { NVIDIA_API_KEY: apiKey }); + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy-diagnostics", + env, + redactionValues: [apiKey], + timeoutMs: 120_000, + }), + ); + await bestEffort(() => + host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-sandbox-delete-diagnostics", + env, + timeoutMs: 60_000, + }), + ); + + const version = await host.command("node", [CLI_ENTRYPOINT, "--version"], { + artifactName: "diagnostics-nemoclaw-version", + env: testEnv(home), + timeoutMs: 30_000, + }); + expect(version.exitCode, resultText(version)).toBe(0); + expect(resultText(version)).toMatch(/\d+\.\d+\.\d+/); + + const quickDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-debug-quick-")); + const quickArchive = path.join(quickDir, "quick-debug.tar.gz"); + const quickStartedAt = Date.now(); + const quick = await host.command( + "node", + [CLI_ENTRYPOINT, "debug", "--quick", "--output", quickArchive], + { + artifactName: "diagnostics-debug-quick", + env: testEnv(home), + timeoutMs: DEBUG_QUICK_TIMEOUT_MS, + }, + ); + const quickElapsedMs = Date.now() - quickStartedAt; + expect(quick.exitCode, resultText(quick)).toBe(0); + expect(fs.existsSync(quickArchive), "debug --quick must create an archive").toBe(true); + expect( + fs.statSync(quickArchive).size, + "debug --quick archive must be non-empty", + ).toBeGreaterThan(0); + expect( + quickElapsedMs, + "debug --quick must complete within the legacy 30s budget", + ).toBeLessThanOrEqual(DEBUG_QUICK_TIMEOUT_MS); + + const install = await host.command( + "bash", + ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], + { + artifactName: "install-and-onboard-diagnostics", + cwd: REPO_ROOT, + env, + redactionValues: [apiKey], + timeoutMs: INSTALL_TIMEOUT_MS, + }, + ); + expect(install.exitCode, resultText(install)).toBe(0); + + const fullDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-debug-full-")); + const fullArchive = path.join(fullDir, "debug-full.tar.gz"); + const extractDir = path.join(fullDir, "extracted"); + fs.mkdirSync(extractDir, { recursive: true }); + const fullDebug = await host.command( + "node", + [CLI_ENTRYPOINT, "debug", "--output", fullArchive], + { + artifactName: "diagnostics-debug-full", + env, + redactionValues: [apiKey], + timeoutMs: 180_000, + }, + ); + expect(fullDebug.exitCode, resultText(fullDebug)).toBe(0); + expect(fs.existsSync(fullArchive), "debug --output must create an archive").toBe(true); + expect( + fs.statSync(fullArchive).size, + "debug --output archive must be non-empty", + ).toBeGreaterThan(0); + const extract = await host.command("tar", ["xzf", fullArchive, "-C", extractDir], { + artifactName: "diagnostics-debug-full-extract", + env: testEnv(home), + timeoutMs: 60_000, + }); + expect(extract.exitCode, resultText(extract)).toBe(0); + assertNoSecretInExtractedArchive(extractDir, apiKey); + + const knownArchive = path.join(fullDir, "known-sandbox.tar.gz"); + const knownSandboxDebug = await host.command( + "node", + [CLI_ENTRYPOINT, "debug", "--quick", "--sandbox", SANDBOX_NAME, "--output", knownArchive], + { + artifactName: "diagnostics-debug-known-sandbox", + env, + redactionValues: [apiKey], + timeoutMs: DEBUG_QUICK_TIMEOUT_MS, + }, + ); + expect(knownSandboxDebug.exitCode, resultText(knownSandboxDebug)).toBe(0); + expect(fs.existsSync(knownArchive), "registered --sandbox must create an archive").toBe(true); + expect( + fs.statSync(knownArchive).size, + "registered --sandbox archive must be non-empty", + ).toBeGreaterThan(0); + + const missingName = `nemoclaw-e2e-missing-${process.pid}-${Date.now()}`; + const missingArchive = path.join(fullDir, "unknown-sandbox.tar.gz"); + const unknownSandboxDebug = await host.command( + "node", + [CLI_ENTRYPOINT, "debug", "--quick", "--sandbox", missingName, "--output", missingArchive], + { + artifactName: "diagnostics-debug-unknown-sandbox", + env, + redactionValues: [apiKey], + timeoutMs: DEBUG_QUICK_TIMEOUT_MS, + }, + ); + const unknownText = resultText(unknownSandboxDebug); + expect(unknownSandboxDebug.exitCode, unknownText).not.toBe(0); + expect(unknownText).toContain(missingName); + expect(unknownText).toMatch(/not registered/i); + expect( + fs.existsSync(missingArchive), + "unknown --sandbox must not leave a partial archive", + ).toBe(false); + + const config = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", "cat /sandbox/.openclaw/openclaw.json"], + { + artifactName: "diagnostics-sandbox-openclaw-config", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }, + ); + expect(config.exitCode, resultText(config)).toBe(0); + expect(config.stdout.trim(), "openclaw.json must be readable inside sandbox").not.toBe(""); + + const status = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { + artifactName: "diagnostics-nemoclaw-status", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }); + expect(status.exitCode, resultText(status)).toBe(0); + expect(resultText(status)).toMatch(/Model/i); + + const rawCredentialsList = runRawNodeCliForLeakAssertion(["credentials", "list"], env, 60_000); + const credentialsText = rawResultText(rawCredentialsList); + expect(rawCredentialsList.status, redactForAssertion(credentialsText, apiKey)).toBe(0); + expect( + credentialsText.includes(apiKey), + "credentials list must not expose the exact NVIDIA_API_KEY", + ).toBe(false); + expect( + /nvapi-[A-Za-z0-9_-]{10,}/.test(credentialsText), + "credentials list must not expose nvapi-shaped values", + ).toBe(false); + expect(/nvidia-prod|No provider credentials registered/i.test(credentialsText)).toBe(true); + + await host.command("node", [CLI_ENTRYPOINT, "credentials", "list"], { + artifactName: "diagnostics-credentials-list", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }); + + if (credentialsText.includes("nvidia-prod")) { + const reset = await host.command( + "node", + [CLI_ENTRYPOINT, "credentials", "reset", "nvidia-prod", "--yes"], + { + artifactName: "diagnostics-credentials-reset", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }, + ); + expect(reset.exitCode, resultText(reset)).toBe(0); + expect(resultText(reset)).toContain("Removed provider 'nvidia-prod'"); + + const rawPostResetList = runRawNodeCliForLeakAssertion(["credentials", "list"], env, 60_000); + const postResetText = rawResultText(rawPostResetList); + expect(rawPostResetList.status, redactForAssertion(postResetText, apiKey)).toBe(0); + expect(postResetText.includes("nvidia-prod")).toBe(false); + expect( + postResetText.includes(apiKey), + "post-reset credentials list must not expose the exact NVIDIA_API_KEY", + ).toBe(false); + expect( + /nvapi-[A-Za-z0-9_-]{10,}/.test(postResetText), + "post-reset credentials list must not expose nvapi-shaped values", + ).toBe(false); + + await host.command("node", [CLI_ENTRYPOINT, "credentials", "list"], { + artifactName: "diagnostics-credentials-list-after-reset", + env, + redactionValues: [apiKey], + timeoutMs: 60_000, + }); + } + + await artifacts.writeJson("scenario-result.json", { + id: "diagnostics", + sandboxName: SANDBOX_NAME, + model: DIAGNOSTICS_MODEL, + assertions: { + versionPrintedSemver: /\d+\.\d+\.\d+/.test(resultText(version)), + quickDebugArchiveCreated: fs.existsSync(quickArchive) && fs.statSync(quickArchive).size > 0, + fullDebugArchiveCreated: fs.existsSync(fullArchive) && fs.statSync(fullArchive).size > 0, + fullDebugArchiveSanitized: true, + registeredSandboxDebugAccepted: knownSandboxDebug.exitCode === 0, + unknownSandboxDebugRejected: unknownSandboxDebug.exitCode !== 0, + sandboxConfigReadable: config.exitCode === 0 && config.stdout.trim().length > 0, + statusShowsModel: /Model/i.test(resultText(status)), + credentialsListRedacted: !credentialsText.includes(apiKey), + }, + }); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 354712ecde..c3dee848c9 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -255,6 +255,20 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["model-router-provider-routed-inference-vitest"], registryScenarios: [], }); + expect(evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "diagnostics" })).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["diagnostics-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "diagnostics-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["diagnostics-vitest"], + registryScenarios: [], + }); }); it("keeps the free-standing inventory internally consistent and data-only", () => { diff --git a/tools/e2e-scenarios/free-standing-jobs.env b/tools/e2e-scenarios/free-standing-jobs.env index 68ce23e36d..ad9f2e2975 100644 --- a/tools/e2e-scenarios/free-standing-jobs.env +++ b/tools/e2e-scenarios/free-standing-jobs.env @@ -1,5 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -allowed_jobs=openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,shields-config-vitest,rebuild-openclaw-vitest,sandbox-rebuild-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,credential-sanitization-vitest,sandbox-survival-vitest -free_standing_scenarios_csv=openshell-version-pin,onboard-negative-paths,skill-agent,inference-routing,runtime-overrides,hermes-e2e,hermes-root-entrypoint-smoke,network-policy,shields-config,rebuild-openclaw,sandbox-rebuild,token-rotation,openclaw-tui-chat-correlation,double-onboard,issue-4434-tui-unreachable-inference,model-router-provider-routed-inference,credential-sanitization,sandbox-survival -free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest,onboard-negative-paths:onboard-negative-paths-vitest,skill-agent:skill-agent-vitest,inference-routing:inference-routing-vitest,runtime-overrides:runtime-overrides-vitest,hermes-e2e:hermes-e2e-vitest,hermes-root-entrypoint-smoke:hermes-root-entrypoint-smoke-vitest,network-policy:network-policy-vitest,shields-config:shields-config-vitest,rebuild-openclaw:rebuild-openclaw-vitest,sandbox-rebuild:sandbox-rebuild-vitest,token-rotation:token-rotation-vitest,openclaw-tui-chat-correlation:openclaw-tui-chat-correlation-vitest,double-onboard:double-onboard-vitest,issue-4434-tui-unreachable-inference:issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference:model-router-provider-routed-inference-vitest,credential-sanitization:credential-sanitization-vitest,sandbox-survival:sandbox-survival-vitest +allowed_jobs=openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,shields-config-vitest,rebuild-openclaw-vitest,sandbox-rebuild-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,credential-sanitization-vitest,sandbox-survival-vitest,diagnostics-vitest +free_standing_scenarios_csv=openshell-version-pin,onboard-negative-paths,skill-agent,inference-routing,runtime-overrides,hermes-e2e,hermes-root-entrypoint-smoke,network-policy,shields-config,rebuild-openclaw,sandbox-rebuild,token-rotation,openclaw-tui-chat-correlation,double-onboard,issue-4434-tui-unreachable-inference,model-router-provider-routed-inference,credential-sanitization,sandbox-survival,diagnostics +free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest,onboard-negative-paths:onboard-negative-paths-vitest,skill-agent:skill-agent-vitest,inference-routing:inference-routing-vitest,runtime-overrides:runtime-overrides-vitest,hermes-e2e:hermes-e2e-vitest,hermes-root-entrypoint-smoke:hermes-root-entrypoint-smoke-vitest,network-policy:network-policy-vitest,shields-config:shields-config-vitest,rebuild-openclaw:rebuild-openclaw-vitest,sandbox-rebuild:sandbox-rebuild-vitest,token-rotation:token-rotation-vitest,openclaw-tui-chat-correlation:openclaw-tui-chat-correlation-vitest,double-onboard:double-onboard-vitest,issue-4434-tui-unreachable-inference:issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference:model-router-provider-routed-inference-vitest,credential-sanitization:credential-sanitization-vitest,sandbox-survival:sandbox-survival-vitest,diagnostics:diagnostics-vitest From 8e98857621def0dd344f71e85dd54f665d9885f2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 17:45:28 -0400 Subject: [PATCH 3/5] test(e2e): guard diagnostics workflow boundary --- test/e2e-scenario/live/diagnostics.test.ts | 16 +++ .../e2e-scenarios-workflow.test.ts | 50 +++++++ tools/e2e-scenarios/workflow-boundary.mts | 129 ++++++++++++++++++ 3 files changed, 195 insertions(+) diff --git a/test/e2e-scenario/live/diagnostics.test.ts b/test/e2e-scenario/live/diagnostics.test.ts index 191c1524eb..d2cddeb0ec 100644 --- a/test/e2e-scenario/live/diagnostics.test.ts +++ b/test/e2e-scenario/live/diagnostics.test.ts @@ -366,7 +366,11 @@ runDiagnosticsTest( timeoutMs: 60_000, }); + let credentialsResetExercised = false; + let postResetCredentialsListRedacted = false; + let providerCredentialAbsentBeforeReset = false; if (credentialsText.includes("nvidia-prod")) { + credentialsResetExercised = true; const reset = await host.command( "node", [CLI_ENTRYPOINT, "credentials", "reset", "nvidia-prod", "--yes"], @@ -392,6 +396,7 @@ runDiagnosticsTest( /nvapi-[A-Za-z0-9_-]{10,}/.test(postResetText), "post-reset credentials list must not expose nvapi-shaped values", ).toBe(false); + postResetCredentialsListRedacted = !postResetText.includes(apiKey); await host.command("node", [CLI_ENTRYPOINT, "credentials", "list"], { artifactName: "diagnostics-credentials-list-after-reset", @@ -399,6 +404,14 @@ runDiagnosticsTest( redactionValues: [apiKey], timeoutMs: 60_000, }); + } else { + providerCredentialAbsentBeforeReset = true; + await artifacts.writeJson("credentials-reset.skip.json", { + provider: "nvidia-prod", + reason: + "credentials list reported no nvidia-prod provider credential after install/onboard", + acceptedNoProviderStore: /No provider credentials registered/i.test(credentialsText), + }); } await artifacts.writeJson("scenario-result.json", { @@ -415,6 +428,9 @@ runDiagnosticsTest( sandboxConfigReadable: config.exitCode === 0 && config.stdout.trim().length > 0, statusShowsModel: /Model/i.test(resultText(status)), credentialsListRedacted: !credentialsText.includes(apiKey), + credentialsResetExercised, + providerCredentialAbsentBeforeReset, + postResetCredentialsListRedacted, }, }); }, diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index c3dee848c9..bc676c5147 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -614,6 +614,7 @@ jobs: "step 'Run double-onboard live Vitest test' run script must not interpolate dispatch inputs directly", "workflow missing hermes-e2e-vitest job", "workflow missing skill-agent-vitest job", + "workflow missing diagnostics-vitest job", "workflow missing model-router-provider-routed-inference-vitest job", "report-to-pr job must wait for live-scenarios", "report-to-pr step must pass jobs through JOBS env", @@ -699,6 +700,55 @@ jobs: } }); + it("rejects diagnostics workflow-boundary drift for secret and Docker auth handling", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record< + string, + { env?: Record; steps: Array> } + >; + }; + const job = workflow.jobs["diagnostics-vitest"]; + expect(job).toBeDefined(); + expect(job.steps).toEqual(expect.any(Array)); + job.env = { + ...job.env, + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + GITHUB_TOKEN: "${{ github.token }}", + }; + const runStep = job.steps.find((step) => step.name === "Run diagnostics live test"); + expect(runStep).toBeDefined(); + runStep!.run = `${runStep!.run}\necho "\${{ inputs.jobs }}"`; + const uploadStep = job.steps.find((step) => step.name === "Upload diagnostics artifacts"); + expect(uploadStep).toBeDefined(); + uploadStep!.with = { + ...((uploadStep!.with as Record) ?? {}), + "include-hidden-files": true, + "retention-days": 1, + }; + const cleanupStep = job.steps.find((step) => step.name === "Clean up Docker auth"); + expect(cleanupStep).toBeDefined(); + cleanupStep!.if = "success()"; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); + expect(errors).toEqual( + expect.arrayContaining([ + "diagnostics-vitest job env must not include NVIDIA_API_KEY", + "diagnostics-vitest job env must not include GITHUB_TOKEN", + "step 'Run diagnostics live test' run script must not interpolate dispatch inputs directly", + "diagnostics-vitest artifact upload must set include-hidden-files: false", + "diagnostics-vitest artifact upload retention-days must be 14", + "diagnostics-vitest Docker auth cleanup must always run", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("rejects Docker Hub auth in the Hermes root-entrypoint smoke job", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 9e55fc3ff3..1dfe8ff6db 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -1705,6 +1705,134 @@ function validateHermesRootEntrypointSmokeVitestJob( } } +function validateDiagnosticsVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "diagnostics-vitest"; + const scenarioName = "diagnostics"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing diagnostics-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("diagnostics-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + if (job["timeout-minutes"] !== 60) { + errors.push("diagnostics-vitest job must keep the 60 minute timeout"); + } + + const jobEnv = asRecord(job.env); + if (jobEnv.DOCKER_CONFIG !== "${{ github.workspace }}/.docker-config-diagnostics") { + errors.push("diagnostics-vitest job must isolate Docker auth under .docker-config-diagnostics"); + } + if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/diagnostics") { + errors.push("diagnostics-vitest job must write artifacts under e2e-artifacts/vitest/diagnostics"); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("diagnostics-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("diagnostics-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { + errors.push("diagnostics-vitest job must set NEMOCLAW_NON_INTERACTIVE=1"); + } + if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { + errors.push("diagnostics-vitest job must set NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1"); + } + if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-diag") { + errors.push("diagnostics-vitest job must use the stable e2e-diag sandbox name"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("diagnostics-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); + } + for (const secret of ["NVIDIA_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + requireEnvDoesNotExposeSecret(errors, "diagnostics-vitest job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `diagnostics-vitest step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run diagnostics live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("diagnostics-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "diagnostics-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("diagnostics-vitest checkout step must set persist-credentials=false"); + } + + const dockerLogin = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerLoginEnv = asRecord(dockerLogin?.env); + if (dockerLoginEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push("diagnostics-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets"); + } + if (dockerLoginEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push("diagnostics-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); + } + requireRunContains(errors, dockerLogin, 'mkdir -p "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, 'chmod 700 "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, "docker login docker.io"); + requireRunContains(errors, dockerLogin, "--password-stdin"); + requireRunContains(errors, dockerLogin, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("diagnostics-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "diagnostics-vitest setup-node"); + + const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run diagnostics live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { + errors.push("diagnostics-vitest Vitest step must receive NVIDIA_API_KEY from secrets"); + } + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/diagnostics.test.ts"); + requireRunDoesNotContain(errors, runVitest, "${{ inputs."); + + const upload = requireJobStep(errors, jobName, steps, "Upload diagnostics artifacts"); + requireFullShaAction(errors, upload, "diagnostics-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-diagnostics") { + errors.push("diagnostics-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/diagnostics/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("diagnostics-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("diagnostics-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("diagnostics-vitest artifact upload retention-days must be 14"); + } + + const cleanup = requireJobStep(errors, jobName, steps, "Clean up Docker auth"); + if (cleanup?.if !== "always()") { + errors.push("diagnostics-vitest Docker auth cleanup must always run"); + } + requireRunContains(errors, cleanup, "docker logout docker.io"); + requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); +} + function validateModelRouterProviderRoutedInferenceVitestJob( errors: string[], jobs: WorkflowRecord, @@ -2132,6 +2260,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( "issue-4434-tui-unreachable-inference-vitest", "issue-4434-tui-unreachable-inference", ); + validateDiagnosticsVitestJob(errors, jobs); validateModelRouterProviderRoutedInferenceVitestJob(errors, jobs); const reportToPr = asRecord(jobs["report-to-pr"]); From 3195c5d37a4c1aef19890558815bdeb3d164a41e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 17:57:20 -0400 Subject: [PATCH 4/5] test(e2e): tighten diagnostics workflow auth --- .github/workflows/e2e-vitest-scenarios.yaml | 34 ---------------- test/e2e-scenario/live/diagnostics.test.ts | 4 +- .../e2e-scenarios-workflow.test.ts | 20 ++++++++-- tools/e2e-scenarios/workflow-boundary.mts | 39 ++++++------------- 4 files changed, 29 insertions(+), 68 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 6e28e63858..c0ee8cf37e 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -1698,7 +1698,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 env: - DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-diagnostics E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/diagnostics NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1711,33 +1710,6 @@ jobs: with: persist-credentials: false - - name: Authenticate to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - shell: bash - run: | - set -euo pipefail - if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then - echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." - exit 0 - fi - mkdir -p "${DOCKER_CONFIG}" - chmod 700 "${DOCKER_CONFIG}" - login_succeeded=0 - for attempt in 1 2 3; do - if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then - login_succeeded=1 - break - fi - if [[ "$attempt" -lt 3 ]]; then - echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." - sleep 5 - fi - done - if [[ "$login_succeeded" -ne 1 ]]; then - echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." - fi - name: Set up Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 @@ -1774,12 +1746,6 @@ jobs: if-no-files-found: ignore retention-days: 14 - - name: Clean up Docker auth - if: always() - run: | - set -euo pipefail - docker logout docker.io || true - rm -rf "${DOCKER_CONFIG}" # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the diff --git a/test/e2e-scenario/live/diagnostics.test.ts b/test/e2e-scenario/live/diagnostics.test.ts index d2cddeb0ec..9e230a0ec0 100644 --- a/test/e2e-scenario/live/diagnostics.test.ts +++ b/test/e2e-scenario/live/diagnostics.test.ts @@ -241,8 +241,8 @@ runDiagnosticsTest( ).toBeGreaterThan(0); expect( quickElapsedMs, - "debug --quick must complete within the legacy 30s budget", - ).toBeLessThanOrEqual(DEBUG_QUICK_TIMEOUT_MS); + "debug --quick must complete within the legacy 30s process timeout plus harness scheduling grace", + ).toBeLessThanOrEqual(DEBUG_QUICK_TIMEOUT_MS + 5_000); const install = await host.command( "bash", diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index bc676c5147..071875b4b8 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -714,9 +714,20 @@ jobs: expect(job.steps).toEqual(expect.any(Array)); job.env = { ...job.env, + DOCKER_CONFIG: "${{ github.workspace }}/.docker-config-diagnostics", NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", GITHUB_TOKEN: "${{ github.token }}", }; + const setupNodeIndex = job.steps.findIndex((step) => step.name === "Set up Node"); + expect(setupNodeIndex).toBeGreaterThan(0); + job.steps.splice(setupNodeIndex, 0, { + name: "Authenticate to Docker Hub", + env: { + DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}", + DOCKERHUB_TOKEN: "${{ secrets.DOCKERHUB_TOKEN }}", + }, + run: 'docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin', + }); const runStep = job.steps.find((step) => step.name === "Run diagnostics live test"); expect(runStep).toBeDefined(); runStep!.run = `${runStep!.run}\necho "\${{ inputs.jobs }}"`; @@ -727,21 +738,22 @@ jobs: "include-hidden-files": true, "retention-days": 1, }; - const cleanupStep = job.steps.find((step) => step.name === "Clean up Docker auth"); - expect(cleanupStep).toBeDefined(); - cleanupStep!.if = "success()"; fs.writeFileSync(workflowPath, YAML.stringify(workflow)); try { const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); expect(errors).toEqual( expect.arrayContaining([ + "diagnostics-vitest job must not expose Docker auth to branch-controlled steps", "diagnostics-vitest job env must not include NVIDIA_API_KEY", "diagnostics-vitest job env must not include GITHUB_TOKEN", + "diagnostics-vitest job must not authenticate to Docker Hub before branch-controlled test code runs", + "diagnostics-vitest step 'Authenticate to Docker Hub' env must not include DOCKERHUB_USERNAME", + "diagnostics-vitest step 'Authenticate to Docker Hub' env must not include DOCKERHUB_TOKEN", + "diagnostics-vitest step 'Authenticate to Docker Hub' run script must not use docker login or inline secret interpolation", "step 'Run diagnostics live test' run script must not interpolate dispatch inputs directly", "diagnostics-vitest artifact upload must set include-hidden-files: false", "diagnostics-vitest artifact upload retention-days must be 14", - "diagnostics-vitest Docker auth cleanup must always run", ]), ); } finally { diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 1dfe8ff6db..2182ef86ed 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -1723,8 +1723,8 @@ function validateDiagnosticsVitestJob(errors: string[], jobs: WorkflowRecord): v } const jobEnv = asRecord(job.env); - if (jobEnv.DOCKER_CONFIG !== "${{ github.workspace }}/.docker-config-diagnostics") { - errors.push("diagnostics-vitest job must isolate Docker auth under .docker-config-diagnostics"); + if ("DOCKER_CONFIG" in jobEnv) { + errors.push("diagnostics-vitest job must not expose Docker auth to branch-controlled steps"); } if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/diagnostics") { errors.push("diagnostics-vitest job must write artifacts under e2e-artifacts/vitest/diagnostics"); @@ -1759,14 +1759,18 @@ function validateDiagnosticsVitestJob(errors: string[], jobs: WorkflowRecord): v if (step.name !== "Run diagnostics live test") { requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); } - if (step.name !== "Authenticate to Docker Hub") { - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); - requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); - } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); } + if (namedStep(steps, "Authenticate to Docker Hub")) { + errors.push( + "diagnostics-vitest job must not authenticate to Docker Hub before branch-controlled test code runs", + ); + } + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); if (!checkout) errors.push("diagnostics-vitest job missing checkout step"); requireFullShaAction(errors, checkout, "diagnostics-vitest checkout"); @@ -1774,20 +1778,6 @@ function validateDiagnosticsVitestJob(errors: string[], jobs: WorkflowRecord): v errors.push("diagnostics-vitest checkout step must set persist-credentials=false"); } - const dockerLogin = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); - const dockerLoginEnv = asRecord(dockerLogin?.env); - if (dockerLoginEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { - errors.push("diagnostics-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets"); - } - if (dockerLoginEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { - errors.push("diagnostics-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); - } - requireRunContains(errors, dockerLogin, 'mkdir -p "${DOCKER_CONFIG}"'); - requireRunContains(errors, dockerLogin, 'chmod 700 "${DOCKER_CONFIG}"'); - requireRunContains(errors, dockerLogin, "docker login docker.io"); - requireRunContains(errors, dockerLogin, "--password-stdin"); - requireRunContains(errors, dockerLogin, "continuing with anonymous pulls"); - const setupNode = namedStep(steps, "Set up Node"); if (!setupNode) errors.push("diagnostics-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "diagnostics-vitest setup-node"); @@ -1824,13 +1814,6 @@ function validateDiagnosticsVitestJob(errors: string[], jobs: WorkflowRecord): v if (uploadWith["retention-days"] !== 14) { errors.push("diagnostics-vitest artifact upload retention-days must be 14"); } - - const cleanup = requireJobStep(errors, jobName, steps, "Clean up Docker auth"); - if (cleanup?.if !== "always()") { - errors.push("diagnostics-vitest Docker auth cleanup must always run"); - } - requireRunContains(errors, cleanup, "docker logout docker.io"); - requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); } function validateModelRouterProviderRoutedInferenceVitestJob( From 0e46c0f68f19b7bb87741fdaa96b0c1cd618abe6 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 18:21:38 -0400 Subject: [PATCH 5/5] test(e2e): let quick diagnostics run pre-onboard --- test/e2e-scenario/live/diagnostics.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-scenario/live/diagnostics.test.ts b/test/e2e-scenario/live/diagnostics.test.ts index 9e230a0ec0..cba9762398 100644 --- a/test/e2e-scenario/live/diagnostics.test.ts +++ b/test/e2e-scenario/live/diagnostics.test.ts @@ -228,7 +228,7 @@ runDiagnosticsTest( [CLI_ENTRYPOINT, "debug", "--quick", "--output", quickArchive], { artifactName: "diagnostics-debug-quick", - env: testEnv(home), + env: testEnv(home, { NEMOCLAW_SANDBOX_NAME: "" }), timeoutMs: DEBUG_QUICK_TIMEOUT_MS, }, );