From 3e3448ae7c3008185fa96b4d431383241d92e580 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 09:00:49 -0400 Subject: [PATCH 1/9] test(e2e): migrate Hermes root entrypoint smoke --- .github/workflows/sandbox-images-and-e2e.yaml | 29 +- .../live/hermes-root-entrypoint-smoke.test.ts | 449 ++++++++++++++++++ 2 files changed, 472 insertions(+), 6 deletions(-) create mode 100644 test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 14efdb8f2e..245a12643d 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -97,18 +97,35 @@ jobs: path: /tmp/nemoclaw-hermes-sandbox-secret-boundary.log if-no-files-found: ignore - - name: Run Hermes root entrypoint smoke + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Run Hermes root entrypoint smoke Vitest test env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-root-entrypoint-smoke NEMOCLAW_HERMES_TEST_IMAGE: nemoclaw-hermes-production - run: bash test/e2e/test-hermes-root-entrypoint-smoke.sh + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts \ + --silent=false --reporter=default - - name: Upload Hermes root entrypoint smoke log on failure - if: failure() + - name: Upload Hermes root entrypoint smoke artifacts + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: hermes-root-entrypoint-smoke-log - path: /tmp/nemoclaw-hermes-root-entrypoint-smoke.log + name: hermes-root-entrypoint-smoke-artifacts + path: e2e-artifacts/vitest/hermes-root-entrypoint-smoke/ + include-hidden-files: false if-no-files-found: ignore + retention-days: 14 build-sandbox-images-arm64: if: inputs.run_arm64 diff --git a/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts b/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts new file mode 100644 index 0000000000..9be68950e3 --- /dev/null +++ b/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts @@ -0,0 +1,449 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +import type { ArtifactSink } from "../fixtures/artifacts.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; + +// Migrated from test/e2e/test-hermes-root-entrypoint-smoke.sh. This remains a +// real Docker/root-entrypoint smoke: it builds the Hermes image when no prebuilt +// NEMOCLAW_HERMES_TEST_IMAGE is supplied, starts /usr/local/bin/nemoclaw-start +// as root, and verifies health, gateway privilege separation, runtime layout, +// sticky config protection, and legacy gateway.pid symlink migration. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const HEALTH_ATTEMPTS = 90; +const HEALTH_POLL_MS = 2_000; +const DEFAULT_COMMAND_TIMEOUT_MS = 30_000; +const BUILD_TIMEOUT_MS = 10 * 60_000; +const RUN_TIMEOUT_MS = 60_000; + +const liveTest = process.env.NEMOCLAW_RUN_E2E_SCENARIOS === "1" ? test : test.skip; + +type CommandResult = { + command: string[]; + exitCode: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; + error?: string; +}; + +function safeName(value: string): string { + return ( + value + .trim() + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, "") || "docker" + ); +} + +function safeTag(value: string): string { + return value.replace(/[^A-Za-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "") || "local"; +} + +function resultText(result: CommandResult): string { + return [ + `$ ${result.command.join(" ")}`, + result.stdout.trim(), + result.stderr.trim(), + result.error ? `error: ${result.error}` : "", + ] + .filter(Boolean) + .join("\n"); +} + +class DockerProbe { + private sequence = 0; + + constructor(private readonly artifacts: ArtifactSink) {} + + async run( + args: string[], + options: { artifactName: string; timeoutMs?: number } = { artifactName: "docker" }, + ): Promise { + const command = ["docker", ...args]; + const result = spawnSync("docker", args, { + cwd: REPO_ROOT, + encoding: "utf8", + env: process.env, + maxBuffer: 10 * 1024 * 1024, + timeout: options.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS, + }); + const commandResult: CommandResult = { + command, + exitCode: result.status, + signal: result.signal, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + error: result.error instanceof Error ? result.error.message : undefined, + }; + const artifactBase = `docker/${String(++this.sequence).padStart(3, "0")}-${safeName( + options.artifactName, + )}`; + await this.artifacts.writeText(`${artifactBase}.stdout.txt`, commandResult.stdout); + await this.artifacts.writeText(`${artifactBase}.stderr.txt`, commandResult.stderr); + await this.artifacts.writeJson(`${artifactBase}.result.json`, commandResult); + return commandResult; + } + + async expect( + args: string[], + options: { artifactName: string; timeoutMs?: number }, + ): Promise { + const result = await this.run(args, options); + expect(result.exitCode, resultText(result)).toBe(0); + return result; + } +} + +async function requireDocker(probe: DockerProbe, skip: (message: string) => void): Promise { + const result = await probe.run(["info"], { artifactName: "docker-info", timeoutMs: 30_000 }); + if (result.exitCode === 0) return; + + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for Hermes root-entrypoint smoke:\n${resultText(result)}`); + } + skip("Docker daemon is required for Hermes root-entrypoint smoke"); +} + +async function buildImageIfNeeded( + probe: DockerProbe, + image: string, + baseImage: string, +): Promise { + if (process.env.NEMOCLAW_HERMES_TEST_IMAGE) { + await probe.expect(["image", "inspect", image], { + artifactName: "inspect-prebuilt-hermes-image", + timeoutMs: 30_000, + }); + return; + } + + await probe.expect( + [ + "build", + "-f", + path.join(REPO_ROOT, "agents/hermes/Dockerfile.base"), + "-t", + baseImage, + REPO_ROOT, + ], + { artifactName: "build-hermes-base-image", timeoutMs: BUILD_TIMEOUT_MS }, + ); + await probe.expect( + [ + "build", + "-f", + path.join(REPO_ROOT, "agents/hermes/Dockerfile"), + "--build-arg", + `BASE_IMAGE=${baseImage}`, + "-t", + image, + REPO_ROOT, + ], + { artifactName: "build-hermes-production-image", timeoutMs: BUILD_TIMEOUT_MS }, + ); +} + +async function dockerExecSh( + probe: DockerProbe, + container: string, + script: string, + artifactName: string, +): Promise { + return probe.run(["exec", container, "sh", "-lc", script], { artifactName }); +} + +async function expectContainerSh( + probe: DockerProbe, + container: string, + message: string, + script: string, +): Promise { + const result = await dockerExecSh(probe, container, script, message); + expect(result.exitCode, `${container}: ${message}\n${resultText(result)}`).toBe(0); + return result; +} + +async function expectContainerShFails( + probe: DockerProbe, + container: string, + message: string, + script: string, +): Promise { + const result = await dockerExecSh(probe, container, script, message); + expect(result.exitCode, `${container}: ${message}\n${resultText(result)}`).not.toBe(0); +} + +async function dumpContainerDiagnostics(probe: DockerProbe, container: string): Promise { + const inspect = await probe.run(["inspect", container], { + artifactName: `diag-${container}-inspect`, + timeoutMs: 30_000, + }); + if (inspect.exitCode !== 0) return; + + await probe.run( + [ + "ps", + "-a", + "--filter", + `name=^/${container}$`, + "--format", + "table {{.Names}}\t{{.Status}}\t{{.Image}}", + ], + { artifactName: `diag-${container}-ps`, timeoutMs: 30_000 }, + ); + await probe.run(["logs", container], { + artifactName: `diag-${container}-logs`, + timeoutMs: 30_000, + }); + await probe.run( + [ + "exec", + container, + "sh", + "-lc", + [ + "set +e", + 'echo "== identity =="', + "id", + 'echo "== hermes tree =="', + "ls -ld /sandbox/.hermes /sandbox/.hermes/runtime /sandbox/.hermes/logs /sandbox/.hermes/logs/curator /sandbox/.hermes/hooks /sandbox/.hermes/image_cache /sandbox/.hermes/audio_cache 2>&1", + "ls -l /sandbox/.hermes/gateway.pid /sandbox/.hermes/runtime/gateway.pid /sandbox/.hermes/config.yaml 2>&1", + 'echo "== processes =="', + 'ps -eo user=,pid=,args= | grep -E "hermes|socat" | grep -v grep', + 'echo "== start log =="', + "tail -n 120 /tmp/nemoclaw-start.log 2>&1", + 'echo "== gateway log =="', + "tail -n 160 /tmp/gateway.log 2>&1", + ].join("; "), + ], + { artifactName: `diag-${container}-runtime`, timeoutMs: 30_000 }, + ); +} + +async function waitForHealth(probe: DockerProbe, container: string): Promise { + for (let attempt = 1; attempt <= HEALTH_ATTEMPTS; attempt++) { + const health = await dockerExecSh( + probe, + container, + "curl -sf --max-time 2 http://127.0.0.1:8642/health", + `${container}-health-${attempt}`, + ); + if (health.exitCode === 0) { + expect(health.stdout, `${container}: health response did not report status ok`).toMatch( + /"status"\s*:\s*"ok"/, + ); + expect(health.stdout, `${container}: health response did not report Hermes platform`).toMatch( + /"platform"\s*:\s*"hermes-agent"/, + ); + return; + } + + const running = await probe.run(["inspect", "-f", "{{.State.Running}}", container], { + artifactName: `${container}-running-${attempt}`, + timeoutMs: 30_000, + }); + if (running.stdout.trim() !== "true") { + throw new Error( + `${container}: container exited before health became ready\n${resultText(running)}`, + ); + } + await delay(HEALTH_POLL_MS); + } + + throw new Error(`${container}: Hermes health did not become ready`); +} + +async function assertGatewayLogClean(probe: DockerProbe, container: string): Promise { + await expectContainerSh( + probe, + container, + "gateway log contains PID race failure", + "! grep -F 'PID file race lost' /tmp/gateway.log", + ); + await expectContainerSh( + probe, + container, + "gateway log contains config load failure", + "! grep -F 'Could not load config.yaml' /tmp/gateway.log", + ); +} + +async function assertRuntimeLayout(probe: DockerProbe, container: string): Promise { + await expectContainerSh( + probe, + container, + "Hermes config root mode is not 3770", + "[ \"$(stat -c '%a' /sandbox/.hermes)\" = '3770' ]", + ); + await expectContainerSh( + probe, + container, + "required Hermes v0.14 directories are missing", + 'for dir in hooks image_cache audio_cache logs/curator; do test -d "/sandbox/.hermes/$dir"; done', + ); + await expectContainerSh( + probe, + container, + "gateway user cannot write required Hermes v0.14 directories", + 'gosu gateway sh -lc \'for dir in hooks image_cache audio_cache logs/curator; do p="/sandbox/.hermes/$dir/.nemoclaw-write-test"; : >"$p" && rm -f "$p"; done\'', + ); + await expectContainerSh( + probe, + container, + "gateway.pid is not a regular top-level file", + "test -f /sandbox/.hermes/gateway.pid && test ! -L /sandbox/.hermes/gateway.pid", + ); + await expectContainerShFails( + probe, + container, + "gateway user was able to remove config.yaml", + "gosu gateway rm /sandbox/.hermes/config.yaml", + ); + await expectContainerSh( + probe, + container, + "config.yaml disappeared after gateway remove attempt", + "test -f /sandbox/.hermes/config.yaml", + ); +} + +async function assertGatewayProcess(probe: DockerProbe, container: string): Promise { + await expectContainerSh( + probe, + container, + "Hermes gateway process is not running as gateway user", + 'ps -eo user=,args= | awk \'$1 == "gateway" && (index($0, "hermes gateway run") || index($0, "hermes.real gateway run")) { found = 1 } END { exit found ? 0 : 1 }\'', + ); + await expectContainerSh( + probe, + container, + "start log does not show gateway privilege separation", + "grep -F \"hermes gateway launched as 'gateway' user\" /tmp/nemoclaw-start.log", + ); +} + +async function runCleanVariant( + probe: DockerProbe, + image: string, + runId: string, + containers: string[], +): Promise { + const container = `nemoclaw-hermes-root-clean-${runId}`; + await probe.expect(["run", "-d", "--name", container, image, "/usr/local/bin/nemoclaw-start"], { + artifactName: "start-clean-root-entrypoint-container", + timeoutMs: RUN_TIMEOUT_MS, + }); + containers.push(container); + + await waitForHealth(probe, container); + await assertGatewayProcess(probe, container); + await assertGatewayLogClean(probe, container); + await assertRuntimeLayout(probe, container); +} + +async function runLegacyVariant( + probe: DockerProbe, + image: string, + runId: string, + containers: string[], +): Promise { + const container = `nemoclaw-hermes-root-legacy-${runId}`; + const legacyBootstrap = `set -euo pipefail +rm -f /sandbox/.hermes/gateway.pid +printf "stale pid\n" >/sandbox/.hermes/runtime/gateway.pid +printf "stale lock\n" >/sandbox/.hermes/runtime/gateway.lock +ln -s runtime/gateway.pid /sandbox/.hermes/gateway.pid +chmod 750 /sandbox/.hermes +rm -rf /sandbox/.hermes/hooks /sandbox/.hermes/image_cache /sandbox/.hermes/audio_cache /sandbox/.hermes/logs/curator +exec /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-start`; + + await probe.expect( + ["run", "-d", "--name", container, "--entrypoint", "/bin/bash", image, "-lc", legacyBootstrap], + { artifactName: "start-legacy-layout-root-entrypoint-container", timeoutMs: RUN_TIMEOUT_MS }, + ); + containers.push(container); + + await waitForHealth(probe, container); + await assertGatewayProcess(probe, container); + await assertGatewayLogClean(probe, container); + await assertRuntimeLayout(probe, container); + await expectContainerSh( + probe, + container, + "legacy gateway.pid symlink migration was not logged", + "grep -F 'Removing unsafe stale Hermes legacy PID file symlink' /tmp/nemoclaw-start.log", + ); +} + +liveTest( + "hermes root-entrypoint smoke preserves runtime layout and legacy pid migration", + async ({ artifacts, cleanup, skip }) => { + const probe = new DockerProbe(artifacts); + const runId = safeTag(`${process.env.GITHUB_RUN_ID ?? "local"}-${process.pid}-${Date.now()}`); + const image = + process.env.NEMOCLAW_HERMES_TEST_IMAGE ?? `nemoclaw-hermes-root-entrypoint-smoke:${runId}`; + const baseImage = `nemoclaw-hermes-root-entrypoint-base:${runId}`; + const containers: string[] = []; + + await artifacts.writeJson("scenario.json", { + id: "hermes-root-entrypoint-smoke", + runner: "vitest", + boundary: "docker-root-entrypoint", + legacySource: "test/e2e/test-hermes-root-entrypoint-smoke.sh", + image, + prebuiltImage: Boolean(process.env.NEMOCLAW_HERMES_TEST_IMAGE), + contract: [ + "clean root-entrypoint startup reaches Hermes health", + "gateway process runs as gateway user", + "gateway log has no PID race or config load failure", + "Hermes v0.14 writable runtime directories are present", + "gateway.pid is migrated to a regular top-level file", + "gateway user cannot remove config.yaml from sticky config root", + "legacy gateway.pid symlink/state shape is repaired and booted", + ], + }); + + cleanup.add("remove Hermes root-entrypoint smoke containers", async () => { + await Promise.all( + containers.map((container) => + probe.run(["rm", "-f", container], { + artifactName: `cleanup-${container}`, + timeoutMs: 30_000, + }), + ), + ); + }); + + await requireDocker(probe, skip); + + try { + await buildImageIfNeeded(probe, image, baseImage); + await runCleanVariant(probe, image, runId, containers); + await runLegacyVariant(probe, image, runId, containers); + } catch (error) { + for (const container of containers) { + await dumpContainerDiagnostics(probe, container); + } + throw error; + } + + await artifacts.writeJson("scenario-result.json", { + id: "hermes-root-entrypoint-smoke", + image, + assertions: { + cleanStartupHealthy: true, + legacyStartupHealthy: true, + runtimeLayoutVerified: true, + gatewayPrivilegeSeparationVerified: true, + legacyPidSymlinkMigrationVerified: true, + }, + }); + }, +); From da55fedbc97345b5ba561173300cdbcc28cdcd35 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 11:15:47 -0400 Subject: [PATCH 2/9] ci(e2e): allow Hermes root smoke dispatch --- .github/workflows/e2e-vitest-scenarios.yaml | 91 ++++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 07bb8c54f5..f7094134ff 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -45,14 +45,33 @@ jobs: run: | set -euo pipefail args=(--emit-live-matrix) + matrix="" if [ -n "${SCENARIOS}" ]; then if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then echo "::error::Invalid scenario input: ${SCENARIOS}" >&2 exit 1 fi - args+=(--scenarios "${SCENARIOS}") + + standalone_scenarios=",hermes-root-entrypoint-smoke," + registry_scenarios=() + IFS=',' read -ra requested_scenarios <<< "${SCENARIOS}" + for scenario in "${requested_scenarios[@]}"; do + if [[ "${standalone_scenarios}" == *",${scenario},"* ]]; then + continue + fi + registry_scenarios+=("${scenario}") + done + + if [ "${#registry_scenarios[@]}" -gt 0 ]; then + registry_csv="$(IFS=,; echo "${registry_scenarios[*]}")" + args+=(--scenarios "${registry_csv}") + else + matrix="[]" + fi + fi + if [ -z "${matrix}" ]; then + matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" fi - matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" MATRIX_JSON="${matrix}" python - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json @@ -69,6 +88,7 @@ jobs: live-scenarios: needs: generate-matrix + if: ${{ needs.generate-matrix.outputs.matrix != '[]' }} runs-on: ${{ matrix.runner }} timeout-minutes: 45 strategy: @@ -250,6 +270,73 @@ jobs: if-no-files-found: ignore retention-days: 14 + hermes-root-entrypoint-smoke-vitest: + if: ${{ inputs.scenarios == '' || contains(format(',{0},', inputs.scenarios), ',hermes-root-entrypoint-smoke,') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-root-entrypoint-smoke + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Run Hermes root entrypoint smoke live test + # Migrated from test/e2e/test-hermes-root-entrypoint-smoke.sh. This + # builds the real Hermes image unless NEMOCLAW_HERMES_TEST_IMAGE points + # at a prebuilt image, then probes the root entrypoint via Docker. + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts \ + --silent=false --reporter=default + + - name: Upload Hermes root entrypoint smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-hermes-root-entrypoint-smoke + path: e2e-artifacts/vitest/hermes-root-entrypoint-smoke/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. From 2647e83a9219903c2cc8475356c72082b34ffffd Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 12:33:37 -0400 Subject: [PATCH 3/9] ci(e2e): align hermes-root-entrypoint-smoke-vitest dispatch --- .github/workflows/e2e-vitest-scenarios.yaml | 75 ++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 65e06c7eda..f84fe4f2ad 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -40,7 +40,7 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,hermes-root-entrypoint-smoke-vitest" if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then echo "::error::Use either scenarios or jobs, not both." >&2 exit 1 @@ -298,6 +298,78 @@ jobs: if-no-files-found: ignore retention-days: 14 + hermes-root-entrypoint-smoke-vitest: + needs: validate-jobs + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-vitest,') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-root-entrypoint-smoke + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Run Hermes root entrypoint smoke live test + # Migrated from test/e2e/test-hermes-root-entrypoint-smoke.sh. This + # builds the real Hermes image unless NEMOCLAW_HERMES_TEST_IMAGE points + # at a prebuilt image, then probes the root entrypoint via Docker. + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts \ + --silent=false --reporter=default + + - name: Upload Hermes root entrypoint smoke artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-hermes-root-entrypoint-smoke + path: e2e-artifacts/vitest/hermes-root-entrypoint-smoke/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + # Focused coverage slice for the #2603/#3145 OpenClaw websocket + # protocol/history contract. The retained legacy bash lane remains the + # source for full closeout until a later PR proves replacement and deletes it. + # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. @@ -481,6 +553,7 @@ jobs: live-scenarios, openshell-version-pin-vitest, onboard-negative-paths-vitest, + hermes-root-entrypoint-smoke-vitest, openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, ] From 38a4c9c7833ef791fa28e6670b7b5dd2a89041c7 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 20:54:49 -0400 Subject: [PATCH 4/9] fix(e2e): harden hermes smoke workflow coverage Signed-off-by: Julie Yaunches --- .github/workflows/sandbox-images-and-e2e.yaml | 2 +- .../live/hermes-root-entrypoint-smoke.test.ts | 20 +- .../e2e-scenarios-workflow.test.ts | 28 +++ tools/e2e-scenarios/workflow-boundary.mts | 175 ++++++++++++++++++ 4 files changed, 217 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 245a12643d..373ec3f829 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -59,7 +59,7 @@ jobs: build-hermes-sandbox-image: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 diff --git a/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts b/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts index c01e3bcd03..273507ff3f 100644 --- a/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts +++ b/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts @@ -7,6 +7,7 @@ import { setTimeout as delay } from "node:timers/promises"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import type { SecretStore } from "../fixtures/secrets.ts"; // Migrated from test/e2e/test-hermes-root-entrypoint-smoke.sh. This remains a // real Docker/root-entrypoint smoke: it builds the Hermes image when no prebuilt @@ -60,7 +61,10 @@ function resultText(result: CommandResult): string { class DockerProbe { private sequence = 0; - constructor(private readonly artifacts: ArtifactSink) {} + constructor( + private readonly artifacts: ArtifactSink, + private readonly redact: SecretStore["redact"], + ) {} async run( args: string[], @@ -75,12 +79,12 @@ class DockerProbe { timeout: options.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS, }); const commandResult: CommandResult = { - command, + command: command.map((part) => this.redact(part)), exitCode: result.status, signal: result.signal, - stdout: result.stdout ?? "", - stderr: result.stderr ?? "", - error: result.error instanceof Error ? result.error.message : undefined, + stdout: this.redact(result.stdout ?? ""), + stderr: this.redact(result.stderr ?? ""), + error: result.error instanceof Error ? this.redact(result.error.message) : undefined, }; const artifactBase = `docker/${String(++this.sequence).padStart(3, "0")}-${safeName( options.artifactName, @@ -384,8 +388,10 @@ exec /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-start`; liveTest( "hermes root-entrypoint smoke preserves runtime layout and legacy pid migration", - async ({ artifacts, cleanup, skip }) => { - const probe = new DockerProbe(artifacts); + async ({ artifacts, cleanup, secrets, skip }) => { + const probe = new DockerProbe(artifacts, (text, extraValues) => + secrets.redact(text, extraValues), + ); const runId = safeTag(`${process.env.GITHUB_RUN_ID ?? "local"}-${process.pid}-${Date.now()}`); const image = process.env.NEMOCLAW_HERMES_TEST_IMAGE ?? `nemoclaw-hermes-root-entrypoint-smoke:${runId}`; diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 183375e4e9..026370f9b6 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -136,6 +136,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["hermes-e2e-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "hermes-root-entrypoint-smoke" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["hermes-root-entrypoint-smoke-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "hermes-root-entrypoint-smoke-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["hermes-root-entrypoint-smoke-vitest"], + registryScenarios: [], + }); expect( evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "rebuild-openclaw" }), ).toMatchObject({ @@ -195,6 +211,18 @@ describe("e2e-vitest-scenarios workflow boundary", () => { hermes_selected: "true", matrix: "[]", }); + expect( + generateMatrixForDispatch({ JOBS: "hermes-root-entrypoint-smoke-vitest", SCENARIOS: "" }), + ).toMatchObject({ + hermes_selected: "false", + matrix: "[]", + }); + expect( + generateMatrixForDispatch({ JOBS: "", SCENARIOS: "hermes-root-entrypoint-smoke" }), + ).toMatchObject({ + hermes_selected: "false", + matrix: "[]", + }); }); it("flags direct dispatch-input interpolation and unsafe artifact upload", () => { diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 9851792dbe..178a20204e 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -23,6 +23,7 @@ const FREE_STANDING_SCENARIO_JOBS = new Map([ ["onboard-negative-paths", "onboard-negative-paths-vitest"], ["runtime-overrides", "runtime-overrides-vitest"], ["hermes-e2e", "hermes-e2e-vitest"], + ["hermes-root-entrypoint-smoke", "hermes-root-entrypoint-smoke-vitest"], ["network-policy", "network-policy-vitest"], ["rebuild-openclaw", "rebuild-openclaw-vitest"], ["token-rotation", "token-rotation-vitest"], @@ -283,6 +284,7 @@ function validateJobsSelector(errors: string[], jobs: WorkflowRecord): void { requireRunContains(errors, validate, "runtime-overrides-vitest"); requireRunContains(errors, validate, "double-onboard-vitest"); requireRunContains(errors, validate, "hermes-e2e-vitest"); + requireRunContains(errors, validate, "hermes-root-entrypoint-smoke-vitest"); requireRunContains(errors, validate, "network-policy-vitest"); requireRunContains(errors, validate, "rebuild-openclaw-vitest"); requireRunContains(errors, validate, "token-rotation-vitest"); @@ -1109,6 +1111,175 @@ function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): voi } } +function validateHermesRootEntrypointSmokeVitestJob( + errors: string[], + jobs: WorkflowRecord, +): void { + const jobName = "hermes-root-entrypoint-smoke-vitest"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing hermes-root-entrypoint-smoke-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("hermes-root-entrypoint-smoke-vitest job must run on ubuntu-latest"); + } + const needs = Array.isArray(job.needs) ? job.needs : []; + if (!needs.includes("validate-jobs") || !needs.includes("generate-matrix")) { + errors.push( + "hermes-root-entrypoint-smoke-vitest job must depend on validate-jobs and generate-matrix", + ); + } + const expectedIf = + "${{ needs.generate-matrix.result == 'success' && ((inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-vitest,') || contains(format(',{0},', inputs.scenarios), ',hermes-root-entrypoint-smoke,')) }}"; + if (job.if !== expectedIf) { + errors.push( + "hermes-root-entrypoint-smoke-vitest job must gate on generate-matrix and the shared selector condition", + ); + } + if (job["timeout-minutes"] !== 45) { + errors.push("hermes-root-entrypoint-smoke-vitest job must keep the 45 minute timeout"); + } + + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("hermes-root-entrypoint-smoke-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/vitest/hermes-root-entrypoint-smoke" + ) { + errors.push( + "hermes-root-entrypoint-smoke-vitest job must write artifacts under e2e-artifacts/vitest/hermes-root-entrypoint-smoke", + ); + } + requireEnvDoesNotExposeSecret( + errors, + "hermes-root-entrypoint-smoke-vitest job", + jobEnv, + "NVIDIA_API_KEY", + ); + requireEnvDoesNotExposeSecret( + errors, + "hermes-root-entrypoint-smoke-vitest job", + jobEnv, + "DOCKERHUB_USERNAME", + ); + requireEnvDoesNotExposeSecret( + errors, + "hermes-root-entrypoint-smoke-vitest job", + jobEnv, + "DOCKERHUB_TOKEN", + ); + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = step.name ?? step.uses ?? ""; + const stepEnv = asRecord(step.env); + requireEnvDoesNotExposeSecret( + errors, + `hermes-root-entrypoint-smoke-vitest step '${stepName}'`, + stepEnv, + "NVIDIA_API_KEY", + ); + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret( + errors, + `hermes-root-entrypoint-smoke-vitest step '${stepName}'`, + stepEnv, + "DOCKERHUB_USERNAME", + ); + requireEnvDoesNotExposeSecret( + errors, + `hermes-root-entrypoint-smoke-vitest step '${stepName}'`, + stepEnv, + "DOCKERHUB_TOKEN", + ); + } + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("hermes-root-entrypoint-smoke-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "hermes-root-entrypoint-smoke-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("hermes-root-entrypoint-smoke-vitest checkout step must set persist-credentials=false"); + } + + const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerHubEnv = asRecord(dockerHubAuth?.env); + if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push( + "hermes-root-entrypoint-smoke-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); + } + if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push( + "hermes-root-entrypoint-smoke-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); + } + requireRunContains(errors, dockerHubAuth, "docker login docker.io"); + requireRunContains(errors, dockerHubAuth, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("hermes-root-entrypoint-smoke-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "hermes-root-entrypoint-smoke-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const runVitest = requireJobStep( + errors, + jobName, + steps, + "Run Hermes root entrypoint smoke live test", + ); + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains( + errors, + runVitest, + "test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts", + ); + requireRunDoesNotContain(errors, runVitest, "${{ inputs."); + + const upload = requireJobStep( + errors, + jobName, + steps, + "Upload Hermes root entrypoint smoke artifacts", + ); + requireFullShaAction(errors, upload, "hermes-root-entrypoint-smoke-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-hermes-root-entrypoint-smoke") { + errors.push("hermes-root-entrypoint-smoke-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains( + errors, + uploadPath, + "e2e-artifacts/vitest/hermes-root-entrypoint-smoke/", + ); + if (uploadWith["include-hidden-files"] !== false) { + errors.push( + "hermes-root-entrypoint-smoke-vitest artifact upload must set include-hidden-files: false", + ); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push( + "hermes-root-entrypoint-smoke-vitest artifact upload must ignore missing fixture artifacts", + ); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("hermes-root-entrypoint-smoke-vitest artifact upload retention-days must be 14"); + } +} + export function validateE2eVitestScenariosWorkflowBoundary( workflowPath = DEFAULT_VITEST_WORKFLOW_PATH, ): string[] { @@ -1170,6 +1341,8 @@ export function validateE2eVitestScenariosWorkflowBoundary( requireRunContains(errors, generate, "runtime-overrides"); requireRunContains(errors, generate, "double-onboard-vitest"); requireRunContains(errors, generate, "hermes-e2e-vitest"); + requireRunContains(errors, generate, "hermes-root-entrypoint-smoke-vitest"); + requireRunContains(errors, generate, "hermes-root-entrypoint-smoke"); requireRunContains(errors, generate, "network-policy-vitest"); requireRunContains(errors, generate, "rebuild-openclaw-vitest"); requireRunContains(errors, generate, "token-rotation-vitest"); @@ -1329,6 +1502,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateRuntimeOverridesVitestJob(errors, jobs); validateDoubleOnboardVitestJob(errors, jobs); validateHermesE2EVitestJob(errors, jobs); + validateHermesRootEntrypointSmokeVitestJob(errors, jobs); validateNetworkPolicyVitestJob(errors, jobs); validateRebuildOpenClawVitestJob(errors, jobs); validateTokenRotationVitestJob(errors, jobs); @@ -1360,6 +1534,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( "credential-migration-vitest", "runtime-overrides-vitest", "hermes-e2e-vitest", + "hermes-root-entrypoint-smoke-vitest", "network-policy-vitest", "rebuild-openclaw-vitest", "token-rotation-vitest", From 6027ed6fa9643ccf54f4d0d391b18fa32f55a17d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 02:36:22 -0400 Subject: [PATCH 5/9] fix(e2e): isolate hermes smoke docker auth --- .github/workflows/e2e-vitest-scenarios.yaml | 26 ---------- .../e2e-scenarios-workflow.test.ts | 35 +++++++++++++ tools/e2e-scenarios/workflow-boundary.mts | 51 +++++++++---------- 3 files changed, 58 insertions(+), 54 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 5efd509f95..6f9c145efe 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -376,32 +376,6 @@ jobs: with: persist-credentials: false - - name: Authenticate to Docker Hub - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - shell: bash - run: | - set -euo pipefail - if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then - echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." - exit 0 - fi - login_succeeded=0 - for attempt in 1 2 3; do - if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then - login_succeeded=1 - break - fi - if [[ "$attempt" -lt 3 ]]; then - echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." - sleep 5 - fi - done - if [[ "$login_succeeded" -ne 1 ]]; then - echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." - fi - - name: Set up Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 with: diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 6fa304a126..de38091ac6 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -687,6 +687,41 @@ jobs: } }); + it("rejects Docker Hub auth in the Hermes root-entrypoint smoke job", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record> }>; + }; + const steps = workflow.jobs["hermes-root-entrypoint-smoke-vitest"]?.steps; + expect(steps).toEqual(expect.any(Array)); + const setupNodeIndex = steps.findIndex((step) => step.name === "Set up Node"); + expect(setupNodeIndex).toBeGreaterThan(0); + steps.splice(setupNodeIndex, 0, { + name: "Authenticate to Docker Hub", + env: { + DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}", + DOCKERHUB_TOKEN: "${{ secrets.DOCKERHUB_TOKEN }}", + }, + run: "docker login docker.io --username user --password ${{ secrets.DOCKERHUB_TOKEN }}", + }); + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); + expect(errors).toEqual( + expect.arrayContaining([ + "hermes-root-entrypoint-smoke-vitest must not authenticate to Docker Hub before branch-controlled test code runs", + "hermes-root-entrypoint-smoke-vitest step 'Authenticate to Docker Hub' env must not include DOCKERHUB_USERNAME", + "hermes-root-entrypoint-smoke-vitest step 'Authenticate to Docker Hub' env must not include DOCKERHUB_TOKEN", + "hermes-root-entrypoint-smoke-vitest step 'Authenticate to Docker Hub' run script must not use docker login or inline secret interpolation", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("rejects raw jobs selector echo from matrix generation", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 8e655c595b..8249918bd0 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -1186,20 +1186,29 @@ function validateHermesRootEntrypointSmokeVitestJob( stepEnv, "NVIDIA_API_KEY", ); - if (step.name !== "Authenticate to Docker Hub") { - requireEnvDoesNotExposeSecret( - errors, - `hermes-root-entrypoint-smoke-vitest step '${stepName}'`, - stepEnv, - "DOCKERHUB_USERNAME", - ); - requireEnvDoesNotExposeSecret( - errors, - `hermes-root-entrypoint-smoke-vitest step '${stepName}'`, - stepEnv, - "DOCKERHUB_TOKEN", - ); - } + requireEnvDoesNotExposeSecret( + errors, + `hermes-root-entrypoint-smoke-vitest step '${stepName}'`, + stepEnv, + "DOCKERHUB_USERNAME", + ); + requireEnvDoesNotExposeSecret( + errors, + `hermes-root-entrypoint-smoke-vitest step '${stepName}'`, + stepEnv, + "DOCKERHUB_TOKEN", + ); + requireNoDockerHubAuthInRun( + errors, + `hermes-root-entrypoint-smoke-vitest step '${stepName}'`, + stringValue(step.run), + ); + } + + if (namedStep(steps, "Authenticate to Docker Hub")) { + errors.push( + "hermes-root-entrypoint-smoke-vitest must not authenticate to Docker Hub before branch-controlled test code runs", + ); } const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); @@ -1209,20 +1218,6 @@ function validateHermesRootEntrypointSmokeVitestJob( errors.push("hermes-root-entrypoint-smoke-vitest checkout step must set persist-credentials=false"); } - const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); - const dockerHubEnv = asRecord(dockerHubAuth?.env); - if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { - errors.push( - "hermes-root-entrypoint-smoke-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", - ); - } - if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { - errors.push( - "hermes-root-entrypoint-smoke-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", - ); - } - requireRunContains(errors, dockerHubAuth, "docker login docker.io"); - requireRunContains(errors, dockerHubAuth, "continuing with anonymous pulls"); const setupNode = namedStep(steps, "Set up Node"); if (!setupNode) errors.push("hermes-root-entrypoint-smoke-vitest job missing step: Set up Node"); From 90fb61d3f442eec238d996cfd203a9b4ba3c87d8 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 02:51:09 -0400 Subject: [PATCH 6/9] fix(e2e): route docker probe through env boundary --- test/e2e-scenario/fixtures/docker-probe.ts | 131 ++++++++++++++++++ .../live/hermes-root-entrypoint-smoke.test.ts | 95 +------------ .../support-tests/docker-probe.test.ts | 60 ++++++++ 3 files changed, 198 insertions(+), 88 deletions(-) create mode 100644 test/e2e-scenario/fixtures/docker-probe.ts create mode 100644 test/e2e-scenario/support-tests/docker-probe.test.ts diff --git a/test/e2e-scenario/fixtures/docker-probe.ts b/test/e2e-scenario/fixtures/docker-probe.ts new file mode 100644 index 0000000000..2fcf1eb258 --- /dev/null +++ b/test/e2e-scenario/fixtures/docker-probe.ts @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import type { ArtifactSink } from "./artifacts.ts"; +import { buildChildEnv } from "./redaction.ts"; +import type { SecretStore } from "./secrets.ts"; + +export type DockerCommandResult = { + command: string[]; + exitCode: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; + error?: string; +}; + +const DOCKER_ENV_ALLOWLIST = [ + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_TLS_VERIFY", + "DOCKER_CERT_PATH", + "XDG_RUNTIME_DIR", +] as const; + +function safeName(value: string): string { + return ( + value + .trim() + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, "") || "docker" + ); +} + +export function buildDockerProbeEnv( + base: NodeJS.ProcessEnv, + dockerConfigDir: string, +): NodeJS.ProcessEnv { + return buildChildEnv(base, { + additionalAllowedEnv: DOCKER_ENV_ALLOWLIST, + fixtureOverlay: { + DOCKER_CONFIG: dockerConfigDir, + }, + }); +} + +export function redactDockerProbeResult( + result: DockerCommandResult, + redact: SecretStore["redact"], +): DockerCommandResult { + return { + command: result.command.map((part) => redact(part)), + exitCode: result.exitCode, + signal: result.signal, + stdout: redact(result.stdout), + stderr: redact(result.stderr), + error: result.error ? redact(result.error) : undefined, + }; +} + +export function resultText(result: DockerCommandResult): string { + return [ + `$ ${result.command.join(" ")}`, + result.stdout.trim(), + result.stderr.trim(), + result.error ? `error: ${result.error}` : "", + ] + .filter(Boolean) + .join("\n"); +} + +export class DockerProbe { + private sequence = 0; + private readonly dockerConfigDir: string; + + constructor( + private readonly artifacts: ArtifactSink, + private readonly redact: SecretStore["redact"], + ) { + this.dockerConfigDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-config-")); + } + + async run( + args: string[], + options: { artifactName: string; timeoutMs?: number } = { artifactName: "docker" }, + ): Promise { + fs.mkdirSync(this.dockerConfigDir, { recursive: true }); + const command = ["docker", ...args]; + const result = spawnSync("docker", args, { + cwd: path.resolve(import.meta.dirname, "../../.."), + encoding: "utf8", + env: buildDockerProbeEnv(process.env, this.dockerConfigDir), + maxBuffer: 10 * 1024 * 1024, + timeout: options.timeoutMs ?? 30_000, + }); + const commandResult = redactDockerProbeResult( + { + command, + exitCode: result.status, + signal: result.signal, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + error: result.error instanceof Error ? result.error.message : undefined, + }, + this.redact, + ); + const artifactBase = `docker/${String(++this.sequence).padStart(3, "0")}-${safeName( + options.artifactName, + )}`; + await this.artifacts.writeText(`${artifactBase}.stdout.txt`, commandResult.stdout); + await this.artifacts.writeText(`${artifactBase}.stderr.txt`, commandResult.stderr); + await this.artifacts.writeJson(`${artifactBase}.result.json`, commandResult); + return commandResult; + } + + async expect( + args: string[], + options: { artifactName: string; timeoutMs?: number }, + ): Promise { + const result = await this.run(args, options); + if (result.exitCode !== 0) { + throw new Error(resultText(result)); + } + return result; + } +} diff --git a/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts b/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts index 273507ff3f..40904ea48a 100644 --- a/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts +++ b/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts @@ -1,13 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; -import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; -import type { ArtifactSink } from "../fixtures/artifacts.ts"; +import { DockerProbe, resultText, type DockerCommandResult } from "../fixtures/docker-probe.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; -import type { SecretStore } from "../fixtures/secrets.ts"; // Migrated from test/e2e/test-hermes-root-entrypoint-smoke.sh. This remains a // real Docker/root-entrypoint smoke: it builds the Hermes image when no prebuilt @@ -15,95 +12,17 @@ import type { SecretStore } from "../fixtures/secrets.ts"; // as root, and verifies health, gateway privilege separation, runtime layout, // sticky config protection, and legacy gateway.pid symlink migration. -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const HEALTH_ATTEMPTS = 90; const HEALTH_POLL_MS = 2_000; -const DEFAULT_COMMAND_TIMEOUT_MS = 30_000; const BUILD_TIMEOUT_MS = 10 * 60_000; const RUN_TIMEOUT_MS = 60_000; const liveTest = process.env.NEMOCLAW_RUN_E2E_SCENARIOS === "1" ? test : test.skip; -type CommandResult = { - command: string[]; - exitCode: number | null; - signal: NodeJS.Signals | null; - stdout: string; - stderr: string; - error?: string; -}; - -function safeName(value: string): string { - return ( - value - .trim() - .toLowerCase() - .replace(/[^a-z0-9._-]+/g, "-") - .replace(/^-+|-+$/g, "") || "docker" - ); -} - function safeTag(value: string): string { return value.replace(/[^A-Za-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "") || "local"; } -function resultText(result: CommandResult): string { - return [ - `$ ${result.command.join(" ")}`, - result.stdout.trim(), - result.stderr.trim(), - result.error ? `error: ${result.error}` : "", - ] - .filter(Boolean) - .join("\n"); -} - -class DockerProbe { - private sequence = 0; - - constructor( - private readonly artifacts: ArtifactSink, - private readonly redact: SecretStore["redact"], - ) {} - - async run( - args: string[], - options: { artifactName: string; timeoutMs?: number } = { artifactName: "docker" }, - ): Promise { - const command = ["docker", ...args]; - const result = spawnSync("docker", args, { - cwd: REPO_ROOT, - encoding: "utf8", - env: process.env, - maxBuffer: 10 * 1024 * 1024, - timeout: options.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS, - }); - const commandResult: CommandResult = { - command: command.map((part) => this.redact(part)), - exitCode: result.status, - signal: result.signal, - stdout: this.redact(result.stdout ?? ""), - stderr: this.redact(result.stderr ?? ""), - error: result.error instanceof Error ? this.redact(result.error.message) : undefined, - }; - const artifactBase = `docker/${String(++this.sequence).padStart(3, "0")}-${safeName( - options.artifactName, - )}`; - await this.artifacts.writeText(`${artifactBase}.stdout.txt`, commandResult.stdout); - await this.artifacts.writeText(`${artifactBase}.stderr.txt`, commandResult.stderr); - await this.artifacts.writeJson(`${artifactBase}.result.json`, commandResult); - return commandResult; - } - - async expect( - args: string[], - options: { artifactName: string; timeoutMs?: number }, - ): Promise { - const result = await this.run(args, options); - expect(result.exitCode, resultText(result)).toBe(0); - return result; - } -} async function requireDocker(probe: DockerProbe, skip: (message: string) => void): Promise { const result = await probe.run(["info"], { artifactName: "docker-info", timeoutMs: 30_000 }); @@ -132,10 +51,10 @@ async function buildImageIfNeeded( [ "build", "-f", - path.join(REPO_ROOT, "agents/hermes/Dockerfile.base"), + "agents/hermes/Dockerfile.base", "-t", baseImage, - REPO_ROOT, + ".", ], { artifactName: "build-hermes-base-image", timeoutMs: BUILD_TIMEOUT_MS }, ); @@ -143,12 +62,12 @@ async function buildImageIfNeeded( [ "build", "-f", - path.join(REPO_ROOT, "agents/hermes/Dockerfile"), + "agents/hermes/Dockerfile", "--build-arg", `BASE_IMAGE=${baseImage}`, "-t", image, - REPO_ROOT, + ".", ], { artifactName: "build-hermes-production-image", timeoutMs: BUILD_TIMEOUT_MS }, ); @@ -159,7 +78,7 @@ async function dockerExecSh( container: string, script: string, artifactName: string, -): Promise { +): Promise { return probe.run(["exec", container, "sh", "-lc", script], { artifactName }); } @@ -168,7 +87,7 @@ async function expectContainerSh( container: string, message: string, script: string, -): Promise { +): Promise { const result = await dockerExecSh(probe, container, script, message); expect(result.exitCode, `${container}: ${message}\n${resultText(result)}`).toBe(0); return result; diff --git a/test/e2e-scenario/support-tests/docker-probe.test.ts b/test/e2e-scenario/support-tests/docker-probe.test.ts new file mode 100644 index 0000000000..d664e29880 --- /dev/null +++ b/test/e2e-scenario/support-tests/docker-probe.test.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { buildDockerProbeEnv, redactDockerProbeResult } from "../fixtures/docker-probe.ts"; +import { SecretStore } from "../fixtures/secrets.ts"; + +describe("DockerProbe secret hygiene", () => { + it("builds Docker command env through the fixture-owned allowlist boundary", () => { + const env = buildDockerProbeEnv( + { + PATH: "/usr/bin", + HOME: "/tmp/home", + DOCKER_HOST: "unix:///tmp/docker.sock", + DOCKER_CONTEXT: "desktop-linux", + DOCKERHUB_TOKEN: "dockerhub-secret-token", + NVIDIA_API_KEY: "nvapi-secret-value", + RANDOM_SECRET: "other-secret-value", + }, + "/tmp/docker-config", + ); + + expect(env).toMatchObject({ + PATH: expect.stringContaining("/usr/bin"), + HOME: "/tmp/home", + DOCKER_HOST: "unix:///tmp/docker.sock", + DOCKER_CONTEXT: "desktop-linux", + DOCKER_CONFIG: "/tmp/docker-config", + }); + expect(env).not.toHaveProperty("DOCKERHUB_TOKEN"); + expect(env).not.toHaveProperty("NVIDIA_API_KEY"); + expect(env).not.toHaveProperty("RANDOM_SECRET"); + }); + + it("redacts secret-shaped Docker diagnostics before artifacts are written", () => { + const secret = "nvapi-supersecret-token"; + const secrets = new SecretStore({ NVIDIA_API_KEY: secret }, (message) => { + throw new Error(message ?? "unexpected skip"); + }); + + const result = redactDockerProbeResult( + { + command: ["docker", "run", "--env", `NVIDIA_API_KEY=${secret}`], + exitCode: 1, + signal: null, + stdout: `stdout ${secret}`, + stderr: `stderr TOKEN=${secret}`, + error: `error ${secret}`, + }, + (text, extraValues) => secrets.redact(text, extraValues), + ); + + expect(JSON.stringify(result)).not.toContain(secret); + expect(result.command.join(" ")).toContain("[REDACTED]"); + expect(result.stdout).toContain("[REDACTED]"); + expect(result.stderr).toContain("[REDACTED]"); + expect(result.error).toContain("[REDACTED]"); + }); +}); From d47a4f55e099a6803c48e16c221e106d3290913c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 02:56:53 -0400 Subject: [PATCH 7/9] style(e2e): format hermes root smoke --- .../live/hermes-root-entrypoint-smoke.test.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts b/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts index 40904ea48a..2fa4bdd46b 100644 --- a/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts +++ b/test/e2e-scenario/live/hermes-root-entrypoint-smoke.test.ts @@ -23,7 +23,6 @@ function safeTag(value: string): string { return value.replace(/[^A-Za-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "") || "local"; } - async function requireDocker(probe: DockerProbe, skip: (message: string) => void): Promise { const result = await probe.run(["info"], { artifactName: "docker-info", timeoutMs: 30_000 }); if (result.exitCode === 0) return; @@ -47,17 +46,10 @@ async function buildImageIfNeeded( return; } - await probe.expect( - [ - "build", - "-f", - "agents/hermes/Dockerfile.base", - "-t", - baseImage, - ".", - ], - { artifactName: "build-hermes-base-image", timeoutMs: BUILD_TIMEOUT_MS }, - ); + await probe.expect(["build", "-f", "agents/hermes/Dockerfile.base", "-t", baseImage, "."], { + artifactName: "build-hermes-base-image", + timeoutMs: BUILD_TIMEOUT_MS, + }); await probe.expect( [ "build", From 354a1feb99b08420508c22be8a2366eb9c18a80f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 03:09:21 -0400 Subject: [PATCH 8/9] test(e2e): cover docker diagnostic redaction --- test/e2e-scenario/fixtures/docker-probe.ts | 15 ++- .../support-tests/docker-probe.test.ts | 96 ++++++++++++++++++- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario/fixtures/docker-probe.ts b/test/e2e-scenario/fixtures/docker-probe.ts index 2fcf1eb258..42d6c64f32 100644 --- a/test/e2e-scenario/fixtures/docker-probe.ts +++ b/test/e2e-scenario/fixtures/docker-probe.ts @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; +import { + spawnSync, + type SpawnSyncOptionsWithStringEncoding, + type SpawnSyncReturns, +} from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -19,6 +23,12 @@ export type DockerCommandResult = { error?: string; }; +export type DockerProbeRunner = ( + command: string, + args: string[], + options: SpawnSyncOptionsWithStringEncoding, +) => SpawnSyncReturns; + const DOCKER_ENV_ALLOWLIST = [ "DOCKER_HOST", "DOCKER_CONTEXT", @@ -81,6 +91,7 @@ export class DockerProbe { constructor( private readonly artifacts: ArtifactSink, private readonly redact: SecretStore["redact"], + private readonly runDocker: DockerProbeRunner = spawnSync, ) { this.dockerConfigDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-config-")); } @@ -91,7 +102,7 @@ export class DockerProbe { ): Promise { fs.mkdirSync(this.dockerConfigDir, { recursive: true }); const command = ["docker", ...args]; - const result = spawnSync("docker", args, { + const result = this.runDocker("docker", args, { cwd: path.resolve(import.meta.dirname, "../../.."), encoding: "utf8", env: buildDockerProbeEnv(process.env, this.dockerConfigDir), diff --git a/test/e2e-scenario/support-tests/docker-probe.test.ts b/test/e2e-scenario/support-tests/docker-probe.test.ts index d664e29880..309893138e 100644 --- a/test/e2e-scenario/support-tests/docker-probe.test.ts +++ b/test/e2e-scenario/support-tests/docker-probe.test.ts @@ -1,11 +1,24 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + import { describe, expect, it } from "vitest"; -import { buildDockerProbeEnv, redactDockerProbeResult } from "../fixtures/docker-probe.ts"; +import { ArtifactSink } from "../fixtures/artifacts.ts"; +import { + buildDockerProbeEnv, + DockerProbe, + redactDockerProbeResult, +} from "../fixtures/docker-probe.ts"; import { SecretStore } from "../fixtures/secrets.ts"; +async function readArtifact(root: string, relativePath: string): Promise { + return fs.readFile(path.join(root, relativePath), "utf8"); +} + describe("DockerProbe secret hygiene", () => { it("builds Docker command env through the fixture-owned allowlist boundary", () => { const env = buildDockerProbeEnv( @@ -57,4 +70,85 @@ describe("DockerProbe secret hygiene", () => { expect(result.stderr).toContain("[REDACTED]"); expect(result.error).toContain("[REDACTED]"); }); + + it("writes DockerProbe stdout, stderr, and result artifacts after redaction", async () => { + const secret = "docker-probe-artifact-secret"; + const artifactsRoot = await fs.mkdtemp(path.join(os.tmpdir(), "docker-probe-artifacts-")); + const artifacts = new ArtifactSink(artifactsRoot); + const secrets = new SecretStore({ NEMOCLAW_TOKEN: secret }, (message) => { + throw new Error(message ?? "unexpected skip"); + }); + const probe = new DockerProbe( + artifacts, + (text, extraValues) => secrets.redact(text, extraValues), + (_command, args) => ({ + pid: 123, + output: [null, `stdout ${secret} ${args.join(" ")}`, `stderr ${secret}`], + stdout: `stdout ${secret} ${args.join(" ")}`, + stderr: `stderr ${secret}`, + status: 17, + signal: null, + error: new Error(`error ${secret}`), + }), + ); + + const result = await probe.run(["logs", "hermes"], { artifactName: "diag-hermes-logs" }); + + expect(JSON.stringify(result)).not.toContain(secret); + for (const relativePath of [ + "docker/001-diag-hermes-logs.stdout.txt", + "docker/001-diag-hermes-logs.stderr.txt", + "docker/001-diag-hermes-logs.result.json", + ]) { + const artifact = await readArtifact(artifactsRoot, relativePath); + expect(artifact).not.toContain(secret); + expect(artifact).toContain("[REDACTED]"); + } + }); + + it("redacts diagnostic-style Docker inspect, logs, process, start-log, and gateway-log artifacts", async () => { + const secret = "docker-diagnostic-artifact-secret"; + const diagnostics = new Map([ + ["diag-hermes-inspect", `inspect env TOKEN=${secret}`], + ["diag-hermes-logs", `container log Bearer ${secret}`], + ["diag-hermes-process", `process --token=${secret}`], + ["diag-hermes-start-log", `nemoclaw start log ${secret}`], + ["diag-hermes-gateway-log", `gateway log ${secret}`], + ]); + const artifactsRoot = await fs.mkdtemp(path.join(os.tmpdir(), "docker-probe-diagnostics-")); + const artifacts = new ArtifactSink(artifactsRoot); + const secrets = new SecretStore({ NEMOCLAW_TOKEN: secret }, (message) => { + throw new Error(message ?? "unexpected skip"); + }); + const probe = new DockerProbe( + artifacts, + (text, extraValues) => secrets.redact(text, extraValues), + (_command, args) => { + const artifactName = args.at(-1) ?? "unknown"; + const stdout = diagnostics.get(artifactName) ?? `diagnostic ${secret}`; + return { + pid: 123, + output: [null, stdout, `stderr ${secret}`], + stdout, + stderr: `stderr ${secret}`, + status: 0, + signal: null, + }; + }, + ); + + for (const artifactName of diagnostics.keys()) { + await probe.run(["fake-diagnostic", artifactName], { artifactName }); + } + + let sequence = 0; + for (const artifactName of diagnostics.keys()) { + const artifactBase = `docker/${String(++sequence).padStart(3, "0")}-${artifactName}`; + for (const suffix of ["stdout.txt", "stderr.txt", "result.json"]) { + const artifact = await readArtifact(artifactsRoot, `${artifactBase}.${suffix}`); + expect(artifact, `${artifactBase}.${suffix}`).not.toContain(secret); + expect(artifact, `${artifactBase}.${suffix}`).toContain("[REDACTED]"); + } + } + }); }); From 366edde69b7b0d5a087779faa15a3eb59a086296 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 03:24:00 -0400 Subject: [PATCH 9/9] ci(e2e): avoid sandbox checkout credentials --- .github/workflows/sandbox-images-and-e2e.yaml | 12 ++++++++++ test/e2e-script-workflow.test.ts | 22 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 373ec3f829..1d7ca7e38f 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -22,6 +22,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Resolve sandbox base image uses: ./.github/actions/resolve-sandbox-base-image @@ -63,6 +65,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Resolve Hermes base image uses: ./.github/actions/resolve-hermes-base-image @@ -134,6 +138,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Resolve sandbox base image uses: ./.github/actions/resolve-sandbox-base-image @@ -151,6 +157,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Download image artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -171,6 +179,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Download image artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -191,6 +201,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Download image artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/test/e2e-script-workflow.test.ts b/test/e2e-script-workflow.test.ts index eb24a4eb9a..db0571e95e 100644 --- a/test/e2e-script-workflow.test.ts +++ b/test/e2e-script-workflow.test.ts @@ -5,7 +5,12 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; -import { loadE2eWorkflowContract, reusableNightlyJobs } from "./helpers/e2e-workflow-contract"; +import { + loadE2eWorkflowContract, + readYaml, + reusableNightlyJobs, + type WorkflowJob, +} from "./helpers/e2e-workflow-contract"; // Direct legacy bash E2Es are being migrated toward Vitest coverage. Keep the // top-level shell suite frozen so new coverage starts in the newer E2E surface @@ -192,6 +197,21 @@ describe("E2E reusable workflow contract", () => { } }); + it("does not persist checkout credentials in sandbox image E2E jobs", () => { + const sandboxWorkflow = readYaml<{ jobs: Record }>( + ".github/workflows/sandbox-images-and-e2e.yaml", + ); + + for (const [jobName, job] of Object.entries(sandboxWorkflow.jobs)) { + const checkoutStep = job.steps?.find((step) => + String(step.uses ?? "").startsWith("actions/checkout@"), + ); + if (!checkoutStep) continue; + + expect(checkoutStep.with?.["persist-credentials"], jobName).toBe(false); + } + }); + it("runs only validated test/e2e shell scripts through the composite action", () => { const runStep = action.runs.steps.find((step) => step.name === "Run E2E script");