From c32697fdf8959bf4de9443031aa421288c06fb5c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 20:25:07 -0400 Subject: [PATCH 1/2] test(e2e): migrate dashboard remote bind --- .../live/dashboard-remote-bind.test.ts | 117 ++++++++++++++++++ test/e2e-script-workflow.test.ts | 3 +- test/e2e/brev-e2e.test.ts | 34 +++-- test/e2e/test-dashboard-remote-bind.sh | 72 ----------- 4 files changed, 143 insertions(+), 83 deletions(-) create mode 100644 test/e2e-scenario/live/dashboard-remote-bind.test.ts delete mode 100755 test/e2e/test-dashboard-remote-bind.sh diff --git a/test/e2e-scenario/live/dashboard-remote-bind.test.ts b/test/e2e-scenario/live/dashboard-remote-bind.test.ts new file mode 100644 index 0000000000..1436b682ee --- /dev/null +++ b/test/e2e-scenario/live/dashboard-remote-bind.test.ts @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import os from "node:os"; + +import { expect, test } from "../fixtures/e2e-test.ts"; + +// Migrated from test/e2e/test-dashboard-remote-bind.sh. +// Branch validation provisions and onboards a real remote sandbox first; this +// test restarts only that sandbox's dashboard forward and proves the explicit +// remote-bind opt-in is honored without adding another harness. + +const runDashboardRemoteBindTest = + process.env.NEMOCLAW_E2E_DASHBOARD_REMOTE_BIND === "1" ? test : test.skip; + +function matchingForwardLine(output: string, sandboxName: string, dashboardPort: string): string { + return ( + output + .split("\n") + .map((line) => line.trim()) + .find((line) => line.includes(sandboxName) && line.includes(dashboardPort)) ?? "" + ); +} + +function bindsAllInterfaces(line: string, dashboardPort: string): boolean { + return ( + line.includes(`0.0.0.0:${dashboardPort}`) || + line.includes(`*:${dashboardPort}`) || + new RegExp(`\\b0\\.0\\.0\\.0\\s+${dashboardPort}\\b`).test(line) + ); +} + +function bindsLoopback(line: string, dashboardPort: string): boolean { + return ( + line.includes(`127.0.0.1:${dashboardPort}`) || + line.includes(`localhost:${dashboardPort}`) || + new RegExp(`\\b127\\.0\\.0\\.1\\s+${dashboardPort}\\b`).test(line) + ); +} + +function remoteHostCandidate(): string { + const externalIpv4 = Object.values(os.networkInterfaces()) + .flat() + .find((iface) => iface && iface.family === "IPv4" && !iface.internal)?.address; + return process.env.NEMOCLAW_E2E_REMOTE_HOST || externalIpv4 || os.hostname(); +} + +runDashboardRemoteBindTest( + "dashboard forward binds all interfaces when remote bind is explicitly requested", + async ({ artifacts, host, sandbox }) => { + const sandboxName = process.env.NEMOCLAW_SANDBOX_NAME || "e2e-test"; + const dashboardPort = process.env.NEMOCLAW_DASHBOARD_PORT || "18789"; + const remoteHost = remoteHostCandidate(); + + await artifacts.writeJson("scenario.json", { + id: "dashboard-remote-bind", + runner: "vitest", + migratedFrom: "test/e2e/test-dashboard-remote-bind.sh", + boundary: "remote-dashboard-forward", + optIn: "NEMOCLAW_E2E_DASHBOARD_REMOTE_BIND=1", + sandboxName, + dashboardPort, + remoteHost, + }); + + const cliProbe = await host.command( + "bash", + ["-lc", "command -v nemoclaw && command -v openshell"], + { + artifactName: "dashboard-remote-bind-cli-probe", + inheritEnv: true, + timeoutMs: 30_000, + }, + ); + expect(cliProbe.exitCode, `required CLI probe failed\n${cliProbe.stderr}`).toBe(0); + expect(cliProbe.stdout).toContain("nemoclaw"); + expect(cliProbe.stdout).toContain("openshell"); + + await sandbox.openshell(["forward", "stop", dashboardPort], { + artifactName: "dashboard-remote-bind-forward-stop", + inheritEnv: true, + timeoutMs: 30_000, + }); + + const connect = await host.nemoclaw([sandboxName, "connect"], { + artifactName: "dashboard-remote-bind-connect", + inheritEnv: true, + env: { + NEMOCLAW_DASHBOARD_BIND: "0.0.0.0", + }, + timeoutMs: 120_000, + }); + expect(connect.exitCode, `nemoclaw connect failed\n${connect.stderr}`).toBe(0); + + const forwardList = await sandbox.openshell(["forward", "list"], { + artifactName: "dashboard-remote-bind-forward-list", + inheritEnv: true, + timeoutMs: 30_000, + }); + expect(forwardList.exitCode, `openshell forward list failed\n${forwardList.stderr}`).toBe(0); + await artifacts.writeText("forward-list.txt", forwardList.stdout); + + const forwardLine = matchingForwardLine(forwardList.stdout, sandboxName, dashboardPort); + expect( + forwardLine, + `No OpenShell forward found for ${sandboxName} on ${dashboardPort}`, + ).not.toBe(""); + expect( + bindsLoopback(forwardLine, dashboardPort), + `Dashboard forward is still localhost-only; expected an all-interface bind: ${forwardLine}`, + ).toBe(false); + expect( + bindsAllInterfaces(forwardLine, dashboardPort), + `Could not prove dashboard forward uses 0.0.0.0:${dashboardPort}: ${forwardLine}`, + ).toBe(true); + }, +); diff --git a/test/e2e-script-workflow.test.ts b/test/e2e-script-workflow.test.ts index da3de1c226..873c60d2be 100644 --- a/test/e2e-script-workflow.test.ts +++ b/test/e2e-script-workflow.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; @@ -23,7 +23,6 @@ const LEGACY_E2E_SHELL_ALLOWLIST = [ "test/e2e/test-credential-migration.sh", "test/e2e/test-credential-sanitization.sh", "test/e2e/test-cron-preflight-inference-local-e2e.sh", - "test/e2e/test-dashboard-remote-bind.sh", "test/e2e/test-device-auth-health.sh", "test/e2e/test-diagnostics.sh", "test/e2e/test-docs-validation.sh", diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index df66f6eceb..938dcd5ec4 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -50,9 +50,9 @@ * TELEGRAM_CHAT_ID_E2E — Telegram chat ID for optional sendMessage test */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { execSync, execFileSync, spawnSync, type StdioOptions } from "node:child_process"; +import { execFileSync, execSync, type StdioOptions, spawnSync } from "node:child_process"; import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; // Instance configuration const BREV_MIN_VCPU = parseInt(process.env.BREV_MIN_VCPU || "4", 10); @@ -396,7 +396,10 @@ function waitForLaunchableReady(maxWaitMs = 1_200_000, pollIntervalMs = 15_000): ); } -function runRemoteTest(scriptPath: string): string { +function runRemoteCommand( + command: string, + timeoutMs = GPU_TEST_SUITE ? 1_800_000 : 900_000, +): string { const cmd = [ `set -o pipefail`, `source ~/.nvm/nvm.sh 2>/dev/null || true`, @@ -404,13 +407,12 @@ function runRemoteTest(scriptPath: string): string { `export npm_config_prefix=$HOME/.local`, `export PATH=$HOME/.local/bin:$PATH`, // Docker socket is chmod 666 by setup script, no sg docker needed. - - `bash ${scriptPath} 2>&1 | tee /tmp/test-output.log`, + `${command} 2>&1 | tee /tmp/test-output.log`, ].join(" && "); // Stream test output to CI log AND capture it for assertions try { - sshEnv(cmd, { timeout: GPU_TEST_SUITE ? 1_800_000 : 900_000, stream: true }); + sshEnv(cmd, { timeout: timeoutMs, stream: true }); } catch (error) { printRemoteFailureDiagnostics(); throw error; @@ -419,6 +421,10 @@ function runRemoteTest(scriptPath: string): string { return ssh("cat /tmp/test-output.log", { timeout: 30_000 }); } +function runRemoteTest(scriptPath: string): string { + return runRemoteCommand(`bash ${scriptPath}`); +} + function printRemoteFailureDiagnostics(): void { try { const diagnostics = ssh( @@ -1235,9 +1241,19 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { it.runIf(TEST_SUITE === "dashboard-remote-bind")( "dashboard forward binds to all interfaces for remote browser origins", () => { - const output = runRemoteTest("test/e2e/test-dashboard-remote-bind.sh"); - expect(output).toContain("PASS"); - expect(output).not.toMatch(/FAIL:/); + const output = runRemoteCommand( + [ + `NEMOCLAW_RUN_E2E_SCENARIOS=1`, + `NEMOCLAW_E2E_DASHBOARD_REMOTE_BIND=1`, + `NEMOCLAW_SANDBOX_NAME=e2e-test`, + `npx vitest run --project e2e-scenarios-live`, + `test/e2e-scenario/live/dashboard-remote-bind.test.ts`, + `--silent=false --reporter=default`, + ].join(" "), + 300_000, + ); + expect(output).toContain("dashboard forward binds all interfaces"); + expect(output).not.toMatch(/FAIL|Failed/i); }, 300_000, ); diff --git a/test/e2e/test-dashboard-remote-bind.sh b/test/e2e/test-dashboard-remote-bind.sh deleted file mode 100755 index 9fa259f8c8..0000000000 --- a/test/e2e/test-dashboard-remote-bind.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -uo pipefail - -section() { printf '\n=== %s ===\n' "$1"; } -pass() { echo "PASS: $1"; } -fail() { - echo "FAIL: $1" - exit 1 -} -info() { echo "INFO: $1"; } - -SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-test}" -DASHBOARD_PORT="${NEMOCLAW_DASHBOARD_PORT:-18789}" -REMOTE_HOST="${NEMOCLAW_E2E_REMOTE_HOST:-$(hostname -I 2>/dev/null | awk '{print $1}')}" -if [ -z "$REMOTE_HOST" ]; then - REMOTE_HOST="$(hostname -f 2>/dev/null || hostname)" -fi - -section "Preconditions" -info "Sandbox: ${SANDBOX_NAME}" -info "Dashboard port: ${DASHBOARD_PORT}" -info "Remote host candidate: ${REMOTE_HOST}" - -if ! command -v nemoclaw >/dev/null 2>&1; then - fail "nemoclaw CLI is not on PATH" -fi -if ! command -v openshell >/dev/null 2>&1; then - fail "openshell CLI is not on PATH" -fi -pass "Required CLIs are available" - -section "Restart dashboard forward with explicit all-interface bind" -# The coverage guard mirrors issue #3259: remote SSH-deployed hosts need an -# explicit operator-controlled way to bind the dashboard forward on all -# interfaces. On main, NEMOCLAW_DASHBOARD_BIND is ignored and the forward stays -# localhost-only; the fix should make this opt-in produce 0.0.0.0:. -openshell forward stop "${DASHBOARD_PORT}" >/dev/null 2>&1 || true -CONNECT_LOG="$(mktemp -t nemoclaw-dashboard-remote-bind.XXXXXX.log)" -trap 'rm -f "${CONNECT_LOG}"' EXIT -if NEMOCLAW_DASHBOARD_BIND=0.0.0.0 nemoclaw "${SANDBOX_NAME}" connect >"${CONNECT_LOG}" 2>&1; then - pass "nemoclaw connect completed with NEMOCLAW_DASHBOARD_BIND=0.0.0.0" -else - cat "${CONNECT_LOG}" - fail "nemoclaw connect failed with NEMOCLAW_DASHBOARD_BIND=0.0.0.0" -fi - -section "Verify OpenShell forward bind" -FORWARD_LIST="$(openshell forward list 2>/dev/null || true)" -printf '%s\n' "${FORWARD_LIST}" -FORWARD_LINE="$(printf '%s\n' "${FORWARD_LIST}" | awk -v sandbox="${SANDBOX_NAME}" -v port="${DASHBOARD_PORT}" '$0 ~ sandbox && $0 ~ port {print; exit}')" -if [ -z "${FORWARD_LINE}" ]; then - fail "No OpenShell forward found for ${SANDBOX_NAME} on ${DASHBOARD_PORT}" -fi -info "Matched forward: ${FORWARD_LINE}" - -case "${FORWARD_LINE}" in - *"0.0.0.0:${DASHBOARD_PORT}"* | *"*:""${DASHBOARD_PORT}"* | *"0.0.0.0 "*" ${DASHBOARD_PORT} "*) - pass "Dashboard forward binds all interfaces for remote origin (${DASHBOARD_PORT})" - ;; - *"127.0.0.1:${DASHBOARD_PORT}"* | *"localhost:${DASHBOARD_PORT}"* | *"127.0.0.1 "*" ${DASHBOARD_PORT} "*) - fail "Dashboard forward is still localhost-only; expected 0.0.0.0:${DASHBOARD_PORT}" - ;; - *) - fail "Could not prove dashboard forward uses 0.0.0.0:${DASHBOARD_PORT} from: ${FORWARD_LINE}" - ;; -esac - -section "Summary" -pass "Remote dashboard bind guard completed" From 2f5f201f4757c66f0e54bcf4195b1586f059819c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 10 Jun 2026 21:01:19 -0400 Subject: [PATCH 2/2] test(e2e): keep dashboard legacy script --- test/e2e-script-workflow.test.ts | 3 +- test/e2e/test-dashboard-remote-bind.sh | 72 ++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100755 test/e2e/test-dashboard-remote-bind.sh diff --git a/test/e2e-script-workflow.test.ts b/test/e2e-script-workflow.test.ts index 873c60d2be..da3de1c226 100644 --- a/test/e2e-script-workflow.test.ts +++ b/test/e2e-script-workflow.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; import { describe, expect, it } from "vitest"; @@ -23,6 +23,7 @@ const LEGACY_E2E_SHELL_ALLOWLIST = [ "test/e2e/test-credential-migration.sh", "test/e2e/test-credential-sanitization.sh", "test/e2e/test-cron-preflight-inference-local-e2e.sh", + "test/e2e/test-dashboard-remote-bind.sh", "test/e2e/test-device-auth-health.sh", "test/e2e/test-diagnostics.sh", "test/e2e/test-docs-validation.sh", diff --git a/test/e2e/test-dashboard-remote-bind.sh b/test/e2e/test-dashboard-remote-bind.sh new file mode 100755 index 0000000000..9fa259f8c8 --- /dev/null +++ b/test/e2e/test-dashboard-remote-bind.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +section() { printf '\n=== %s ===\n' "$1"; } +pass() { echo "PASS: $1"; } +fail() { + echo "FAIL: $1" + exit 1 +} +info() { echo "INFO: $1"; } + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-test}" +DASHBOARD_PORT="${NEMOCLAW_DASHBOARD_PORT:-18789}" +REMOTE_HOST="${NEMOCLAW_E2E_REMOTE_HOST:-$(hostname -I 2>/dev/null | awk '{print $1}')}" +if [ -z "$REMOTE_HOST" ]; then + REMOTE_HOST="$(hostname -f 2>/dev/null || hostname)" +fi + +section "Preconditions" +info "Sandbox: ${SANDBOX_NAME}" +info "Dashboard port: ${DASHBOARD_PORT}" +info "Remote host candidate: ${REMOTE_HOST}" + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw CLI is not on PATH" +fi +if ! command -v openshell >/dev/null 2>&1; then + fail "openshell CLI is not on PATH" +fi +pass "Required CLIs are available" + +section "Restart dashboard forward with explicit all-interface bind" +# The coverage guard mirrors issue #3259: remote SSH-deployed hosts need an +# explicit operator-controlled way to bind the dashboard forward on all +# interfaces. On main, NEMOCLAW_DASHBOARD_BIND is ignored and the forward stays +# localhost-only; the fix should make this opt-in produce 0.0.0.0:. +openshell forward stop "${DASHBOARD_PORT}" >/dev/null 2>&1 || true +CONNECT_LOG="$(mktemp -t nemoclaw-dashboard-remote-bind.XXXXXX.log)" +trap 'rm -f "${CONNECT_LOG}"' EXIT +if NEMOCLAW_DASHBOARD_BIND=0.0.0.0 nemoclaw "${SANDBOX_NAME}" connect >"${CONNECT_LOG}" 2>&1; then + pass "nemoclaw connect completed with NEMOCLAW_DASHBOARD_BIND=0.0.0.0" +else + cat "${CONNECT_LOG}" + fail "nemoclaw connect failed with NEMOCLAW_DASHBOARD_BIND=0.0.0.0" +fi + +section "Verify OpenShell forward bind" +FORWARD_LIST="$(openshell forward list 2>/dev/null || true)" +printf '%s\n' "${FORWARD_LIST}" +FORWARD_LINE="$(printf '%s\n' "${FORWARD_LIST}" | awk -v sandbox="${SANDBOX_NAME}" -v port="${DASHBOARD_PORT}" '$0 ~ sandbox && $0 ~ port {print; exit}')" +if [ -z "${FORWARD_LINE}" ]; then + fail "No OpenShell forward found for ${SANDBOX_NAME} on ${DASHBOARD_PORT}" +fi +info "Matched forward: ${FORWARD_LINE}" + +case "${FORWARD_LINE}" in + *"0.0.0.0:${DASHBOARD_PORT}"* | *"*:""${DASHBOARD_PORT}"* | *"0.0.0.0 "*" ${DASHBOARD_PORT} "*) + pass "Dashboard forward binds all interfaces for remote origin (${DASHBOARD_PORT})" + ;; + *"127.0.0.1:${DASHBOARD_PORT}"* | *"localhost:${DASHBOARD_PORT}"* | *"127.0.0.1 "*" ${DASHBOARD_PORT} "*) + fail "Dashboard forward is still localhost-only; expected 0.0.0.0:${DASHBOARD_PORT}" + ;; + *) + fail "Could not prove dashboard forward uses 0.0.0.0:${DASHBOARD_PORT} from: ${FORWARD_LINE}" + ;; +esac + +section "Summary" +pass "Remote dashboard bind guard completed"