From 066c80986109c5b0efee0923b90c1ec02d9e93c8 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 12 Jun 2026 11:42:02 -0700 Subject: [PATCH 1/6] test(e2e): migrate channels add remove scenario Signed-off-by: Carlos Villela --- .github/workflows/e2e-vitest-scenarios.yaml | 112 ++++ .../live/channels-add-remove.test.ts | 488 ++++++++++++++++++ .../e2e-scenarios-workflow.test.ts | 18 +- tools/e2e-scenarios/free-standing-jobs.env | 6 +- tools/e2e-scenarios/workflow-boundary.mts | 137 +++++ 5 files changed, 757 insertions(+), 4 deletions(-) create mode 100644 test/e2e-scenario/live/channels-add-remove.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index f199050c83..813077c09b 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -1271,6 +1271,117 @@ jobs: if-no-files-found: ignore retention-days: 14 + channels-add-remove-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',channels-add-remove-vitest,') || contains(format(',{0},', inputs.scenarios), ',channels-add-remove,') }} + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-channels-add-remove + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/channels-add-remove + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-channels-add-remove" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell + env: + NEMOCLAW_NON_INTERACTIVE: "1" + run: | + set -euo pipefail + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + + - name: Run channels add/remove live test + # Migrated from test/e2e/test-channels-add-remove.sh. Preserves the + # real OpenClaw + Docker/OpenShell boundary for onboard-empty, + # channels add, rebuild, gateway credential reuse, policy-list, and + # channels remove cleanup. + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + TELEGRAM_BOT_TOKEN: "test-fake-telegram-token-add-remove-e2e" + TELEGRAM_ALLOWED_IDS: "123456789" + TELEGRAM_REQUIRE_MENTION: "0" + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/channels-add-remove.test.ts \ + --silent=false --reporter=default + + - name: Upload channels add/remove artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-channels-add-remove + path: e2e-artifacts/vitest/channels-add-remove/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + double-onboard-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',double-onboard-vitest,') || contains(format(',{0},', inputs.scenarios), ',double-onboard,') }} @@ -1885,6 +1996,7 @@ jobs: shields-config-vitest, rebuild-openclaw-vitest, sandbox-rebuild-vitest, + channels-add-remove-vitest, token-rotation-vitest, launchable-smoke-vitest, double-onboard-vitest, diff --git a/test/e2e-scenario/live/channels-add-remove.test.ts b/test/e2e-scenario/live/channels-add-remove.test.ts new file mode 100644 index 0000000000..952192a720 --- /dev/null +++ b/test/e2e-scenario/live/channels-add-remove.test.ts @@ -0,0 +1,488 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { testTimeoutOptions } from "../../helpers/timeouts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertExitZero, resultText } from "../fixtures/clients/command.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { + type SandboxClient, + sandboxAccessEnv, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +// Direct Vitest replacement coverage for test/e2e/test-channels-add-remove.sh. +// Preserve the user-visible contract: onboard OpenClaw without messaging, +// add Telegram later, rebuild through the real CLI/OpenShell boundary, verify +// registry/gateway/policy/in-sandbox state, then remove Telegram and rebuild +// back to a clean state. + +const TEST_SANDBOX_PREFIX = "e2e-channels-add-remove"; +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? TEST_SANDBOX_PREFIX; +validateSandboxName(SANDBOX_NAME); + +const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); +const TELEGRAM_TOKEN = process.env.TELEGRAM_BOT_TOKEN ?? "test-fake-telegram-token-add-remove-e2e"; +const TELEGRAM_ALLOWED_IDS = process.env.TELEGRAM_ALLOWED_IDS ?? "123456789"; +const TELEGRAM_REQUIRE_MENTION = process.env.TELEGRAM_REQUIRE_MENTION ?? "0"; +const PROVIDER_NAME = `${SANDBOX_NAME}-telegram-bridge`; + +const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 4_500) * 1_000; +const ONBOARD_TIMEOUT_MS = 25 * 60_000; +const REBUILD_TIMEOUT_MS = 30 * 60_000; +const COMMAND_TIMEOUT_MS = 2 * 60_000; + +type JsonRecord = Record; + +interface RegistrySandboxEntry extends JsonRecord { + messaging?: { + schemaVersion?: unknown; + plan?: JsonRecord; + }; +} + +type EgressProbeStatus = "open" | "denied" | "inconclusive"; + +function stripAnsi(value: string): string { + return value.replace(/\u001b\[[0-9;]*m/g, ""); +} + +function escapeRegex(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function isFakeTelegramToken(value: string): boolean { + return value.includes("fake"); +} + +function baseEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: "nemoclaw", + ...extra, + }; +} + +function channelEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return baseEnv({ + TELEGRAM_ALLOWED_IDS, + TELEGRAM_BOT_TOKEN: TELEGRAM_TOKEN, + TELEGRAM_REQUIRE_MENTION, + ...(isFakeTelegramToken(TELEGRAM_TOKEN) ? { NEMOCLAW_SKIP_TELEGRAM_REACHABILITY: "1" } : {}), + ...extra, + }); +} + +function redactionValues(apiKey: string): string[] { + return [apiKey, TELEGRAM_TOKEN].filter((value) => value.length > 0); +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup and pre-cleanup must not mask the primary phase failure. + } +} + +function readSandboxEntry(): RegistrySandboxEntry { + expect(fs.existsSync(REGISTRY_FILE), `registry file not found: ${REGISTRY_FILE}`).toBe(true); + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { + sandboxes?: Record; + }; + const entry = registry.sandboxes?.[SANDBOX_NAME]; + expect(entry, `sandbox ${SANDBOX_NAME} missing from registry`).toBeTruthy(); + if (!entry) throw new Error(`sandbox ${SANDBOX_NAME} missing from registry`); + return entry; +} + +function planArray(plan: JsonRecord, key: string): JsonRecord[] { + const value = plan[key]; + return Array.isArray(value) + ? (value.filter((item) => item && typeof item === "object") as JsonRecord[]) + : []; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function messagingPlan(): JsonRecord { + const state = readSandboxEntry().messaging; + expect(state?.schemaVersion, "messaging state missing or schemaVersion != 1").toBe(1); + const plan = state?.plan; + expect(plan && typeof plan === "object", "messaging.plan missing").toBe(true); + if (!plan || typeof plan !== "object") throw new Error("messaging.plan missing"); + expect(plan.schemaVersion, "messaging.plan missing or schemaVersion != 1").toBe(1); + expect(plan.sandboxName, "messaging.plan.sandboxName mismatch").toBe(SANDBOX_NAME); + expect(plan.agent, "messaging.plan.agent mismatch").toBe("openclaw"); + return plan; +} + +function expectHostTelegramConfig(context: string): void { + const plan = messagingPlan(); + const channel = planArray(plan, "channels").find((item) => item.channelId === "telegram"); + expect(channel, `telegram channel missing from messaging.plan.channels ${context}`).toBeTruthy(); + const inputs = planArray(channel ?? {}, "inputs"); + const inputValue = (id: string): unknown => inputs.find((input) => input.inputId === id)?.value; + expect(inputValue("allowedIds"), `allowedIds input mismatch ${context}`).toBe( + TELEGRAM_ALLOWED_IDS, + ); + expect(inputValue("requireMention"), `requireMention input mismatch ${context}`).toBe( + TELEGRAM_REQUIRE_MENTION, + ); +} + +function expectHostTelegramPlan(expected: "active" | "removed", context: string): void { + const plan = messagingPlan(); + const channels = planArray(plan, "channels"); + const channel = channels.find((item) => item.channelId === "telegram"); + const disabledChannels = stringArray(plan.disabledChannels); + const credentialBindings = planArray(plan, "credentialBindings"); + const networkPolicy = + plan.networkPolicy && typeof plan.networkPolicy === "object" + ? (plan.networkPolicy as JsonRecord) + : {}; + const networkEntries = planArray(networkPolicy, "entries"); + const networkPresets = stringArray(networkPolicy.presets); + const agentRender = planArray(plan, "agentRender"); + + if (expected === "active") { + expect( + channel, + `telegram channel missing from messaging.plan.channels ${context}`, + ).toBeTruthy(); + expect(channel?.active, `telegram plan active expected true ${context}`).toBe(true); + expect(channel?.disabled, `telegram plan disabled unexpectedly true ${context}`).not.toBe(true); + expect( + networkPresets, + `telegram missing from messaging.plan.networkPolicy.presets ${context}`, + ).toContain("telegram"); + expect( + networkEntries.some((entry) => entry.channelId === "telegram"), + `telegram missing from messaging.plan.networkPolicy.entries ${context}`, + ).toBe(true); + expect( + credentialBindings.some( + (entry) => entry.channelId === "telegram" && entry.providerEnvKey === "TELEGRAM_BOT_TOKEN", + ), + `telegram TELEGRAM_BOT_TOKEN credential binding missing ${context}`, + ).toBe(true); + expect( + agentRender.some((entry) => entry.channelId === "telegram" && entry.agent === "openclaw"), + `telegram openclaw agent render entry missing ${context}`, + ).toBe(true); + expect(disabledChannels, `telegram unexpectedly disabled ${context}`).not.toContain("telegram"); + return; + } + + expect(channel, `telegram still present in messaging.plan.channels ${context}`).toBeUndefined(); + expect(disabledChannels, `telegram still present in disabledChannels ${context}`).not.toContain( + "telegram", + ); + expect( + networkPresets, + `telegram still present in networkPolicy.presets ${context}`, + ).not.toContain("telegram"); + expect( + networkEntries.some((entry) => entry.channelId === "telegram"), + `telegram still present in networkPolicy.entries ${context}`, + ).toBe(false); + expect( + credentialBindings.some((entry) => entry.channelId === "telegram"), + `telegram credential binding still present ${context}`, + ).toBe(false); + expect( + agentRender.some((entry) => entry.channelId === "telegram"), + `telegram agent render entry still present ${context}`, + ).toBe(false); +} + +async function expectSandboxReady( + sandbox: SandboxClient, + artifactName: string, +): Promise { + const result = await sandbox.list({ + artifactName, + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(result, "openshell sandbox list"); + const lines = stripAnsi(resultText(result)).split(/\r?\n/); + expect( + lines.some((line) => { + const trimmed = line.trim(); + const [name] = trimmed.split(/\s+/); + return name === SANDBOX_NAME && /\bReady\b/i.test(trimmed); + }), + `${SANDBOX_NAME} was not Ready:\n${resultText(result)}`, + ).toBe(true); + return result; +} + +async function expectProvider( + host: HostCliClient, + expected: "present" | "absent", + artifactName: string, +): Promise { + const result = await host.command("openshell", ["provider", "get", PROVIDER_NAME], { + artifactName, + env: baseEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + if (expected === "present") { + assertExitZero(result, `openshell provider get ${PROVIDER_NAME}`); + } else { + expect( + result.exitCode, + `${PROVIDER_NAME} unexpectedly exists:\n${resultText(result)}`, + ).not.toBe(0); + } +} + +async function openClawHasTelegram(sandbox: SandboxClient, artifactName: string): Promise { + const result = await sandbox.exec( + SANDBOX_NAME, + [ + "python3", + "-c", + [ + "import json", + "data=json.load(open('/sandbox/.openclaw/openclaw.json'))", + "print('yes' if 'telegram' in data.get('channels', {}) else 'no')", + ].join("; "), + ], + { + artifactName, + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }, + ); + assertExitZero(result, "read /sandbox/.openclaw/openclaw.json"); + const verdict = stripAnsi(result.stdout).trim().split(/\r?\n/).at(-1); + expect(["yes", "no"], `unexpected openclaw.json verdict:\n${resultText(result)}`).toContain( + verdict, + ); + return verdict === "yes"; +} + +async function expectOpenClawTelegram( + sandbox: SandboxClient, + expected: boolean, + artifactName: string, +): Promise { + await expect(openClawHasTelegram(sandbox, artifactName)).resolves.toBe(expected); +} + +function policyListHasActivePreset(output: string, preset: string): boolean { + const activePreset = new RegExp(`^\\s*\\u25cf\\s+${escapeRegex(preset)}\\b`, "im"); + return activePreset.test(stripAnsi(output)); +} + +async function expectPolicyPreset( + host: HostCliClient, + preset: string, + expected: "applied" | "not-applied", + artifactName: string, +): Promise { + const result = await host.nemoclaw([SANDBOX_NAME, "policy-list"], { + artifactName, + env: baseEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(result, `nemoclaw ${SANDBOX_NAME} policy-list`); + const applied = policyListHasActivePreset(resultText(result), preset); + expect(applied, `${preset} preset expected ${expected}:\n${resultText(result)}`).toBe( + expected === "applied", + ); +} + +async function telegramEgressProbe( + sandbox: SandboxClient, + artifactName: string, +): Promise<{ result: ShellProbeResult; status: EgressProbeStatus }> { + const source = [ + `const url = ${JSON.stringify(`https://api.telegram.org/bot${TELEGRAM_TOKEN}/getMe`)};`, + "fetch(url, { signal: AbortSignal.timeout(15000) })", + " .then((response) => console.log(`STATUS_${response.status}`))", + " .catch((error) => console.log(`ERROR_${error.cause?.code || error.code || error.message}`));", + ].join("\n"); + const result = await sandbox.exec(SANDBOX_NAME, ["node", "-e", source], { + artifactName, + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(result, "telegram egress probe"); + const output = resultText(result); + if (/STATUS_[24][0-9][0-9]/.test(output)) return { result, status: "open" }; + if (/policy_denied|engine:ssrf|forbidden by policy|CONNECT.*40[0-9]/i.test(output)) { + return { result, status: "denied" }; + } + return { result, status: "inconclusive" }; +} + +const liveTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +liveTest( + "channels add/remove telegram updates registry, gateway, policy, and sandbox state", + testTimeoutOptions(TEST_TIMEOUT_MS), + async ({ artifacts, cleanup, environment, host, lifecycle, onboard, sandbox, secrets }) => { + if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { + throw new Error( + `channels-add-remove live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, + ); + } + const apiKey = secrets.required("NVIDIA_API_KEY"); + const secretsToRedact = redactionValues(apiKey); + + const ready = await environment.assertReady({ + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + }); + + await artifacts.writeJson("scenario.json", { + id: "channels-add-remove", + legacySource: "test/e2e/test-channels-add-remove.sh", + runner: "vitest", + sandboxName: SANDBOX_NAME, + contract: [ + "onboard creates an OpenClaw sandbox with no Telegram channel", + "channels add telegram registers the bridge and persists messaging.plan", + "post-add rebuild reuses the gateway-stored inference credential when NVIDIA_API_KEY is absent", + "post-add rebuild applies the Telegram policy preset and renders openclaw.json channel state", + "channels remove telegram removes provider, policy, registry plan, and rendered channel state after rebuild", + ], + }); + + cleanup.add(`destroy ${SANDBOX_NAME} after channels add/remove live test`, async () => { + await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "cleanup-nemoclaw-destroy")); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete", + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + await bestEffort(() => + host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy", + env: baseEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + }); + + await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy")); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-sandbox-delete", + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + await bestEffort(() => + host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "pre-cleanup-openshell-gateway-destroy", + env: baseEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + + const instance = await onboard.from(ready, { + sandboxName: SANDBOX_NAME, + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + await expectSandboxReady(sandbox, "phase-1-sandbox-ready-after-onboard"); + + await expectProvider(host, "absent", "phase-2-provider-get-baseline"); + await expectOpenClawTelegram(sandbox, false, "phase-2-openclaw-json-baseline"); + await expectPolicyPreset(host, "telegram", "not-applied", "phase-2-policy-list-baseline"); + + const add = await host.nemoclaw([SANDBOX_NAME, "channels", "add", "telegram"], { + artifactName: "phase-3-channels-add-telegram", + env: channelEnv(), + redactionValues: secretsToRedact, + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(add, `nemoclaw ${SANDBOX_NAME} channels add telegram`); + expect(resultText(add)).toContain("Registered telegram"); + expectHostTelegramConfig("after channels add"); + expectHostTelegramPlan("active", "after channels add"); + + const rebuildAdd = await host.nemoclaw([SANDBOX_NAME, "rebuild", "--yes"], { + artifactName: "phase-3-rebuild-after-add-without-host-nvidia-key", + env: channelEnv(), + redactionValues: secretsToRedact, + timeoutMs: REBUILD_TIMEOUT_MS, + }); + expect(resultText(rebuildAdd)).not.toContain("provider credential not found"); + assertExitZero(rebuildAdd, `nemoclaw ${SANDBOX_NAME} rebuild --yes after add`); + await lifecycle.assertSandboxReadyAfterRebuild(instance, { + artifactNamePrefix: "phase-3-sandbox-ready-after-add-rebuild", + env: sandboxAccessEnv(), + attempts: 12, + delayMs: 5_000, + }); + + await expectPolicyPreset(host, "telegram", "applied", "phase-4-policy-list-after-add"); + await expectOpenClawTelegram(sandbox, true, "phase-4-openclaw-json-after-add"); + await expectProvider(host, "present", "phase-4-provider-get-after-add"); + expectHostTelegramConfig("after add+rebuild"); + expectHostTelegramPlan("active", "after add+rebuild"); + + const egress = await telegramEgressProbe(sandbox, "phase-4-telegram-egress-probe"); + if (egress.status === "denied") { + throw new Error(`egress to api.telegram.org was blocked:\n${resultText(egress.result)}`); + } + if (egress.status === "inconclusive") { + await artifacts.writeText( + "phase-4-telegram-egress-inconclusive.txt", + `Telegram egress probe was inconclusive; preserving the legacy soft-skip behavior.\n\n${resultText( + egress.result, + )}`, + ); + } + + const remove = await host.nemoclaw([SANDBOX_NAME, "channels", "remove", "telegram"], { + artifactName: "phase-5-channels-remove-telegram", + env: channelEnv({ NVIDIA_API_KEY: apiKey }), + redactionValues: secretsToRedact, + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(remove, `nemoclaw ${SANDBOX_NAME} channels remove telegram`); + expect(resultText(remove)).toContain("Removed telegram"); + expectHostTelegramPlan("removed", "after channels remove"); + + const rebuildRemove = await host.nemoclaw([SANDBOX_NAME, "rebuild", "--yes"], { + artifactName: "phase-5-rebuild-after-remove", + env: channelEnv({ NVIDIA_API_KEY: apiKey }), + redactionValues: secretsToRedact, + timeoutMs: REBUILD_TIMEOUT_MS, + }); + assertExitZero(rebuildRemove, `nemoclaw ${SANDBOX_NAME} rebuild --yes after remove`); + await lifecycle.assertSandboxReadyAfterRebuild(instance, { + artifactNamePrefix: "phase-5-sandbox-ready-after-remove-rebuild", + env: sandboxAccessEnv(), + attempts: 12, + delayMs: 5_000, + }); + + await expectOpenClawTelegram(sandbox, false, "phase-6-openclaw-json-after-remove"); + await expectProvider(host, "absent", "phase-6-provider-get-after-remove"); + await expectPolicyPreset(host, "telegram", "not-applied", "phase-6-policy-list-after-remove"); + expectHostTelegramPlan("removed", "after remove+rebuild"); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 354712ecde..9acde64902 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -181,6 +181,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["inference-routing-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "channels-add-remove" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["channels-add-remove-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "channels-add-remove-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["channels-add-remove-vitest"], + registryScenarios: [], + }); expect(evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "hermes-e2e" })).toMatchObject({ valid: true, liveScenariosRuns: false, @@ -376,7 +392,7 @@ describe("e2e-vitest-scenarios workflow boundary", () => { registryScenarios: [], }); } - }); + }, 15_000); it("flags direct dispatch-input interpolation and unsafe artifact upload", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); diff --git a/tools/e2e-scenarios/free-standing-jobs.env b/tools/e2e-scenarios/free-standing-jobs.env index 68ce23e36d..fa8e0b5afd 100644 --- a/tools/e2e-scenarios/free-standing-jobs.env +++ b/tools/e2e-scenarios/free-standing-jobs.env @@ -1,5 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -allowed_jobs=openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,shields-config-vitest,rebuild-openclaw-vitest,sandbox-rebuild-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,credential-sanitization-vitest,sandbox-survival-vitest -free_standing_scenarios_csv=openshell-version-pin,onboard-negative-paths,skill-agent,inference-routing,runtime-overrides,hermes-e2e,hermes-root-entrypoint-smoke,network-policy,shields-config,rebuild-openclaw,sandbox-rebuild,token-rotation,openclaw-tui-chat-correlation,double-onboard,issue-4434-tui-unreachable-inference,model-router-provider-routed-inference,credential-sanitization,sandbox-survival -free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest,onboard-negative-paths:onboard-negative-paths-vitest,skill-agent:skill-agent-vitest,inference-routing:inference-routing-vitest,runtime-overrides:runtime-overrides-vitest,hermes-e2e:hermes-e2e-vitest,hermes-root-entrypoint-smoke:hermes-root-entrypoint-smoke-vitest,network-policy:network-policy-vitest,shields-config:shields-config-vitest,rebuild-openclaw:rebuild-openclaw-vitest,sandbox-rebuild:sandbox-rebuild-vitest,token-rotation:token-rotation-vitest,openclaw-tui-chat-correlation:openclaw-tui-chat-correlation-vitest,double-onboard:double-onboard-vitest,issue-4434-tui-unreachable-inference:issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference:model-router-provider-routed-inference-vitest,credential-sanitization:credential-sanitization-vitest,sandbox-survival:sandbox-survival-vitest +allowed_jobs=openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,shields-config-vitest,rebuild-openclaw-vitest,sandbox-rebuild-vitest,channels-add-remove-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,credential-sanitization-vitest,sandbox-survival-vitest +free_standing_scenarios_csv=openshell-version-pin,onboard-negative-paths,skill-agent,inference-routing,runtime-overrides,hermes-e2e,hermes-root-entrypoint-smoke,network-policy,shields-config,rebuild-openclaw,sandbox-rebuild,channels-add-remove,token-rotation,openclaw-tui-chat-correlation,double-onboard,issue-4434-tui-unreachable-inference,model-router-provider-routed-inference,credential-sanitization,sandbox-survival +free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest,onboard-negative-paths:onboard-negative-paths-vitest,skill-agent:skill-agent-vitest,inference-routing:inference-routing-vitest,runtime-overrides:runtime-overrides-vitest,hermes-e2e:hermes-e2e-vitest,hermes-root-entrypoint-smoke:hermes-root-entrypoint-smoke-vitest,network-policy:network-policy-vitest,shields-config:shields-config-vitest,rebuild-openclaw:rebuild-openclaw-vitest,sandbox-rebuild:sandbox-rebuild-vitest,channels-add-remove:channels-add-remove-vitest,token-rotation:token-rotation-vitest,openclaw-tui-chat-correlation:openclaw-tui-chat-correlation-vitest,double-onboard:double-onboard-vitest,issue-4434-tui-unreachable-inference:issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference:model-router-provider-routed-inference-vitest,credential-sanitization:credential-sanitization-vitest,sandbox-survival:sandbox-survival-vitest diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 9e55fc3ff3..243af82e98 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -1049,6 +1049,142 @@ function validateSandboxRebuildVitestJob(errors: string[], jobs: WorkflowRecord) } } +function validateChannelsAddRemoveVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "channels-add-remove-vitest"; + const scenarioName = "channels-add-remove"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing channels-add-remove-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("channels-add-remove-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + if (job["timeout-minutes"] !== 75) { + errors.push("channels-add-remove-vitest job must keep the legacy 75 minute timeout"); + } + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/vitest/channels-add-remove" + ) { + errors.push( + "channels-add-remove-vitest job must write artifacts under e2e-artifacts/vitest/channels-add-remove", + ); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("channels-add-remove-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-channels-add-remove") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_SANDBOX_NAME=e2e-channels-add-remove"); + } + if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_NON_INTERACTIVE=1"); + } + if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("channels-add-remove-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); + } + for (const secret of ["NVIDIA_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + requireEnvDoesNotExposeSecret(errors, "channels-add-remove-vitest job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `channels-add-remove-vitest step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run channels add/remove live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("channels-add-remove-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "channels-add-remove-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("channels-add-remove-vitest checkout step must set persist-credentials=false"); + } + + const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerHubEnv = asRecord(dockerHubAuth?.env); + if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push("channels-add-remove-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets"); + } + if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push("channels-add-remove-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); + } + requireRunContains(errors, dockerHubAuth, "docker login docker.io"); + requireRunContains(errors, dockerHubAuth, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("channels-add-remove-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "channels-add-remove-vitest setup-node"); + + const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); + requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); + requireRunContains(errors, installOpenShell, "-u NVIDIA_API_KEY"); + requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run channels add/remove live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { + errors.push("channels-add-remove-vitest step must receive NVIDIA_API_KEY from secrets"); + } + if (runVitestEnv.TELEGRAM_BOT_TOKEN !== "test-fake-telegram-token-add-remove-e2e") { + errors.push("channels-add-remove-vitest step must set the fake Telegram token"); + } + if (runVitestEnv.TELEGRAM_ALLOWED_IDS !== "123456789") { + errors.push("channels-add-remove-vitest step must set TELEGRAM_ALLOWED_IDS"); + } + if (runVitestEnv.TELEGRAM_REQUIRE_MENTION !== "0") { + errors.push("channels-add-remove-vitest step must set TELEGRAM_REQUIRE_MENTION"); + } + requireRunContains(errors, runVitest, "OPENSHELL_BIN"); + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/channels-add-remove.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload channels add/remove artifacts"); + requireFullShaAction(errors, upload, "channels-add-remove-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-channels-add-remove") { + errors.push("channels-add-remove-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/channels-add-remove/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("channels-add-remove-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("channels-add-remove-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("channels-add-remove-vitest artifact upload retention-days must be 14"); + } +} + function validateTokenRotationVitestJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "token-rotation-vitest"; const job = asRecord(jobs[jobName]); @@ -2118,6 +2254,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateShieldsConfigVitestJob(errors, jobs); validateRebuildOpenClawVitestJob(errors, jobs); validateSandboxRebuildVitestJob(errors, jobs); + validateChannelsAddRemoveVitestJob(errors, jobs); validateTokenRotationVitestJob(errors, jobs); validateFreeStandingJobSelector( errors, From acd26c021b732dfb121bd44a9eedefe885831c81 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 12 Jun 2026 12:13:27 -0700 Subject: [PATCH 2/6] test(e2e): skip channels scenario on endpoint throttling Signed-off-by: Carlos Villela --- .../fixtures/phases/onboarding.ts | 1 + .../live/channels-add-remove.test.ts | 29 +++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/test/e2e-scenario/fixtures/phases/onboarding.ts b/test/e2e-scenario/fixtures/phases/onboarding.ts index 00cbd0f545..aa6b533ccc 100644 --- a/test/e2e-scenario/fixtures/phases/onboarding.ts +++ b/test/e2e-scenario/fixtures/phases/onboarding.ts @@ -38,6 +38,7 @@ const MISSING_SANDBOX_DELETE_PATTERNS = [ /sandbox not found/i, /sandbox .* not found/i, /sandbox .* not present/i, + /sandbox .* does not exist/i, /sandbox does not exist/i, /no such sandbox/i, ]; diff --git a/test/e2e-scenario/live/channels-add-remove.test.ts b/test/e2e-scenario/live/channels-add-remove.test.ts index 952192a720..5cc036c1d1 100644 --- a/test/e2e-scenario/live/channels-add-remove.test.ts +++ b/test/e2e-scenario/live/channels-add-remove.test.ts @@ -61,6 +61,11 @@ function isFakeTelegramToken(value: string): boolean { return value.includes("fake"); } +function isEndpointRateLimited(error: unknown): boolean { + const text = error instanceof Error ? error.message : String(error); + return /HTTP 429|rate limit|too many requests/i.test(text); +} + function baseEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { return { ...buildAvailabilityProbeEnv(), @@ -337,7 +342,7 @@ const liveTest = shouldRunLiveE2EScenarios() ? test : test.skip; liveTest( "channels add/remove telegram updates registry, gateway, policy, and sandbox state", testTimeoutOptions(TEST_TIMEOUT_MS), - async ({ artifacts, cleanup, environment, host, lifecycle, onboard, sandbox, secrets }) => { + async ({ artifacts, cleanup, environment, host, lifecycle, onboard, sandbox, secrets, skip }) => { if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { throw new Error( `channels-add-remove live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, @@ -401,10 +406,24 @@ liveTest( }), ); - const instance = await onboard.from(ready, { - sandboxName: SANDBOX_NAME, - timeoutMs: ONBOARD_TIMEOUT_MS, - }); + let instance; + try { + instance = await onboard.from(ready, { + sandboxName: SANDBOX_NAME, + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + } catch (error) { + if (isEndpointRateLimited(error)) { + await artifacts.writeText( + "endpoint-rate-limit-skip.txt", + error instanceof Error ? error.message : String(error), + ); + skip( + "NVIDIA endpoint validation was rate-limited before the channels add/remove contract could run", + ); + } + throw error; + } await expectSandboxReady(sandbox, "phase-1-sandbox-ready-after-onboard"); await expectProvider(host, "absent", "phase-2-provider-get-baseline"); From 8bcc73cc5e5dcb972be4e3fe11f805b511173b5c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 12 Jun 2026 12:29:17 -0700 Subject: [PATCH 3/6] test(e2e): keep channels egress probe single-line Signed-off-by: Carlos Villela --- test/e2e-scenario/live/channels-add-remove.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-scenario/live/channels-add-remove.test.ts b/test/e2e-scenario/live/channels-add-remove.test.ts index 5cc036c1d1..c2616e85cc 100644 --- a/test/e2e-scenario/live/channels-add-remove.test.ts +++ b/test/e2e-scenario/live/channels-add-remove.test.ts @@ -322,7 +322,7 @@ async function telegramEgressProbe( "fetch(url, { signal: AbortSignal.timeout(15000) })", " .then((response) => console.log(`STATUS_${response.status}`))", " .catch((error) => console.log(`ERROR_${error.cause?.code || error.code || error.message}`));", - ].join("\n"); + ].join(" "); const result = await sandbox.exec(SANDBOX_NAME, ["node", "-e", source], { artifactName, env: sandboxAccessEnv(), From 2ed8887347aa405adb39a41e39cee41e9cf59356 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 12 Jun 2026 14:30:05 -0700 Subject: [PATCH 4/6] test(e2e): tighten channels provider absence check Signed-off-by: Carlos Villela --- test/e2e-scenario/live/channels-add-remove.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/e2e-scenario/live/channels-add-remove.test.ts b/test/e2e-scenario/live/channels-add-remove.test.ts index c2616e85cc..1a2a30da7c 100644 --- a/test/e2e-scenario/live/channels-add-remove.test.ts +++ b/test/e2e-scenario/live/channels-add-remove.test.ts @@ -249,10 +249,15 @@ async function expectProvider( if (expected === "present") { assertExitZero(result, `openshell provider get ${PROVIDER_NAME}`); } else { + const text = stripAnsi(resultText(result)); expect( result.exitCode, `${PROVIDER_NAME} unexpectedly exists:\n${resultText(result)}`, ).not.toBe(0); + expect( + /not found|does not exist|no provider|unknown provider/i.test(text), + `${PROVIDER_NAME} absence check failed for an unexpected reason:\n${resultText(result)}`, + ).toBe(true); } } From 7ded55813ce1fb2d6b0ad423d93d32a579c7d468 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 12 Jun 2026 14:36:28 -0700 Subject: [PATCH 5/6] test(e2e): skip channels on validation outage Signed-off-by: Carlos Villela --- .../live/channels-add-remove.test.ts | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/e2e-scenario/live/channels-add-remove.test.ts b/test/e2e-scenario/live/channels-add-remove.test.ts index 1a2a30da7c..f55d1b0e18 100644 --- a/test/e2e-scenario/live/channels-add-remove.test.ts +++ b/test/e2e-scenario/live/channels-add-remove.test.ts @@ -62,8 +62,18 @@ function isFakeTelegramToken(value: string): boolean { } function isEndpointRateLimited(error: unknown): boolean { - const text = error instanceof Error ? error.message : String(error); - return /HTTP 429|rate limit|too many requests/i.test(text); + const text = errorText(error); + return ( + /NVIDIA Endpoints endpoint validation failed/i.test(text) && + (/Validation details were omitted/i.test(text) || + /HTTP 429|rate limit|too many requests|quota|temporarily unavailable|timed out|timeout/i.test( + text, + )) + ); +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); } function baseEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { @@ -419,12 +429,9 @@ liveTest( }); } catch (error) { if (isEndpointRateLimited(error)) { - await artifacts.writeText( - "endpoint-rate-limit-skip.txt", - error instanceof Error ? error.message : String(error), - ); + await artifacts.writeText("endpoint-rate-limit-skip.txt", errorText(error)); skip( - "NVIDIA endpoint validation was rate-limited before the channels add/remove contract could run", + "NVIDIA endpoint validation was unavailable/rate-limited before the channels add/remove contract could run", ); } throw error; From 36d8c58c91e43d3877080df157c9cd7521dc3b8b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 12 Jun 2026 21:49:43 -0700 Subject: [PATCH 6/6] fix(e2e): restore channels workflow boundary validator --- tools/e2e-scenarios/workflow-boundary.mts | 153 +++++++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index bf72169480..6f25f04bca 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -2480,6 +2480,157 @@ function validateModelRouterProviderRoutedInferenceVitestJob( requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); } +function validateChannelsAddRemoveVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "channels-add-remove-vitest"; + const scenarioName = "channels-add-remove"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing channels-add-remove-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("channels-add-remove-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + if (job["timeout-minutes"] !== 75) { + errors.push("channels-add-remove-vitest job must keep the legacy 75 minute timeout"); + } + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/channels-add-remove" + ) { + errors.push( + "channels-add-remove-vitest job must write artifacts under e2e-artifacts/vitest/channels-add-remove", + ); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("channels-add-remove-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-channels-add-remove") { + errors.push( + "channels-add-remove-vitest job must set NEMOCLAW_SANDBOX_NAME=e2e-channels-add-remove", + ); + } + if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_NON_INTERACTIVE=1"); + } + if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("channels-add-remove-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); + } + for (const secret of [ + "NVIDIA_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { + requireEnvDoesNotExposeSecret(errors, "channels-add-remove-vitest job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `channels-add-remove-vitest step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run channels add/remove live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("channels-add-remove-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "channels-add-remove-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("channels-add-remove-vitest checkout step must set persist-credentials=false"); + } + + const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerHubEnv = asRecord(dockerHubAuth?.env); + if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push( + "channels-add-remove-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); + } + if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push( + "channels-add-remove-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); + } + requireRunContains(errors, dockerHubAuth, "docker login docker.io"); + requireRunContains(errors, dockerHubAuth, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("channels-add-remove-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "channels-add-remove-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); + requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); + requireRunContains(errors, installOpenShell, "-u NVIDIA_API_KEY"); + requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run channels add/remove live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { + errors.push("channels-add-remove-vitest step must receive NVIDIA_API_KEY from secrets"); + } + if (runVitestEnv.TELEGRAM_BOT_TOKEN !== "test-fake-telegram-token-add-remove-e2e") { + errors.push("channels-add-remove-vitest step must set the fake Telegram token"); + } + if (runVitestEnv.TELEGRAM_ALLOWED_IDS !== "123456789") { + errors.push("channels-add-remove-vitest step must set TELEGRAM_ALLOWED_IDS"); + } + if (runVitestEnv.TELEGRAM_REQUIRE_MENTION !== "0") { + errors.push("channels-add-remove-vitest step must set TELEGRAM_REQUIRE_MENTION"); + } + requireRunContains(errors, runVitest, "OPENSHELL_BIN"); + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/channels-add-remove.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload channels add/remove artifacts"); + requireFullShaAction(errors, upload, "channels-add-remove-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-channels-add-remove") { + errors.push("channels-add-remove-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/channels-add-remove/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("channels-add-remove-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("channels-add-remove-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("channels-add-remove-vitest artifact upload retention-days must be 14"); + } +} + function validateBedrockRuntimeCompatibleAnthropicVitestJob( errors: string[], jobs: WorkflowRecord, @@ -3007,7 +3158,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateBedrockRuntimeCompatibleAnthropicVitestJob(errors, jobs); - asRecord(errors, jobs); + validateChannelsAddRemoveVitestJob(errors, jobs); const reportToPr = asRecord(jobs["report-to-pr"]); if (Object.keys(reportToPr).length === 0) {