diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index a30fbf50b7..d1df811d7a 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -2622,6 +2622,119 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + channels-add-remove-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',channels-add-remove-vitest,') || contains(format(',{0},', inputs.scenarios), ',channels-add-remove,') }} + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "channels-add-remove" + DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-channels-add-remove + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/channels-add-remove + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-channels-add-remove" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell + env: + NEMOCLAW_NON_INTERACTIVE: "1" + run: | + set -euo pipefail + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + + - name: Run channels add/remove live test + # Migrated from test/e2e/test-channels-add-remove.sh. Preserves the + # real OpenClaw + Docker/OpenShell boundary for onboard-empty, + # channels add, rebuild, gateway credential reuse, policy-list, and + # channels remove cleanup. + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + TELEGRAM_BOT_TOKEN: "test-fake-telegram-token-add-remove-e2e" + TELEGRAM_ALLOWED_IDS: "123456789" + TELEGRAM_REQUIRE_MENTION: "0" + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/channels-add-remove.test.ts \ + --silent=false --reporter=default + + - name: Upload channels add/remove artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-channels-add-remove + path: e2e-artifacts/vitest/channels-add-remove/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + # ── PR result comment (mirrors nightly-e2e.yaml's report-to-pr) ─────────── # Posts a results table on the open PR for the dispatching branch (or the # PR identified by `inputs.pr_number`). `if: always()` so the comment lands @@ -2662,8 +2775,9 @@ jobs: openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, issue-4434-tui-unreachable-inference-vitest, - openclaw-inference-switch-vitest, bedrock-runtime-compatible-anthropic-vitest, - + openclaw-inference-switch-vitest, + bedrock-runtime-compatible-anthropic-vitest, + channels-add-remove-vitest, ] if: ${{ always() && github.event_name == 'workflow_dispatch' }} permissions: diff --git a/test/e2e-scenario/live/channels-add-remove.test.ts b/test/e2e-scenario/live/channels-add-remove.test.ts new file mode 100644 index 0000000000..f55d1b0e18 --- /dev/null +++ b/test/e2e-scenario/live/channels-add-remove.test.ts @@ -0,0 +1,519 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { testTimeoutOptions } from "../../helpers/timeouts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertExitZero, resultText } from "../fixtures/clients/command.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { + type SandboxClient, + sandboxAccessEnv, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +// Direct Vitest replacement coverage for test/e2e/test-channels-add-remove.sh. +// Preserve the user-visible contract: onboard OpenClaw without messaging, +// add Telegram later, rebuild through the real CLI/OpenShell boundary, verify +// registry/gateway/policy/in-sandbox state, then remove Telegram and rebuild +// back to a clean state. + +const TEST_SANDBOX_PREFIX = "e2e-channels-add-remove"; +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? TEST_SANDBOX_PREFIX; +validateSandboxName(SANDBOX_NAME); + +const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); +const TELEGRAM_TOKEN = process.env.TELEGRAM_BOT_TOKEN ?? "test-fake-telegram-token-add-remove-e2e"; +const TELEGRAM_ALLOWED_IDS = process.env.TELEGRAM_ALLOWED_IDS ?? "123456789"; +const TELEGRAM_REQUIRE_MENTION = process.env.TELEGRAM_REQUIRE_MENTION ?? "0"; +const PROVIDER_NAME = `${SANDBOX_NAME}-telegram-bridge`; + +const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 4_500) * 1_000; +const ONBOARD_TIMEOUT_MS = 25 * 60_000; +const REBUILD_TIMEOUT_MS = 30 * 60_000; +const COMMAND_TIMEOUT_MS = 2 * 60_000; + +type JsonRecord = Record; + +interface RegistrySandboxEntry extends JsonRecord { + messaging?: { + schemaVersion?: unknown; + plan?: JsonRecord; + }; +} + +type EgressProbeStatus = "open" | "denied" | "inconclusive"; + +function stripAnsi(value: string): string { + return value.replace(/\u001b\[[0-9;]*m/g, ""); +} + +function escapeRegex(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function isFakeTelegramToken(value: string): boolean { + return value.includes("fake"); +} + +function isEndpointRateLimited(error: unknown): boolean { + const text = errorText(error); + return ( + /NVIDIA Endpoints endpoint validation failed/i.test(text) && + (/Validation details were omitted/i.test(text) || + /HTTP 429|rate limit|too many requests|quota|temporarily unavailable|timed out|timeout/i.test( + text, + )) + ); +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function baseEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: "nemoclaw", + ...extra, + }; +} + +function channelEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return baseEnv({ + TELEGRAM_ALLOWED_IDS, + TELEGRAM_BOT_TOKEN: TELEGRAM_TOKEN, + TELEGRAM_REQUIRE_MENTION, + ...(isFakeTelegramToken(TELEGRAM_TOKEN) ? { NEMOCLAW_SKIP_TELEGRAM_REACHABILITY: "1" } : {}), + ...extra, + }); +} + +function redactionValues(apiKey: string): string[] { + return [apiKey, TELEGRAM_TOKEN].filter((value) => value.length > 0); +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup and pre-cleanup must not mask the primary phase failure. + } +} + +function readSandboxEntry(): RegistrySandboxEntry { + expect(fs.existsSync(REGISTRY_FILE), `registry file not found: ${REGISTRY_FILE}`).toBe(true); + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { + sandboxes?: Record; + }; + const entry = registry.sandboxes?.[SANDBOX_NAME]; + expect(entry, `sandbox ${SANDBOX_NAME} missing from registry`).toBeTruthy(); + if (!entry) throw new Error(`sandbox ${SANDBOX_NAME} missing from registry`); + return entry; +} + +function planArray(plan: JsonRecord, key: string): JsonRecord[] { + const value = plan[key]; + return Array.isArray(value) + ? (value.filter((item) => item && typeof item === "object") as JsonRecord[]) + : []; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function messagingPlan(): JsonRecord { + const state = readSandboxEntry().messaging; + expect(state?.schemaVersion, "messaging state missing or schemaVersion != 1").toBe(1); + const plan = state?.plan; + expect(plan && typeof plan === "object", "messaging.plan missing").toBe(true); + if (!plan || typeof plan !== "object") throw new Error("messaging.plan missing"); + expect(plan.schemaVersion, "messaging.plan missing or schemaVersion != 1").toBe(1); + expect(plan.sandboxName, "messaging.plan.sandboxName mismatch").toBe(SANDBOX_NAME); + expect(plan.agent, "messaging.plan.agent mismatch").toBe("openclaw"); + return plan; +} + +function expectHostTelegramConfig(context: string): void { + const plan = messagingPlan(); + const channel = planArray(plan, "channels").find((item) => item.channelId === "telegram"); + expect(channel, `telegram channel missing from messaging.plan.channels ${context}`).toBeTruthy(); + const inputs = planArray(channel ?? {}, "inputs"); + const inputValue = (id: string): unknown => inputs.find((input) => input.inputId === id)?.value; + expect(inputValue("allowedIds"), `allowedIds input mismatch ${context}`).toBe( + TELEGRAM_ALLOWED_IDS, + ); + expect(inputValue("requireMention"), `requireMention input mismatch ${context}`).toBe( + TELEGRAM_REQUIRE_MENTION, + ); +} + +function expectHostTelegramPlan(expected: "active" | "removed", context: string): void { + const plan = messagingPlan(); + const channels = planArray(plan, "channels"); + const channel = channels.find((item) => item.channelId === "telegram"); + const disabledChannels = stringArray(plan.disabledChannels); + const credentialBindings = planArray(plan, "credentialBindings"); + const networkPolicy = + plan.networkPolicy && typeof plan.networkPolicy === "object" + ? (plan.networkPolicy as JsonRecord) + : {}; + const networkEntries = planArray(networkPolicy, "entries"); + const networkPresets = stringArray(networkPolicy.presets); + const agentRender = planArray(plan, "agentRender"); + + if (expected === "active") { + expect( + channel, + `telegram channel missing from messaging.plan.channels ${context}`, + ).toBeTruthy(); + expect(channel?.active, `telegram plan active expected true ${context}`).toBe(true); + expect(channel?.disabled, `telegram plan disabled unexpectedly true ${context}`).not.toBe(true); + expect( + networkPresets, + `telegram missing from messaging.plan.networkPolicy.presets ${context}`, + ).toContain("telegram"); + expect( + networkEntries.some((entry) => entry.channelId === "telegram"), + `telegram missing from messaging.plan.networkPolicy.entries ${context}`, + ).toBe(true); + expect( + credentialBindings.some( + (entry) => entry.channelId === "telegram" && entry.providerEnvKey === "TELEGRAM_BOT_TOKEN", + ), + `telegram TELEGRAM_BOT_TOKEN credential binding missing ${context}`, + ).toBe(true); + expect( + agentRender.some((entry) => entry.channelId === "telegram" && entry.agent === "openclaw"), + `telegram openclaw agent render entry missing ${context}`, + ).toBe(true); + expect(disabledChannels, `telegram unexpectedly disabled ${context}`).not.toContain("telegram"); + return; + } + + expect(channel, `telegram still present in messaging.plan.channels ${context}`).toBeUndefined(); + expect(disabledChannels, `telegram still present in disabledChannels ${context}`).not.toContain( + "telegram", + ); + expect( + networkPresets, + `telegram still present in networkPolicy.presets ${context}`, + ).not.toContain("telegram"); + expect( + networkEntries.some((entry) => entry.channelId === "telegram"), + `telegram still present in networkPolicy.entries ${context}`, + ).toBe(false); + expect( + credentialBindings.some((entry) => entry.channelId === "telegram"), + `telegram credential binding still present ${context}`, + ).toBe(false); + expect( + agentRender.some((entry) => entry.channelId === "telegram"), + `telegram agent render entry still present ${context}`, + ).toBe(false); +} + +async function expectSandboxReady( + sandbox: SandboxClient, + artifactName: string, +): Promise { + const result = await sandbox.list({ + artifactName, + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(result, "openshell sandbox list"); + const lines = stripAnsi(resultText(result)).split(/\r?\n/); + expect( + lines.some((line) => { + const trimmed = line.trim(); + const [name] = trimmed.split(/\s+/); + return name === SANDBOX_NAME && /\bReady\b/i.test(trimmed); + }), + `${SANDBOX_NAME} was not Ready:\n${resultText(result)}`, + ).toBe(true); + return result; +} + +async function expectProvider( + host: HostCliClient, + expected: "present" | "absent", + artifactName: string, +): Promise { + const result = await host.command("openshell", ["provider", "get", PROVIDER_NAME], { + artifactName, + env: baseEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + if (expected === "present") { + assertExitZero(result, `openshell provider get ${PROVIDER_NAME}`); + } else { + const text = stripAnsi(resultText(result)); + expect( + result.exitCode, + `${PROVIDER_NAME} unexpectedly exists:\n${resultText(result)}`, + ).not.toBe(0); + expect( + /not found|does not exist|no provider|unknown provider/i.test(text), + `${PROVIDER_NAME} absence check failed for an unexpected reason:\n${resultText(result)}`, + ).toBe(true); + } +} + +async function openClawHasTelegram(sandbox: SandboxClient, artifactName: string): Promise { + const result = await sandbox.exec( + SANDBOX_NAME, + [ + "python3", + "-c", + [ + "import json", + "data=json.load(open('/sandbox/.openclaw/openclaw.json'))", + "print('yes' if 'telegram' in data.get('channels', {}) else 'no')", + ].join("; "), + ], + { + artifactName, + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }, + ); + assertExitZero(result, "read /sandbox/.openclaw/openclaw.json"); + const verdict = stripAnsi(result.stdout).trim().split(/\r?\n/).at(-1); + expect(["yes", "no"], `unexpected openclaw.json verdict:\n${resultText(result)}`).toContain( + verdict, + ); + return verdict === "yes"; +} + +async function expectOpenClawTelegram( + sandbox: SandboxClient, + expected: boolean, + artifactName: string, +): Promise { + await expect(openClawHasTelegram(sandbox, artifactName)).resolves.toBe(expected); +} + +function policyListHasActivePreset(output: string, preset: string): boolean { + const activePreset = new RegExp(`^\\s*\\u25cf\\s+${escapeRegex(preset)}\\b`, "im"); + return activePreset.test(stripAnsi(output)); +} + +async function expectPolicyPreset( + host: HostCliClient, + preset: string, + expected: "applied" | "not-applied", + artifactName: string, +): Promise { + const result = await host.nemoclaw([SANDBOX_NAME, "policy-list"], { + artifactName, + env: baseEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(result, `nemoclaw ${SANDBOX_NAME} policy-list`); + const applied = policyListHasActivePreset(resultText(result), preset); + expect(applied, `${preset} preset expected ${expected}:\n${resultText(result)}`).toBe( + expected === "applied", + ); +} + +async function telegramEgressProbe( + sandbox: SandboxClient, + artifactName: string, +): Promise<{ result: ShellProbeResult; status: EgressProbeStatus }> { + const source = [ + `const url = ${JSON.stringify(`https://api.telegram.org/bot${TELEGRAM_TOKEN}/getMe`)};`, + "fetch(url, { signal: AbortSignal.timeout(15000) })", + " .then((response) => console.log(`STATUS_${response.status}`))", + " .catch((error) => console.log(`ERROR_${error.cause?.code || error.code || error.message}`));", + ].join(" "); + const result = await sandbox.exec(SANDBOX_NAME, ["node", "-e", source], { + artifactName, + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(result, "telegram egress probe"); + const output = resultText(result); + if (/STATUS_[24][0-9][0-9]/.test(output)) return { result, status: "open" }; + if (/policy_denied|engine:ssrf|forbidden by policy|CONNECT.*40[0-9]/i.test(output)) { + return { result, status: "denied" }; + } + return { result, status: "inconclusive" }; +} + +const liveTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +liveTest( + "channels add/remove telegram updates registry, gateway, policy, and sandbox state", + testTimeoutOptions(TEST_TIMEOUT_MS), + async ({ artifacts, cleanup, environment, host, lifecycle, onboard, sandbox, secrets, skip }) => { + if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { + throw new Error( + `channels-add-remove live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, + ); + } + const apiKey = secrets.required("NVIDIA_API_KEY"); + const secretsToRedact = redactionValues(apiKey); + + const ready = await environment.assertReady({ + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + }); + + await artifacts.writeJson("scenario.json", { + id: "channels-add-remove", + legacySource: "test/e2e/test-channels-add-remove.sh", + runner: "vitest", + sandboxName: SANDBOX_NAME, + contract: [ + "onboard creates an OpenClaw sandbox with no Telegram channel", + "channels add telegram registers the bridge and persists messaging.plan", + "post-add rebuild reuses the gateway-stored inference credential when NVIDIA_API_KEY is absent", + "post-add rebuild applies the Telegram policy preset and renders openclaw.json channel state", + "channels remove telegram removes provider, policy, registry plan, and rendered channel state after rebuild", + ], + }); + + cleanup.add(`destroy ${SANDBOX_NAME} after channels add/remove live test`, async () => { + await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "cleanup-nemoclaw-destroy")); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete", + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + await bestEffort(() => + host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy", + env: baseEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + }); + + await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy")); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-sandbox-delete", + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + await bestEffort(() => + host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "pre-cleanup-openshell-gateway-destroy", + env: baseEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + + let instance; + try { + instance = await onboard.from(ready, { + sandboxName: SANDBOX_NAME, + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + } catch (error) { + if (isEndpointRateLimited(error)) { + await artifacts.writeText("endpoint-rate-limit-skip.txt", errorText(error)); + skip( + "NVIDIA endpoint validation was unavailable/rate-limited before the channels add/remove contract could run", + ); + } + throw error; + } + await expectSandboxReady(sandbox, "phase-1-sandbox-ready-after-onboard"); + + await expectProvider(host, "absent", "phase-2-provider-get-baseline"); + await expectOpenClawTelegram(sandbox, false, "phase-2-openclaw-json-baseline"); + await expectPolicyPreset(host, "telegram", "not-applied", "phase-2-policy-list-baseline"); + + const add = await host.nemoclaw([SANDBOX_NAME, "channels", "add", "telegram"], { + artifactName: "phase-3-channels-add-telegram", + env: channelEnv(), + redactionValues: secretsToRedact, + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(add, `nemoclaw ${SANDBOX_NAME} channels add telegram`); + expect(resultText(add)).toContain("Registered telegram"); + expectHostTelegramConfig("after channels add"); + expectHostTelegramPlan("active", "after channels add"); + + const rebuildAdd = await host.nemoclaw([SANDBOX_NAME, "rebuild", "--yes"], { + artifactName: "phase-3-rebuild-after-add-without-host-nvidia-key", + env: channelEnv(), + redactionValues: secretsToRedact, + timeoutMs: REBUILD_TIMEOUT_MS, + }); + expect(resultText(rebuildAdd)).not.toContain("provider credential not found"); + assertExitZero(rebuildAdd, `nemoclaw ${SANDBOX_NAME} rebuild --yes after add`); + await lifecycle.assertSandboxReadyAfterRebuild(instance, { + artifactNamePrefix: "phase-3-sandbox-ready-after-add-rebuild", + env: sandboxAccessEnv(), + attempts: 12, + delayMs: 5_000, + }); + + await expectPolicyPreset(host, "telegram", "applied", "phase-4-policy-list-after-add"); + await expectOpenClawTelegram(sandbox, true, "phase-4-openclaw-json-after-add"); + await expectProvider(host, "present", "phase-4-provider-get-after-add"); + expectHostTelegramConfig("after add+rebuild"); + expectHostTelegramPlan("active", "after add+rebuild"); + + const egress = await telegramEgressProbe(sandbox, "phase-4-telegram-egress-probe"); + if (egress.status === "denied") { + throw new Error(`egress to api.telegram.org was blocked:\n${resultText(egress.result)}`); + } + if (egress.status === "inconclusive") { + await artifacts.writeText( + "phase-4-telegram-egress-inconclusive.txt", + `Telegram egress probe was inconclusive; preserving the legacy soft-skip behavior.\n\n${resultText( + egress.result, + )}`, + ); + } + + const remove = await host.nemoclaw([SANDBOX_NAME, "channels", "remove", "telegram"], { + artifactName: "phase-5-channels-remove-telegram", + env: channelEnv({ NVIDIA_API_KEY: apiKey }), + redactionValues: secretsToRedact, + timeoutMs: COMMAND_TIMEOUT_MS, + }); + assertExitZero(remove, `nemoclaw ${SANDBOX_NAME} channels remove telegram`); + expect(resultText(remove)).toContain("Removed telegram"); + expectHostTelegramPlan("removed", "after channels remove"); + + const rebuildRemove = await host.nemoclaw([SANDBOX_NAME, "rebuild", "--yes"], { + artifactName: "phase-5-rebuild-after-remove", + env: channelEnv({ NVIDIA_API_KEY: apiKey }), + redactionValues: secretsToRedact, + timeoutMs: REBUILD_TIMEOUT_MS, + }); + assertExitZero(rebuildRemove, `nemoclaw ${SANDBOX_NAME} rebuild --yes after remove`); + await lifecycle.assertSandboxReadyAfterRebuild(instance, { + artifactNamePrefix: "phase-5-sandbox-ready-after-remove-rebuild", + env: sandboxAccessEnv(), + attempts: 12, + delayMs: 5_000, + }); + + await expectOpenClawTelegram(sandbox, false, "phase-6-openclaw-json-after-remove"); + await expectProvider(host, "absent", "phase-6-provider-get-after-remove"); + await expectPolicyPreset(host, "telegram", "not-applied", "phase-6-policy-list-after-remove"); + expectHostTelegramPlan("removed", "after remove+rebuild"); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 5b98d71aea..e6a870599f 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -387,6 +387,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["bedrock-runtime-compatible-anthropic-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "channels-add-remove" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["channels-add-remove-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "channels-add-remove-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["channels-add-remove-vitest"], + registryScenarios: [], + }); }); it("derives the free-standing inventory from workflow job metadata", () => { diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index eaae514130..6f25f04bca 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -17,7 +17,12 @@ export const FREE_STANDING_WORKFLOW_INVENTORY_SCRIPT = "tools/e2e-scenarios/free-standing-workflow-inventory.mts"; type WorkflowRecord = Record; -type WorkflowStep = WorkflowRecord & { name?: string; run?: string; uses?: string; with?: WorkflowRecord }; +type WorkflowStep = WorkflowRecord & { + name?: string; + run?: string; + uses?: string; + with?: WorkflowRecord; +}; export interface FreeStandingJobsInventory { allowedJobs: string[]; @@ -76,7 +81,9 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { errors.push(`free-standing workflow metadata contains invalid job id: ${jobId}`); } if (!hasJobMarker) { - errors.push(`${jobId} job ${FREE_STANDING_SCENARIO_MARKER} requires ${FREE_STANDING_JOB_MARKER}`); + errors.push( + `${jobId} job ${FREE_STANDING_SCENARIO_MARKER} requires ${FREE_STANDING_JOB_MARKER}`, + ); continue; } if (env[FREE_STANDING_JOB_MARKER] !== "1") { @@ -260,7 +267,11 @@ function requireInput(errors: string[], inputs: WorkflowRecord, name: string): v if (!Object.hasOwn(inputs, name)) errors.push(`workflow_dispatch missing input: ${name}`); } -function requireStep(errors: string[], steps: readonly WorkflowStep[], name: string): WorkflowStep | undefined { +function requireStep( + errors: string[], + steps: readonly WorkflowStep[], + name: string, +): WorkflowStep | undefined { const step = namedStep(steps, name); if (!step) errors.push(`run-scenario job missing step: ${name}`); return step; @@ -277,14 +288,22 @@ function requireJobStep( return step; } -function requireRunContains(errors: string[], step: WorkflowStep | undefined, expected: string): void { +function requireRunContains( + errors: string[], + step: WorkflowStep | undefined, + expected: string, +): void { if (!step) return; if (!stringValue(step.run).includes(expected)) { errors.push(`step '${step.name ?? ""}' run script must include ${expected}`); } } -function requireRunDoesNotContain(errors: string[], step: WorkflowStep | undefined, forbidden: string): void { +function requireRunDoesNotContain( + errors: string[], + step: WorkflowStep | undefined, + forbidden: string, +): void { if (!step) return; if (stringValue(step.run).includes(forbidden)) { errors.push(`step '${step.name ?? ""}' run script must not include ${forbidden}`); @@ -310,7 +329,8 @@ function requireEnvDoesNotExposeSecret( function requireWorkflowDispatch(errors: string[], triggers: WorkflowRecord): WorkflowRecord { const workflowDispatch = asRecord(triggers.workflow_dispatch); - if (Object.keys(workflowDispatch).length === 0) errors.push("workflow must support workflow_dispatch"); + if (Object.keys(workflowDispatch).length === 0) + errors.push("workflow must support workflow_dispatch"); return workflowDispatch; } @@ -320,7 +340,11 @@ function rejectAutomaticTriggers(errors: string[], triggers: WorkflowRecord): vo } } -function requireFullShaAction(errors: string[], step: WorkflowStep | undefined, description: string): void { +function requireFullShaAction( + errors: string[], + step: WorkflowStep | undefined, + description: string, +): void { if (!step) return; if (!/@[0-9a-f]{40}$/i.test(stringValue(step.uses))) { errors.push(`${description} action must be pinned to a full commit SHA`); @@ -389,11 +413,7 @@ function validateFreeStandingInventoryBoundary( requireNoDispatchInputInterpolation(errors, steps); for (const step of steps) { if (step.uses) { - requireFullShaAction( - errors, - step, - `${jobName} step '${step.name ?? step.uses}'`, - ); + requireFullShaAction(errors, step, `${jobName} step '${step.name ?? step.uses}'`); } if (/\$\{\{\s*secrets\./.test(stringValue(step.run))) { errors.push( @@ -428,9 +448,12 @@ function validateFreeStandingInventoryCoverage( const jobIf = stringValue(job.if); const mappingIsRepresented = jobIf.includes(`,${scenario},`) || - (jobId === "hermes-e2e-vitest" && jobIf.includes("needs.generate-matrix.outputs.hermes_selected")); + (jobId === "hermes-e2e-vitest" && + jobIf.includes("needs.generate-matrix.outputs.hermes_selected")); if (!mappingIsRepresented) { - errors.push(`free-standing inventory mapping ${scenario}:${jobId} must match the workflow job selector`); + errors.push( + `free-standing inventory mapping ${scenario}:${jobId} must match the workflow job selector`, + ); } } } @@ -453,14 +476,18 @@ function validateOpenShellVersionPinVitestJob(errors: string[], jobs: WorkflowRe errors.push("openshell-version-pin-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } if ( - jobEnv.E2E_ARTIFACT_DIR !== - "${{ github.workspace }}/e2e-artifacts/vitest/openshell-version-pin" + jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/openshell-version-pin" ) { errors.push( "openshell-version-pin-vitest job must write artifacts under e2e-artifacts/vitest/openshell-version-pin", ); } - requireEnvDoesNotExposeSecret(errors, "openshell-version-pin-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret( + errors, + "openshell-version-pin-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); const steps = asSteps(job.steps); requireNoDispatchInputInterpolation(errors, steps); @@ -505,17 +532,20 @@ function validateOpenShellVersionPinVitestJob(errors: string[], jobs: WorkflowRe const uploadPath = stringValue(uploadWith.path); requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/openshell-version-pin/"); if (uploadWith["include-hidden-files"] !== false) { - errors.push("openshell-version-pin-vitest artifact upload must set include-hidden-files: false"); + errors.push( + "openshell-version-pin-vitest artifact upload must set include-hidden-files: false", + ); } if (uploadWith["if-no-files-found"] !== "ignore") { - errors.push("openshell-version-pin-vitest artifact upload must ignore missing fixture artifacts"); + errors.push( + "openshell-version-pin-vitest artifact upload must ignore missing fixture artifacts", + ); } if (uploadWith["retention-days"] !== 14) { errors.push("openshell-version-pin-vitest artifact upload retention-days must be 14"); } } - function validateSkillAgentVitestJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "skill-agent-vitest"; const job = asRecord(jobs[jobName]); @@ -534,12 +564,19 @@ function validateSkillAgentVitestJob(errors: string[], jobs: WorkflowRecord): vo errors.push("skill-agent-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/skill-agent") { - errors.push("skill-agent-vitest job must write artifacts under e2e-artifacts/vitest/skill-agent"); + errors.push( + "skill-agent-vitest job must write artifacts under e2e-artifacts/vitest/skill-agent", + ); } if (!stringValue(jobEnv.NEMOCLAW_CLI_BIN).includes("bin/nemoclaw.js")) { errors.push("skill-agent-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); } - requireEnvDoesNotExposeSecret(errors, "skill-agent-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret( + errors, + "skill-agent-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); const steps = asSteps(job.steps); requireNoDispatchInputInterpolation(errors, steps); @@ -565,7 +602,12 @@ function validateSkillAgentVitestJob(errors: string[], jobs: WorkflowRecord): vo if (!setupNode) errors.push("skill-agent-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "skill-agent-vitest setup-node"); - const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); @@ -579,7 +621,11 @@ function validateSkillAgentVitestJob(errors: string[], jobs: WorkflowRecord): vo if (runEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { errors.push("skill-agent-vitest run step must receive NVIDIA_INFERENCE_API_KEY from secrets"); } - requireRunContains(errors, runVitest, 'export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"'); + requireRunContains( + errors, + runVitest, + 'export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"', + ); requireRunContains(errors, runVitest, 'OPENSHELL_BIN="$(command -v openshell)"'); requireRunContains(errors, runVitest, "export OPENSHELL_BIN"); requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); @@ -606,7 +652,9 @@ function validateSkillAgentVitestJob(errors: string[], jobs: WorkflowRecord): vo } for (const line of uploadPath.split("\n")) { if (line.trim() === "e2e-artifacts/vitest/skill-agent/") { - errors.push("skill-agent-vitest artifact upload path must not list the whole skill-agent artifact directory"); + errors.push( + "skill-agent-vitest artifact upload path must not list the whole skill-agent artifact directory", + ); } } if (uploadWith["include-hidden-files"] !== false) { @@ -634,7 +682,9 @@ function validateNetworkPolicyVitestJob(errors: string[], jobs: WorkflowRecord): errors.push("network-policy-vitest job must depend on generate-matrix"); } if (job.if !== freeStandingJobIf(jobName, "network-policy")) { - errors.push("network-policy-vitest job must map scenarios=network-policy to the network-policy job"); + errors.push( + "network-policy-vitest job must map scenarios=network-policy to the network-policy job", + ); } const jobEnv = asRecord(job.env); @@ -652,7 +702,12 @@ function validateNetworkPolicyVitestJob(errors: string[], jobs: WorkflowRecord): if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { errors.push("network-policy-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); } - for (const secret of ["NVIDIA_INFERENCE_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + for (const secret of [ + "NVIDIA_INFERENCE_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { requireEnvDoesNotExposeSecret(errors, "network-policy-vitest job", jobEnv, secret); } @@ -728,7 +783,9 @@ function validateNetworkPolicyVitestJob(errors: string[], jobs: WorkflowRecord): const runVitest = requireJobStep(errors, jobName, steps, "Run network-policy live test"); const runVitestEnv = asRecord(runVitest?.env); if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { - errors.push("network-policy-vitest Vitest step must receive NVIDIA_INFERENCE_API_KEY from secrets"); + errors.push( + "network-policy-vitest Vitest step must receive NVIDIA_INFERENCE_API_KEY from secrets", + ); } requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); requireRunContains(errors, runVitest, "test/e2e-scenario/live/network-policy.test.ts"); @@ -773,8 +830,7 @@ function validateCommonEgressAgentVitestJob(errors: string[], jobs: WorkflowReco errors.push("common-egress-agent-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } if ( - jobEnv.E2E_ARTIFACT_DIR !== - "${{ github.workspace }}/e2e-artifacts/vitest/common-egress-agent" + jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/common-egress-agent" ) { errors.push( "common-egress-agent-vitest job must write artifacts under e2e-artifacts/vitest/common-egress-agent", @@ -795,7 +851,12 @@ function validateCommonEgressAgentVitestJob(errors: string[], jobs: WorkflowReco if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { errors.push("common-egress-agent-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); } - for (const secret of ["NVIDIA_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + for (const secret of [ + "NVIDIA_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { requireEnvDoesNotExposeSecret(errors, "common-egress-agent-vitest job", jobEnv, secret); } @@ -900,7 +961,9 @@ function validateShieldsConfigVitestJob(errors: string[], jobs: WorkflowRecord): errors.push("shields-config-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/shields-config") { - errors.push("shields-config-vitest job must write artifacts under e2e-artifacts/vitest/shields-config"); + errors.push( + "shields-config-vitest job must write artifacts under e2e-artifacts/vitest/shields-config", + ); } if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { errors.push("shields-config-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); @@ -914,7 +977,12 @@ function validateShieldsConfigVitestJob(errors: string[], jobs: WorkflowRecord): if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-shields") { errors.push("shields-config-vitest job must set NEMOCLAW_SANDBOX_NAME=e2e-shields"); } - requireEnvDoesNotExposeSecret(errors, "shields-config-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret( + errors, + "shields-config-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); requireEnvDoesNotExposeSecret(errors, "shields-config-vitest job", jobEnv, "DOCKERHUB_USERNAME"); requireEnvDoesNotExposeSecret(errors, "shields-config-vitest job", jobEnv, "DOCKERHUB_TOKEN"); @@ -957,7 +1025,9 @@ function validateShieldsConfigVitestJob(errors: string[], jobs: WorkflowRecord): const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); const dockerHubEnv = asRecord(dockerHubAuth?.env); if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { - errors.push("shields-config-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets"); + errors.push( + "shields-config-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); } if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { errors.push("shields-config-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); @@ -968,7 +1038,12 @@ function validateShieldsConfigVitestJob(errors: string[], jobs: WorkflowRecord): if (!setupNode) errors.push("shields-config-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "shields-config-vitest setup-node"); - const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); const runVitest = requireJobStep(errors, jobName, steps, "Run shields-config live test"); @@ -1019,12 +1094,19 @@ function validateRebuildOpenClawVitestJob(errors: string[], jobs: WorkflowRecord errors.push("rebuild-openclaw-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/rebuild-openclaw") { - errors.push("rebuild-openclaw-vitest job must write artifacts under e2e-artifacts/vitest/rebuild-openclaw"); + errors.push( + "rebuild-openclaw-vitest job must write artifacts under e2e-artifacts/vitest/rebuild-openclaw", + ); } if (!stringValue(jobEnv.NEMOCLAW_CLI_BIN).includes("bin/nemoclaw.js")) { errors.push("rebuild-openclaw-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); } - requireEnvDoesNotExposeSecret(errors, "rebuild-openclaw-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret( + errors, + "rebuild-openclaw-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); const steps = asSteps(job.steps); requireNoDispatchInputInterpolation(errors, steps); @@ -1049,10 +1131,14 @@ function validateRebuildOpenClawVitestJob(errors: string[], jobs: WorkflowRecord const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); const dockerHubEnv = asRecord(dockerHubAuth?.env); if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { - errors.push("rebuild-openclaw-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets"); + errors.push( + "rebuild-openclaw-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); } if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { - errors.push("rebuild-openclaw-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); + errors.push( + "rebuild-openclaw-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); } requireRunContains(errors, dockerHubAuth, "docker login docker.io"); @@ -1060,14 +1146,24 @@ function validateRebuildOpenClawVitestJob(errors: string[], jobs: WorkflowRecord if (!setupNode) errors.push("rebuild-openclaw-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "rebuild-openclaw-vitest setup-node"); - const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); requireRunContains(errors, buildCli, "npm run build:cli"); const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); - requireEnvDoesNotExposeSecret(errors, "rebuild-openclaw-vitest step 'Install OpenShell'", asRecord(installOpenShell?.env), "GITHUB_TOKEN"); + requireEnvDoesNotExposeSecret( + errors, + "rebuild-openclaw-vitest step 'Install OpenShell'", + asRecord(installOpenShell?.env), + "GITHUB_TOKEN", + ); requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); @@ -1124,7 +1220,9 @@ function validateSandboxRebuildVitestJob(errors: string[], jobs: WorkflowRecord) errors.push("sandbox-rebuild-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/sandbox-rebuild") { - errors.push("sandbox-rebuild-vitest job must write artifacts under e2e-artifacts/vitest/sandbox-rebuild"); + errors.push( + "sandbox-rebuild-vitest job must write artifacts under e2e-artifacts/vitest/sandbox-rebuild", + ); } if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { errors.push("sandbox-rebuild-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); @@ -1132,7 +1230,12 @@ function validateSandboxRebuildVitestJob(errors: string[], jobs: WorkflowRecord) if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { errors.push("sandbox-rebuild-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); } - for (const secret of ["NVIDIA_INFERENCE_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + for (const secret of [ + "NVIDIA_INFERENCE_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { requireEnvDoesNotExposeSecret(errors, "sandbox-rebuild-vitest job", jobEnv, secret); } @@ -1162,7 +1265,9 @@ function validateSandboxRebuildVitestJob(errors: string[], jobs: WorkflowRecord) const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); const dockerHubEnv = asRecord(dockerHubAuth?.env); if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { - errors.push("sandbox-rebuild-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets"); + errors.push( + "sandbox-rebuild-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); } if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { errors.push("sandbox-rebuild-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); @@ -1174,7 +1279,12 @@ function validateSandboxRebuildVitestJob(errors: string[], jobs: WorkflowRecord) if (!setupNode) errors.push("sandbox-rebuild-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "sandbox-rebuild-vitest setup-node"); - const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); @@ -1237,8 +1347,7 @@ function validateStateBackupRestoreVitestJob(errors: string[], jobs: WorkflowRec errors.push("state-backup-restore-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } if ( - jobEnv.E2E_ARTIFACT_DIR !== - "${{ github.workspace }}/e2e-artifacts/vitest/state-backup-restore" + jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/state-backup-restore" ) { errors.push( "state-backup-restore-vitest job must write artifacts under e2e-artifacts/vitest/state-backup-restore", @@ -1259,7 +1368,12 @@ function validateStateBackupRestoreVitestJob(errors: string[], jobs: WorkflowRec if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-state-backup") { errors.push("state-backup-restore-vitest job must set NEMOCLAW_SANDBOX_NAME=e2e-state-backup"); } - for (const secret of ["NVIDIA_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + for (const secret of [ + "NVIDIA_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { requireEnvDoesNotExposeSecret(errors, "state-backup-restore-vitest job", jobEnv, secret); } @@ -1294,7 +1408,9 @@ function validateStateBackupRestoreVitestJob(errors: string[], jobs: WorkflowRec ); } if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { - errors.push("state-backup-restore-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); + errors.push( + "state-backup-restore-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); } requireRunContains(errors, dockerHubAuth, "docker login docker.io"); requireRunContains(errors, dockerHubAuth, "continuing with anonymous pulls"); @@ -1303,7 +1419,12 @@ function validateStateBackupRestoreVitestJob(errors: string[], jobs: WorkflowRec if (!setupNode) errors.push("state-backup-restore-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "state-backup-restore-vitest setup-node"); - const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); @@ -1338,7 +1459,9 @@ function validateStateBackupRestoreVitestJob(errors: string[], jobs: WorkflowRec errors.push("state-backup-restore-vitest artifact upload must set include-hidden-files: false"); } if (uploadWith["if-no-files-found"] !== "ignore") { - errors.push("state-backup-restore-vitest artifact upload must ignore missing fixture artifacts"); + errors.push( + "state-backup-restore-vitest artifact upload must ignore missing fixture artifacts", + ); } if (uploadWith["retention-days"] !== 14) { errors.push("state-backup-restore-vitest artifact upload retention-days must be 14"); @@ -1365,12 +1488,19 @@ function validateTokenRotationVitestJob(errors: string[], jobs: WorkflowRecord): errors.push("token-rotation-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/token-rotation") { - errors.push("token-rotation-vitest job must write artifacts under e2e-artifacts/vitest/token-rotation"); + errors.push( + "token-rotation-vitest job must write artifacts under e2e-artifacts/vitest/token-rotation", + ); } if (!stringValue(jobEnv.NEMOCLAW_CLI_BIN).includes("bin/nemoclaw.js")) { errors.push("token-rotation-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); } - requireEnvDoesNotExposeSecret(errors, "token-rotation-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret( + errors, + "token-rotation-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); const steps = asSteps(job.steps); requireNoDispatchInputInterpolation(errors, steps); @@ -1395,7 +1525,9 @@ function validateTokenRotationVitestJob(errors: string[], jobs: WorkflowRecord): const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); const dockerHubEnv = asRecord(dockerHubAuth?.env); if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { - errors.push("token-rotation-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets"); + errors.push( + "token-rotation-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); } if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { errors.push("token-rotation-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); @@ -1406,7 +1538,12 @@ function validateTokenRotationVitestJob(errors: string[], jobs: WorkflowRecord): if (!setupNode) errors.push("token-rotation-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "token-rotation-vitest setup-node"); - const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); @@ -1489,7 +1626,12 @@ function validateOnboardNegativePathsVitestJob(errors: string[], jobs: WorkflowR "onboard-negative-paths-vitest job must write artifacts under e2e-artifacts/vitest/onboard-negative-paths", ); } - requireEnvDoesNotExposeSecret(errors, "onboard-negative-paths-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret( + errors, + "onboard-negative-paths-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); const steps = asSteps(job.steps); requireNoDispatchInputInterpolation(errors, steps); @@ -1537,10 +1679,14 @@ function validateOnboardNegativePathsVitestJob(errors: string[], jobs: WorkflowR const uploadPath = stringValue(uploadWith.path); requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/onboard-negative-paths/"); if (uploadWith["include-hidden-files"] !== false) { - errors.push("onboard-negative-paths-vitest artifact upload must set include-hidden-files: false"); + errors.push( + "onboard-negative-paths-vitest artifact upload must set include-hidden-files: false", + ); } if (uploadWith["if-no-files-found"] !== "ignore") { - errors.push("onboard-negative-paths-vitest artifact upload must ignore missing fixture artifacts"); + errors.push( + "onboard-negative-paths-vitest artifact upload must ignore missing fixture artifacts", + ); } if (uploadWith["retention-days"] !== 14) { errors.push("onboard-negative-paths-vitest artifact upload retention-days must be 14"); @@ -1564,10 +1710,7 @@ function validateCloudInferenceVitestJob(errors: string[], jobs: WorkflowRecord) } const jobEnv = asRecord(job.env); - if ( - jobEnv.E2E_ARTIFACT_DIR !== - "${{ github.workspace }}/e2e-artifacts/vitest/cloud-inference" - ) { + if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/cloud-inference") { errors.push( "cloud-inference-vitest job must write artifacts under e2e-artifacts/vitest/cloud-inference", ); @@ -1648,11 +1791,7 @@ function validateCloudInferenceVitestJob(errors: string[], jobs: WorkflowRecord) } } -function requireNoDockerHubAuthInRun( - errors: string[], - owner: string, - runScript: string, -): void { +function requireNoDockerHubAuthInRun(errors: string[], owner: string, runScript: string): void { if (!runScript) return; const usesDockerLogin = /\bdocker\s+login\b/i.test(runScript); const referencesSecret = /\bsecrets\.[A-Za-z0-9_]+\b|\$\{\{\s*secrets\.[^}]+\}\}/.test(runScript); @@ -1681,15 +1820,17 @@ function validateDoubleOnboardVitestJob(errors: string[], jobs: WorkflowRecord): if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { errors.push("double-onboard-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); } - if ( - jobEnv.E2E_ARTIFACT_DIR !== - "${{ github.workspace }}/e2e-artifacts/vitest/double-onboard" - ) { + if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/double-onboard") { errors.push( "double-onboard-vitest job must write artifacts under e2e-artifacts/vitest/double-onboard", ); } - requireEnvDoesNotExposeSecret(errors, "double-onboard-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret( + errors, + "double-onboard-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); requireEnvDoesNotExposeSecret(errors, "double-onboard-vitest job", jobEnv, "DOCKERHUB_TOKEN"); const steps = asSteps(job.steps); @@ -1721,7 +1862,9 @@ function validateDoubleOnboardVitestJob(errors: string[], jobs: WorkflowRecord): const dockerLogin = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); const dockerLoginEnv = asRecord(dockerLogin?.env); if (dockerLoginEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { - errors.push("double-onboard-vitest Docker login step must read DOCKERHUB_USERNAME from secrets"); + errors.push( + "double-onboard-vitest Docker login step must read DOCKERHUB_USERNAME from secrets", + ); } if (dockerLoginEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { errors.push("double-onboard-vitest Docker login step must read DOCKERHUB_TOKEN from secrets"); @@ -1769,7 +1912,8 @@ function validateDoubleOnboardVitestJob(errors: string[], jobs: WorkflowRecord): if (uploadWith["retention-days"] !== 14) { errors.push("double-onboard-vitest artifact upload retention-days must be 14"); } -}function validateRuntimeOverridesVitestJob(errors: string[], jobs: WorkflowRecord): void { +} +function validateRuntimeOverridesVitestJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "runtime-overrides-vitest"; const job = asRecord(jobs[jobName]); if (Object.keys(job).length === 0) { @@ -1786,11 +1930,25 @@ function validateDoubleOnboardVitestJob(errors: string[], jobs: WorkflowRecord): if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { errors.push("runtime-overrides-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); } - if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/runtime-overrides") { - errors.push("runtime-overrides-vitest job must write artifacts under e2e-artifacts/vitest/runtime-overrides"); + if ( + jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/runtime-overrides" + ) { + errors.push( + "runtime-overrides-vitest job must write artifacts under e2e-artifacts/vitest/runtime-overrides", + ); } - requireEnvDoesNotExposeSecret(errors, "runtime-overrides-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); - requireEnvDoesNotExposeSecret(errors, "runtime-overrides-vitest job", jobEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret( + errors, + "runtime-overrides-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); + requireEnvDoesNotExposeSecret( + errors, + "runtime-overrides-vitest job", + jobEnv, + "DOCKERHUB_USERNAME", + ); requireEnvDoesNotExposeSecret(errors, "runtime-overrides-vitest job", jobEnv, "DOCKERHUB_TOKEN"); const steps = asSteps(job.steps); @@ -1815,7 +1973,12 @@ function validateDoubleOnboardVitestJob(errors: string[], jobs: WorkflowRecord): if (!setupNode) errors.push("runtime-overrides-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "runtime-overrides-vitest setup-node"); - const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); const runVitest = requireJobStep(errors, jobName, steps, "Run runtime overrides live test"); @@ -1881,7 +2044,12 @@ function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): voi if (jobEnv.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS !== "60") { errors.push("hermes-e2e-vitest job must give hosted endpoint validation a CI-safe timeout"); } - requireEnvDoesNotExposeSecret(errors, "hermes-e2e-vitest job", jobEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret( + errors, + "hermes-e2e-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); const steps = asSteps(job.steps); requireNoDispatchInputInterpolation(errors, steps); @@ -1907,7 +2075,12 @@ function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): voi if (!setupNode) errors.push("hermes-e2e-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "hermes-e2e-vitest setup-node"); - const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); @@ -1941,10 +2114,7 @@ function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): voi } } -function validateHermesRootEntrypointSmokeVitestJob( - errors: string[], - jobs: WorkflowRecord, -): void { +function validateHermesRootEntrypointSmokeVitestJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "hermes-root-entrypoint-smoke-vitest"; const job = asRecord(jobs[jobName]); if (Object.keys(job).length === 0) { @@ -2040,10 +2210,11 @@ function validateHermesRootEntrypointSmokeVitestJob( if (!checkout) errors.push("hermes-root-entrypoint-smoke-vitest job missing checkout step"); requireFullShaAction(errors, checkout, "hermes-root-entrypoint-smoke-vitest checkout"); if (asRecord(checkout?.with)["persist-credentials"] !== false) { - errors.push("hermes-root-entrypoint-smoke-vitest checkout step must set persist-credentials=false"); + errors.push( + "hermes-root-entrypoint-smoke-vitest checkout step must set persist-credentials=false", + ); } - const setupNode = namedStep(steps, "Set up Node"); if (!setupNode) errors.push("hermes-root-entrypoint-smoke-vitest job missing step: Set up Node"); requireFullShaAction(errors, setupNode, "hermes-root-entrypoint-smoke-vitest setup-node"); @@ -2148,7 +2319,12 @@ function validateModelRouterProviderRoutedInferenceVitestJob( "model-router-provider-routed-inference-vitest job must force OPENSHELL_GATEWAY=nemoclaw", ); } - for (const secret of ["NVIDIA_INFERENCE_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + for (const secret of [ + "NVIDIA_INFERENCE_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { requireEnvDoesNotExposeSecret( errors, "model-router-provider-routed-inference-vitest job", @@ -2219,7 +2395,11 @@ function validateModelRouterProviderRoutedInferenceVitestJob( if (!setupNode) { errors.push("model-router-provider-routed-inference-vitest job missing step: Set up Node"); } - requireFullShaAction(errors, setupNode, "model-router-provider-routed-inference-vitest setup-node"); + requireFullShaAction( + errors, + setupNode, + "model-router-provider-routed-inference-vitest setup-node", + ); const installRootDependencies = requireJobStep( errors, @@ -2264,7 +2444,9 @@ function validateModelRouterProviderRoutedInferenceVitestJob( ); const uploadWith = asRecord(upload?.with); if (uploadWith.name !== "e2e-vitest-scenarios-model-router-provider-routed-inference") { - errors.push("model-router-provider-routed-inference-vitest artifact upload name must be stable"); + errors.push( + "model-router-provider-routed-inference-vitest artifact upload name must be stable", + ); } const uploadPath = stringValue(uploadWith.path); requireUploadPathContains( @@ -2283,17 +2465,172 @@ function validateModelRouterProviderRoutedInferenceVitestJob( ); } if (uploadWith["retention-days"] !== 14) { - errors.push("model-router-provider-routed-inference-vitest artifact upload retention-days must be 14"); + errors.push( + "model-router-provider-routed-inference-vitest artifact upload retention-days must be 14", + ); } const cleanup = requireJobStep(errors, jobName, steps, "Clean up Docker auth"); if (cleanup?.if !== "always()") { - errors.push("model-router-provider-routed-inference-vitest Docker auth cleanup must always run"); + errors.push( + "model-router-provider-routed-inference-vitest Docker auth cleanup must always run", + ); } requireRunContains(errors, cleanup, "docker logout docker.io"); requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); } +function validateChannelsAddRemoveVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "channels-add-remove-vitest"; + const scenarioName = "channels-add-remove"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing channels-add-remove-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("channels-add-remove-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + if (job["timeout-minutes"] !== 75) { + errors.push("channels-add-remove-vitest job must keep the legacy 75 minute timeout"); + } + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/channels-add-remove" + ) { + errors.push( + "channels-add-remove-vitest job must write artifacts under e2e-artifacts/vitest/channels-add-remove", + ); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("channels-add-remove-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-channels-add-remove") { + errors.push( + "channels-add-remove-vitest job must set NEMOCLAW_SANDBOX_NAME=e2e-channels-add-remove", + ); + } + if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_NON_INTERACTIVE=1"); + } + if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { + errors.push("channels-add-remove-vitest job must set NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("channels-add-remove-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); + } + for (const secret of [ + "NVIDIA_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { + requireEnvDoesNotExposeSecret(errors, "channels-add-remove-vitest job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `channels-add-remove-vitest step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run channels add/remove live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("channels-add-remove-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "channels-add-remove-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("channels-add-remove-vitest checkout step must set persist-credentials=false"); + } + + const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerHubEnv = asRecord(dockerHubAuth?.env); + if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push( + "channels-add-remove-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); + } + if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push( + "channels-add-remove-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); + } + requireRunContains(errors, dockerHubAuth, "docker login docker.io"); + requireRunContains(errors, dockerHubAuth, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("channels-add-remove-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "channels-add-remove-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); + requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); + requireRunContains(errors, installOpenShell, "-u NVIDIA_API_KEY"); + requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run channels add/remove live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { + errors.push("channels-add-remove-vitest step must receive NVIDIA_API_KEY from secrets"); + } + if (runVitestEnv.TELEGRAM_BOT_TOKEN !== "test-fake-telegram-token-add-remove-e2e") { + errors.push("channels-add-remove-vitest step must set the fake Telegram token"); + } + if (runVitestEnv.TELEGRAM_ALLOWED_IDS !== "123456789") { + errors.push("channels-add-remove-vitest step must set TELEGRAM_ALLOWED_IDS"); + } + if (runVitestEnv.TELEGRAM_REQUIRE_MENTION !== "0") { + errors.push("channels-add-remove-vitest step must set TELEGRAM_REQUIRE_MENTION"); + } + requireRunContains(errors, runVitest, "OPENSHELL_BIN"); + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/channels-add-remove.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload channels add/remove artifacts"); + requireFullShaAction(errors, upload, "channels-add-remove-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-channels-add-remove") { + errors.push("channels-add-remove-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/channels-add-remove/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("channels-add-remove-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("channels-add-remove-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("channels-add-remove-vitest artifact upload retention-days must be 14"); + } +} + function validateBedrockRuntimeCompatibleAnthropicVitestJob( errors: string[], jobs: WorkflowRecord, @@ -2348,7 +2685,9 @@ function validateBedrockRuntimeCompatibleAnthropicVitestJob( ); } if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { - errors.push("bedrock-runtime-compatible-anthropic-vitest job must set NEMOCLAW_NON_INTERACTIVE=1"); + errors.push( + "bedrock-runtime-compatible-anthropic-vitest job must set NEMOCLAW_NON_INTERACTIVE=1", + ); } if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { errors.push( @@ -2356,7 +2695,9 @@ function validateBedrockRuntimeCompatibleAnthropicVitestJob( ); } if (jobEnv.NEMOCLAW_RECREATE_SANDBOX !== "1") { - errors.push("bedrock-runtime-compatible-anthropic-vitest job must set NEMOCLAW_RECREATE_SANDBOX=1"); + errors.push( + "bedrock-runtime-compatible-anthropic-vitest job must set NEMOCLAW_RECREATE_SANDBOX=1", + ); } if (jobEnv.NEMOCLAW_AGENT !== "${{ matrix.agent }}") { errors.push( @@ -2447,11 +2788,7 @@ function validateBedrockRuntimeCompatibleAnthropicVitestJob( if (!setupNode) { errors.push("bedrock-runtime-compatible-anthropic-vitest job missing step: Set up Node"); } - requireFullShaAction( - errors, - setupNode, - "bedrock-runtime-compatible-anthropic-vitest setup-node", - ); + requireFullShaAction(errors, setupNode, "bedrock-runtime-compatible-anthropic-vitest setup-node"); const installRootDependencies = requireJobStep( errors, @@ -2528,7 +2865,6 @@ function validateBedrockRuntimeCompatibleAnthropicVitestJob( requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); } - export function validateE2eVitestScenariosWorkflowBoundary( workflowPath = DEFAULT_VITEST_WORKFLOW_PATH, ): string[] { @@ -2568,7 +2904,9 @@ export function validateE2eVitestScenariosWorkflowBoundary( } const generateSteps = asSteps(generateMatrix.steps); requireNoDispatchInputInterpolation(errors, generateSteps); - const generateCheckout = generateSteps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + const generateCheckout = generateSteps.find((step) => + stringValue(step.uses).startsWith("actions/checkout@"), + ); if (!generateCheckout) errors.push("generate-matrix job missing checkout step"); requireFullShaAction(errors, generateCheckout, "generate-matrix checkout"); if (asRecord(generateCheckout?.with)["persist-credentials"] !== false) { @@ -2586,8 +2924,16 @@ export function validateE2eVitestScenariosWorkflowBoundary( errors.push("matrix generation step must pass scenarios through SCENARIOS env"); } requireRunContains(errors, generate, FREE_STANDING_WORKFLOW_INVENTORY_SCRIPT); - requireRunContains(errors, generate, "free-standing workflow inventory must be data-only key=value"); - requireRunContains(errors, generate, "free_standing_scenarios_csv must match scenario mapping keys"); + requireRunContains( + errors, + generate, + "free-standing workflow inventory must be data-only key=value", + ); + requireRunContains( + errors, + generate, + "free_standing_scenarios_csv must match scenario mapping keys", + ); requireRunContains(errors, generate, "Free-standing scenario maps to unknown job"); requireRunContains(errors, generate, "Use either scenarios or jobs, not both"); requireRunContains(errors, generate, "Unknown free-standing Vitest job"); @@ -2603,7 +2949,11 @@ export function validateE2eVitestScenariosWorkflowBoundary( requireRunDoesNotContain(errors, generate, "^[A-Za-z0-9._-]+"); requireRunContains(errors, generate, "hermes_selected=false"); requireRunContains(errors, generate, "hermes_selected=true"); - requireRunContains(errors, generate, 'echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT"'); + requireRunContains( + errors, + generate, + 'echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT"', + ); requireRunContains(errors, generate, "## Vitest E2E Scenario Matrix"); requireRunContains(errors, generate, "| Scenario | Runner | Label |"); @@ -2615,7 +2965,9 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (liveScenarios.needs !== "generate-matrix") { errors.push("live-scenarios job must depend on generate-matrix"); } - if (liveScenarios.if !== "${{ inputs.jobs == '' && needs.generate-matrix.outputs.matrix != '[]' }}") { + if ( + liveScenarios.if !== "${{ inputs.jobs == '' && needs.generate-matrix.outputs.matrix != '[]' }}" + ) { errors.push("live-scenarios job must not run when a free-standing jobs selector is supplied"); } const strategy = asRecord(liveScenarios.strategy); @@ -2690,7 +3042,11 @@ export function validateE2eVitestScenariosWorkflowBoundary( errors.push("summary step must pass matrix.label through SCENARIO_LABEL env"); } requireRunContains(errors, summary, "run-plan.json"); - requireRunContains(errors, summary, 'Path(os.environ["E2E_ARTIFACT_DIR"]) / os.environ["SCENARIO_ID"]'); + requireRunContains( + errors, + summary, + 'Path(os.environ["E2E_ARTIFACT_DIR"]) / os.environ["SCENARIO_ID"]', + ); requireRunContains(errors, summary, "| Scenario | Manifest | Expected state | Suites | Phases |"); requireRunContains(errors, summary, "SCENARIO_ID"); @@ -2701,8 +3057,16 @@ export function validateE2eVitestScenariosWorkflowBoundary( errors.push("artifact upload name must include matrix.id"); } const uploadPath = stringValue(uploadWith.path); - requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/${{ matrix.id }}/run-plan.json"); - requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/${{ matrix.id }}/scenario.json"); + requireUploadPathContains( + errors, + uploadPath, + "e2e-artifacts/vitest/${{ matrix.id }}/run-plan.json", + ); + requireUploadPathContains( + errors, + uploadPath, + "e2e-artifacts/vitest/${{ matrix.id }}/scenario.json", + ); requireUploadPathContains( errors, uploadPath, @@ -2745,7 +3109,12 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateOnboardNegativePathsVitestJob(errors, jobs); validateSkillAgentVitestJob(errors, jobs); validateFreeStandingJobSelector(errors, jobs, "credential-migration-vitest"); - validateFreeStandingJobSelector(errors, jobs, "sessions-agents-cli-vitest", "sessions-agents-cli"); + validateFreeStandingJobSelector( + errors, + jobs, + "sessions-agents-cli-vitest", + "sessions-agents-cli", + ); validateFreeStandingJobSelector(errors, jobs, "inference-routing-vitest", "inference-routing"); validateCloudInferenceVitestJob(errors, jobs); validateRuntimeOverridesVitestJob(errors, jobs); @@ -2773,12 +3142,24 @@ export function validateE2eVitestScenariosWorkflowBoundary( "issue-4434-tui-unreachable-inference", ); validateModelRouterProviderRoutedInferenceVitestJob(errors, jobs); - validateFreeStandingJobSelector(errors, jobs, "gateway-drift-preflight-vitest", "gateway-drift-preflight"); + validateFreeStandingJobSelector( + errors, + jobs, + "gateway-drift-preflight-vitest", + "gateway-drift-preflight", + ); - validateFreeStandingJobSelector(errors, jobs, "openclaw-inference-switch-vitest", "openclaw-inference-switch"); + validateFreeStandingJobSelector( + errors, + jobs, + "openclaw-inference-switch-vitest", + "openclaw-inference-switch", + ); validateBedrockRuntimeCompatibleAnthropicVitestJob(errors, jobs); + validateChannelsAddRemoveVitestJob(errors, jobs); + const reportToPr = asRecord(jobs["report-to-pr"]); if (Object.keys(reportToPr).length === 0) { errors.push("workflow missing report-to-pr job"); @@ -2789,7 +3170,12 @@ export function validateE2eVitestScenariosWorkflowBoundary( } validateFreeStandingInventoryCoverage(errors, jobs, needs, freeStandingInventory); const reportSteps = asSteps(reportToPr.steps); - const report = requireJobStep(errors, "report-to-pr", reportSteps, "Post Vitest scenario results to PR"); + const report = requireJobStep( + errors, + "report-to-pr", + reportSteps, + "Post Vitest scenario results to PR", + ); const reportEnv = asRecord(report?.env); if (reportEnv.JOBS !== "${{ inputs.jobs }}") { errors.push("report-to-pr step must pass jobs through JOBS env"); @@ -2802,25 +3188,39 @@ export function validateE2eVitestScenariosWorkflowBoundary( } const reportScript = stringValue(asRecord(report?.with).script ?? report?.run); if (!reportScript.includes("process.env.JOBS")) { - errors.push("step 'Post Vitest scenario results to PR' run script must include process.env.JOBS"); + errors.push( + "step 'Post Vitest scenario results to PR' run script must include process.env.JOBS", + ); } if (!reportScript.includes("process.env.JOB_SCENARIOS")) { - errors.push("step 'Post Vitest scenario results to PR' run script must include process.env.JOB_SCENARIOS"); + errors.push( + "step 'Post Vitest scenario results to PR' run script must include process.env.JOB_SCENARIOS", + ); } if (!reportScript.includes("selectorValidationPassed")) { - errors.push("step 'Post Vitest scenario results to PR' run script must check selector validation before echoing selectors"); + errors.push( + "step 'Post Vitest scenario results to PR' run script must check selector validation before echoing selectors", + ); } if (!reportScript.includes("jobsRejected")) { - errors.push("step 'Post Vitest scenario results to PR' run script must omit rejected job selectors"); + errors.push( + "step 'Post Vitest scenario results to PR' run script must omit rejected job selectors", + ); } if (!reportScript.includes("scenariosRejected")) { - errors.push("step 'Post Vitest scenario results to PR' run script must omit rejected scenario selectors"); + errors.push( + "step 'Post Vitest scenario results to PR' run script must omit rejected scenario selectors", + ); } if (!reportScript.includes("**Requested jobs:**")) { - errors.push("step 'Post Vitest scenario results to PR' run script must include **Requested jobs:**"); + errors.push( + "step 'Post Vitest scenario results to PR' run script must include **Requested jobs:**", + ); } if (!reportScript.includes("**Requested scenarios:**")) { - errors.push("step 'Post Vitest scenario results to PR' run script must include **Requested scenarios:**"); + errors.push( + "step 'Post Vitest scenario results to PR' run script must include **Requested scenarios:**", + ); } for (const forbidden of ["toJSON(inputs.pr_number)", "toJSON(inputs.scenarios)"]) { if (reportScript.includes(forbidden)) {