From 34c072af073e9a72d022286bb2a60b931e8a0f35 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 09:13:48 -0400 Subject: [PATCH 1/6] ci(e2e): use jobs selector only --- .github/workflows/e2e-vitest-scenarios.yaml | 139 +++-------- .../e2e-scenarios-workflow.test.ts | 218 ++---------------- tools/e2e-scenarios/workflow-boundary.mts | 184 ++++----------- 3 files changed, 97 insertions(+), 444 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 174b24ae26..cb380ea7ab 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -6,13 +6,8 @@ name: E2E / Vitest Scenarios on: workflow_dispatch: inputs: - scenarios: - description: "Optional comma-separated typed scenario ids. Empty runs all live Vitest-supported scenarios." - required: false - default: "" - type: string jobs: - description: "Optional comma-separated free-standing live Vitest job ids. Empty runs all jobs only when scenarios is also empty." + description: "Optional comma-separated free-standing live Vitest job ids. Empty runs all jobs and live Vitest-supported scenarios." required: false default: "" type: string @@ -26,7 +21,7 @@ permissions: contents: read concurrency: - group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }}-${{ inputs.jobs || 'all-jobs' }} + group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.jobs || 'all-jobs' }} cancel-in-progress: false jobs: @@ -37,18 +32,9 @@ jobs: name: Validate free-standing job selector env: JOBS: ${{ inputs.jobs }} - SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest" - if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then - echo "::error::Use either scenarios or jobs, not both." >&2 - exit 1 - fi - if [ -n "${SCENARIOS}" ] && [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then - echo "::error::Invalid scenario input; use comma-separated scenario ids containing only letters, numbers, underscores, and hyphens." >&2 - exit 1 - fi if [ -z "${JOBS}" ]; then echo "No free-standing job selector supplied; default workflow behavior applies." exit 0 @@ -71,7 +57,6 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.matrix.outputs.matrix }} - hermes_selected: ${{ steps.matrix.outputs.hermes_selected }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -90,29 +75,9 @@ jobs: name: Generate Vitest scenario matrix env: JOBS: ${{ inputs.jobs }} - SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest" - args=(--emit-live-matrix) - matrix="" - hermes_selected=false - registry_scenarios=() - free_standing_scenarios=(openshell-version-pin onboard-negative-paths inference-routing runtime-overrides hermes-e2e hermes-root-entrypoint-smoke network-policy rebuild-openclaw token-rotation openclaw-tui-chat-correlation double-onboard issue-4434-tui-unreachable-inference model-router-provider-routed-inference) - is_free_standing_scenario() { - local id="$1" - local known - for known in "${free_standing_scenarios[@]}"; do - if [[ "${id}" == "${known}" ]]; then - return 0 - fi - done - return 1 - } - if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then - echo "::error::Use either scenarios or jobs, not both." >&2 - exit 1 - fi if [ -n "${JOBS}" ]; then if [[ ! "${JOBS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then echo "::error::Invalid jobs input; use comma-separated job ids" >&2 @@ -125,43 +90,12 @@ jobs: echo "::error::Allowed jobs: ${allowed_jobs}" >&2 exit 1 fi - case "${job}" in - hermes-e2e-vitest) - hermes_selected=true - ;; - esac done matrix="[]" - elif [ -n "${SCENARIOS}" ]; then - if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then - echo "::error::Invalid scenario input; use comma-separated scenario ids containing only letters, numbers, underscores, and hyphens." >&2 - exit 1 - fi - IFS=',' read -r -a requested_scenarios <<< "${SCENARIOS}" - for scenario in "${requested_scenarios[@]}"; do - case "${scenario}" in - hermes-e2e) - hermes_selected=true - ;; - esac - if is_free_standing_scenario "${scenario}"; then - continue - fi - registry_scenarios+=("${scenario}") - done - if [ "${#registry_scenarios[@]}" -gt 0 ]; then - registry_csv="$(IFS=,; echo "${registry_scenarios[*]}")" - args+=(--scenarios "${registry_csv}") - matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" - else - matrix="[]" - fi else - hermes_selected=true - matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" + matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts --emit-live-matrix)" fi echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" - echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT" MATRIX_JSON="${matrix}" python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json import os @@ -278,8 +212,8 @@ jobs: # because the matrix above only runs registry-scenarios.test.ts. Modeled on # #5049's free-standing pattern. openshell-version-pin-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin-vitest,') || contains(format(',{0},', inputs.scenarios), ',openshell-version-pin,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',openshell-version-pin-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -319,8 +253,8 @@ jobs: retention-days: 14 onboard-negative-paths-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths-vitest,') || contains(format(',{0},', inputs.scenarios), ',onboard-negative-paths,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -364,8 +298,8 @@ jobs: retention-days: 14 hermes-root-entrypoint-smoke-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ needs.generate-matrix.result == 'success' && ((inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-vitest,') || contains(format(',{0},', inputs.scenarios), ',hermes-root-entrypoint-smoke,')) }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 45 env: @@ -406,8 +340,8 @@ jobs: retention-days: 14 inference-routing-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',inference-routing-vitest,') || contains(format(',{0},', inputs.scenarios), ',inference-routing,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',inference-routing-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 45 env: @@ -454,8 +388,8 @@ jobs: retention-days: 14 issue-4434-tui-unreachable-inference-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',issue-4434-tui-unreachable-inference-vitest,') || contains(format(',{0},', inputs.scenarios), ',issue-4434-tui-unreachable-inference,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',issue-4434-tui-unreachable-inference-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 120 env: @@ -553,7 +487,7 @@ jobs: credential-migration-vitest: needs: validate-jobs - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',credential-migration-vitest,') }} + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',credential-migration-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 50 env: @@ -630,8 +564,8 @@ jobs: runtime-overrides-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',runtime-overrides-vitest,') || contains(format(',{0},', inputs.scenarios), ',runtime-overrides,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',runtime-overrides-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 45 env: @@ -671,8 +605,8 @@ jobs: retention-days: 14 hermes-e2e-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',hermes-e2e-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 75 env: @@ -723,8 +657,8 @@ jobs: retention-days: 14 network-policy-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',network-policy-vitest,') || contains(format(',{0},', inputs.scenarios), ',network-policy,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',network-policy-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 90 env: @@ -785,8 +719,8 @@ jobs: retention-days: 14 rebuild-openclaw-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',rebuild-openclaw-vitest,') || contains(format(',{0},', inputs.scenarios), ',rebuild-openclaw,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',rebuild-openclaw-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 130 env: @@ -878,7 +812,7 @@ jobs: double-onboard-vitest: needs: validate-jobs - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',double-onboard-vitest,') }} + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',double-onboard-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 90 env: @@ -964,8 +898,8 @@ jobs: retention-days: 14 token-rotation-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') || contains(format(',{0},', inputs.scenarios), ',token-rotation,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 45 env: @@ -1048,7 +982,7 @@ jobs: launchable-smoke-vitest: needs: validate-jobs - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',launchable-smoke-vitest,') }} + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',launchable-smoke-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -1121,8 +1055,8 @@ jobs: # contract. The retained legacy bash lane remains the source for full # closeout until a later PR proves replacement and deletes it. model-router-provider-routed-inference-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',model-router-provider-routed-inference-vitest,') || contains(format(',{0},', inputs.scenarios), ',model-router-provider-routed-inference,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',model-router-provider-routed-inference-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 45 env: @@ -1211,8 +1145,8 @@ jobs: # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. openclaw-tui-chat-correlation-vitest: - needs: [validate-jobs, generate-matrix] - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',openclaw-tui-chat-correlation-vitest,') || contains(format(',{0},', inputs.scenarios), ',openclaw-tui-chat-correlation,') }} + needs: validate-jobs + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',openclaw-tui-chat-correlation-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 75 env: @@ -1292,7 +1226,7 @@ jobs: # until the #2701 fix lands; flips green afterwards. gateway-guard-recovery: needs: validate-jobs - if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',gateway-guard-recovery,') }} + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',gateway-guard-recovery,') }} runs-on: ubuntu-latest timeout-minutes: 45 env: @@ -1414,7 +1348,6 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: JOB_PR_NUMBER: ${{ inputs.pr_number }} - JOB_SCENARIOS: ${{ inputs.scenarios }} JOBS: ${{ inputs.jobs }} with: script: | @@ -1422,12 +1355,9 @@ jobs: const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const workflowBranch = context.ref.replace('refs/heads/', ''); const prNumberInput = process.env.JOB_PR_NUMBER || ''; - const rawRequestedScenarios = process.env.JOB_SCENARIOS || ''; const rawRequestedJobs = process.env.JOBS || ''; const selectorValidationPassed = needs['validate-jobs']?.result === 'success'; - const requestedScenarios = selectorValidationPassed ? rawRequestedScenarios : ''; const requestedJobs = selectorValidationPassed ? rawRequestedJobs : ''; - const scenariosRejected = rawRequestedScenarios && !selectorValidationPassed; const jobsRejected = rawRequestedJobs && !selectorValidationPassed; let prNumber = prNumberInput ? Number.parseInt(prNumberInput, 10) : undefined; @@ -1466,16 +1396,11 @@ jobs: '', `**Run:** [${context.runId}](${runUrl})`, `**Workflow ref:** \`${workflowBranch}\``, - scenariosRejected - ? '**Requested scenarios:** _(selector rejected by validate-jobs)_' - : requestedScenarios - ? `**Requested scenarios:** \`${requestedScenarios}\`` - : '**Requested scenarios:** _(default — all supported)_', jobsRejected ? '**Requested jobs:** _(selector rejected by validate-jobs)_' : requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` - : '**Requested jobs:** _(default — all free-standing when no scenarios are requested)_', + : '**Requested jobs:** _(default — all free-standing jobs and supported registry scenarios)_', `**Summary:** ${passed.length} passed, ${failed.length} failed, ${skipped.length} skipped`, '', '| Job | Result |', diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index c4cfe853e5..0e8aaf3328 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -22,10 +22,7 @@ function readWorkflow(): Record { ) as Record; } -function generateMatrixForDispatch(env: { - JOBS: string; - SCENARIOS: string; -}): Record { +function generateMatrixForDispatch(jobsSelector: string): Record { const workflow = readWorkflow(); const jobs = workflow.jobs as Record> }>; const generateStep = jobs["generate-matrix"]?.steps?.find( @@ -46,8 +43,7 @@ function generateMatrixForDispatch(env: { ...process.env, GITHUB_OUTPUT: outputPath, GITHUB_STEP_SUMMARY: summaryPath, - JOBS: env.JOBS, - SCENARIOS: env.SCENARIOS, + JOBS: jobsSelector, }, }); expect(result.signal).toBeNull(); @@ -70,50 +66,14 @@ describe("e2e-vitest-scenarios workflow boundary", () => { expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); }); - it("evaluates high-risk dispatch selector behavior before secret-bearing jobs run", () => { + it("evaluates jobs-only dispatch selector behavior before secret-bearing jobs run", () => { expect( - evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "network-policy,../escape" }), + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "network-policy,../escape" }), ).toMatchObject({ valid: false, liveScenariosRuns: false, selectedFreeStandingJobs: [], }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ - jobs: "network-policy-vitest", - scenarios: "network-policy", - }), - ).toMatchObject({ - valid: false, - liveScenariosRuns: false, - selectedFreeStandingJobs: [], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "network-policy" }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["network-policy-vitest"], - registryScenarios: [], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ - scenarios: "network-policy,ubuntu-repo-cloud-openclaw", - }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: true, - selectedFreeStandingJobs: ["network-policy-vitest"], - registryScenarios: ["ubuntu-repo-cloud-openclaw"], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "openshell-version-pin" }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["openshell-version-pin-vitest"], - registryScenarios: [], - }); expect( evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "runtime-overrides-vitest" }), ).toMatchObject({ @@ -122,22 +82,6 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["runtime-overrides-vitest"], registryScenarios: [], }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "runtime-overrides" }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["runtime-overrides-vitest"], - registryScenarios: [], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "inference-routing" }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["inference-routing-vitest"], - registryScenarios: [], - }); expect( evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "inference-routing-vitest" }), ).toMatchObject({ @@ -146,52 +90,9 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["inference-routing-vitest"], registryScenarios: [], }); - expect(evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "hermes-e2e" })).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["hermes-e2e-vitest"], - registryScenarios: [], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "hermes-root-entrypoint-smoke" }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["hermes-root-entrypoint-smoke-vitest"], - registryScenarios: [], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "hermes-root-entrypoint-smoke-vitest" }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["hermes-root-entrypoint-smoke-vitest"], - registryScenarios: [], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "rebuild-openclaw" }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["rebuild-openclaw-vitest"], - registryScenarios: [], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "rebuild-openclaw-vitest" }), - ).toMatchObject({ + expect(evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "" })).toMatchObject({ valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["rebuild-openclaw-vitest"], - registryScenarios: [], - }); - expect( - evaluateE2eVitestWorkflowDispatchSelectors({ - scenarios: "model-router-provider-routed-inference", - }), - ).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: ["model-router-provider-routed-inference-vitest"], + liveScenariosRuns: true, registryScenarios: [], }); expect( @@ -206,85 +107,19 @@ describe("e2e-vitest-scenarios workflow boundary", () => { }); }); - it("keeps jobs-only dispatches from selecting the Hermes secret-bearing job", () => { - expect( - generateMatrixForDispatch({ JOBS: "openshell-version-pin-vitest", SCENARIOS: "" }), - ).toMatchObject({ - hermes_selected: "false", - matrix: "[]", - }); - expect(generateMatrixForDispatch({ JOBS: "hermes-e2e-vitest", SCENARIOS: "" })).toMatchObject({ - hermes_selected: "true", + it("keeps jobs-only dispatches from running the registry matrix", () => { + expect(generateMatrixForDispatch("openshell-version-pin-vitest")).toMatchObject({ matrix: "[]", }); - expect( - generateMatrixForDispatch({ JOBS: "network-policy-vitest", SCENARIOS: "" }), - ).toMatchObject({ - hermes_selected: "false", - matrix: "[]", - }); - expect( - generateMatrixForDispatch({ JOBS: "runtime-overrides-vitest", SCENARIOS: "" }), - ).toMatchObject({ - hermes_selected: "false", + expect(generateMatrixForDispatch("hermes-e2e-vitest")).toMatchObject({ matrix: "[]", }); - expect(generateMatrixForDispatch({ JOBS: "", SCENARIOS: "runtime-overrides" })).toMatchObject({ - hermes_selected: "false", + expect(generateMatrixForDispatch("network-policy-vitest")).toMatchObject({ matrix: "[]", }); expect( - generateMatrixForDispatch({ JOBS: "inference-routing-vitest", SCENARIOS: "" }), - ).toMatchObject({ - hermes_selected: "false", - matrix: "[]", - }); - expect(generateMatrixForDispatch({ JOBS: "", SCENARIOS: "inference-routing" })).toMatchObject({ - hermes_selected: "false", - matrix: "[]", - }); - expect( - generateMatrixForDispatch({ JOBS: "rebuild-openclaw-vitest", SCENARIOS: "" }), - ).toMatchObject({ - hermes_selected: "false", - matrix: "[]", - }); - expect(generateMatrixForDispatch({ JOBS: "", SCENARIOS: "rebuild-openclaw" })).toMatchObject({ - hermes_selected: "false", - matrix: "[]", - }); - expect(generateMatrixForDispatch({ JOBS: "", SCENARIOS: "hermes-e2e" })).toMatchObject({ - hermes_selected: "true", - matrix: "[]", - }); - expect( - generateMatrixForDispatch({ JOBS: "hermes-root-entrypoint-smoke-vitest", SCENARIOS: "" }), - ).toMatchObject({ - hermes_selected: "false", - matrix: "[]", - }); - expect( - generateMatrixForDispatch({ JOBS: "", SCENARIOS: "hermes-root-entrypoint-smoke" }), - ).toMatchObject({ - hermes_selected: "false", - matrix: "[]", - }); - expect( - generateMatrixForDispatch({ - JOBS: "model-router-provider-routed-inference-vitest", - SCENARIOS: "", - }), - ).toMatchObject({ - hermes_selected: "false", - matrix: "[]", - }); - expect( - generateMatrixForDispatch({ - JOBS: "", - SCENARIOS: "model-router-provider-routed-inference", - }), + generateMatrixForDispatch("model-router-provider-routed-inference-vitest"), ).toMatchObject({ - hermes_selected: "false", matrix: "[]", }); }); @@ -318,7 +153,7 @@ jobs: - name: Post Vitest scenario results to PR env: JOBS: bad - run: echo "\${{ inputs.pr_number }} \${{ inputs.scenarios }}" + run: echo "\${{ inputs.pr_number }} \${{ inputs.jobs }}" live-scenarios: runs-on: ubuntu-latest env: @@ -349,7 +184,7 @@ jobs: openshell-version-pin-vitest: runs-on: ubuntu-latest needs: generate-matrix - if: \${{ inputs.scenarios != '' }} + if: \${{ inputs.jobs != '' }} env: E2E_ARTIFACT_DIR: \${{ github.workspace }}/.e2e/openshell-version-pin NEMOCLAW_RUN_E2E_SCENARIOS: "0" @@ -378,7 +213,7 @@ jobs: onboard-negative-paths-vitest: runs-on: ubuntu-latest needs: generate-matrix - if: \${{ inputs.scenarios != '' }} + if: \${{ inputs.jobs != '' }} env: E2E_ARTIFACT_DIR: \${{ github.workspace }}/.e2e/onboard-negative-paths NEMOCLAW_RUN_E2E_SCENARIOS: "0" @@ -407,7 +242,7 @@ jobs: network-policy-vitest: runs-on: macos-latest needs: generate-matrix - if: \${{ inputs.scenarios != '' }} + if: \${{ inputs.jobs != '' }} env: E2E_ARTIFACT_DIR: \${{ github.workspace }}/.e2e/network-policy NEMOCLAW_CLI_BIN: bin/not-nemoclaw.js @@ -451,7 +286,7 @@ jobs: double-onboard-vitest: runs-on: ubuntu-latest needs: generate-matrix - if: \${{ inputs.scenarios != '' }} + if: \${{ inputs.jobs != '' }} env: E2E_ARTIFACT_DIR: \${{ github.workspace }}/.e2e/double-onboard NEMOCLAW_CLI_BIN: ./bad-cli.js @@ -494,23 +329,19 @@ jobs: const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); expect(errors).toEqual( expect.arrayContaining([ - "workflow_dispatch missing input: scenarios", "workflow_dispatch missing input: jobs", "workflow_dispatch must not expose legacy test_filter input", "validate-jobs job must run on ubuntu-latest", "validate-jobs step must pass jobs through JOBS env", - "validate-jobs step must pass scenarios through SCENARIOS env", - "step 'Validate free-standing job selector' run script must include Use either scenarios or jobs, not both", - "step 'Validate free-standing job selector' run script must include Invalid scenario input; use comma-separated scenario ids", "step 'Validate free-standing job selector' run script must include allowed_jobs=", "step 'Validate free-standing job selector' run script must include runtime-overrides-vitest", "step 'Validate free-standing job selector' run script must include double-onboard-vitest", "step 'Validate free-standing job selector' run script must include hermes-e2e-vitest", + "step 'Validate free-standing job selector' run script must include model-router-provider-routed-inference-vitest", "step 'Validate free-standing job selector' run script must include Invalid jobs input; use comma-separated job ids", "step 'Validate free-standing job selector' run script must not include Invalid jobs input: ${JOBS}", "step 'Validate free-standing job selector' run script must include Unknown free-standing Vitest job", "workflow missing generate-matrix job", - "generate-matrix job must expose hermes_selected output", "generate-matrix job must run on ubuntu-latest", "live-scenarios job must run on the matrix runner", "live-scenarios job must depend on generate-matrix", @@ -543,7 +374,7 @@ jobs: "artifact upload path must include e2e-artifacts/vitest/${{ matrix.id }}/shell/", "artifact upload retention-days must be 14", "upload-artifact action must be pinned to a full commit SHA", - "openshell-version-pin-vitest job must depend on validate-jobs and generate-matrix", + "openshell-version-pin-vitest job must depend on validate-jobs", "openshell-version-pin-vitest job must use the shared jobs selector condition", "openshell-version-pin-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1", "openshell-version-pin-vitest job must write artifacts under e2e-artifacts/vitest/openshell-version-pin", @@ -562,11 +393,10 @@ jobs: "openshell-version-pin-vitest artifact upload must set include-hidden-files: false", "openshell-version-pin-vitest artifact upload must ignore missing fixture artifacts", "openshell-version-pin-vitest artifact upload retention-days must be 14", - "onboard-negative-paths-vitest job must depend on validate-jobs and generate-matrix", + "onboard-negative-paths-vitest job must depend on validate-jobs", "onboard-negative-paths-vitest job must use the shared jobs selector condition", "network-policy-vitest job must run on ubuntu-latest", - "network-policy-vitest job must depend on validate-jobs and generate-matrix", - "network-policy-vitest job must map scenarios=network-policy to the network-policy job", + "network-policy-vitest job must depend on validate-jobs", "network-policy-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1", "network-policy-vitest job must write artifacts under e2e-artifacts/vitest/network-policy", "network-policy-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI", @@ -648,8 +478,10 @@ jobs: "double-onboard-vitest artifact upload must ignore missing fixture artifacts", "double-onboard-vitest artifact upload retention-days must be 14", "workflow missing hermes-e2e-vitest job", + "workflow missing model-router-provider-routed-inference-vitest job", "report-to-pr job must wait for hermes-e2e-vitest", - "openclaw-tui-chat-correlation-vitest job must depend on validate-jobs and generate-matrix", + "report-to-pr job must wait for model-router-provider-routed-inference-vitest", + "openclaw-tui-chat-correlation-vitest job must depend on validate-jobs", "openclaw-tui-chat-correlation-vitest job must use the shared jobs selector condition", "gateway-guard-recovery job must depend on validate-jobs", "gateway-guard-recovery job must use the shared jobs selector condition", @@ -657,14 +489,10 @@ jobs: "report-to-pr job must wait for live-scenarios", "report-to-pr job must wait for double-onboard-vitest", "report-to-pr step must pass pr_number through JOB_PR_NUMBER env", - "report-to-pr step must pass scenarios through JOB_SCENARIOS env", "step 'Post Vitest scenario results to PR' run script must include process.env.JOBS", - "step 'Post Vitest scenario results to PR' run script must include process.env.JOB_SCENARIOS", "step 'Post Vitest scenario results to PR' run script must check validate-jobs before echoing selectors", "step 'Post Vitest scenario results to PR' run script must omit rejected job selectors", - "step 'Post Vitest scenario results to PR' run script must omit rejected scenario selectors", "step 'Post Vitest scenario results to PR' run script must include **Requested jobs:**", - "step 'Post Vitest scenario results to PR' run script must include **Requested scenarios:**", ]), ); } finally { diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 0f259f6d11..d1268b74bb 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -18,25 +18,23 @@ type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { name?: string; run?: string; uses?: string; with?: WorkflowRecord }; const SELECTOR_PATTERN = /^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$/; -const FREE_STANDING_SCENARIO_JOBS = new Map([ - ["openshell-version-pin", "openshell-version-pin-vitest"], - ["onboard-negative-paths", "onboard-negative-paths-vitest"], - ["inference-routing", "inference-routing-vitest"], - ["runtime-overrides", "runtime-overrides-vitest"], - ["hermes-e2e", "hermes-e2e-vitest"], - ["hermes-root-entrypoint-smoke", "hermes-root-entrypoint-smoke-vitest"], - ["network-policy", "network-policy-vitest"], - ["rebuild-openclaw", "rebuild-openclaw-vitest"], - ["token-rotation", "token-rotation-vitest"], - ["openclaw-tui-chat-correlation", "openclaw-tui-chat-correlation-vitest"], - ["issue-4434-tui-unreachable-inference", "issue-4434-tui-unreachable-inference-vitest"], - ["model-router-provider-routed-inference", "model-router-provider-routed-inference-vitest"], -]); const ALLOWED_FREE_STANDING_JOBS = new Set([ - ...FREE_STANDING_SCENARIO_JOBS.values(), + "openshell-version-pin-vitest", + "onboard-negative-paths-vitest", + "inference-routing-vitest", "credential-migration-vitest", + "runtime-overrides-vitest", + "hermes-e2e-vitest", + "hermes-root-entrypoint-smoke-vitest", + "network-policy-vitest", + "rebuild-openclaw-vitest", + "token-rotation-vitest", + "launchable-smoke-vitest", + "openclaw-tui-chat-correlation-vitest", "gateway-guard-recovery", "double-onboard-vitest", + "issue-4434-tui-unreachable-inference-vitest", + "model-router-provider-routed-inference-vitest", ]); export interface WorkflowDispatchSelectorEvaluation { @@ -72,18 +70,10 @@ function splitSelector(value: string): string[] { export function evaluateE2eVitestWorkflowDispatchSelectors(input: { jobs?: string; - scenarios?: string; }): WorkflowDispatchSelectorEvaluation { const jobs = input.jobs ?? ""; - const scenarios = input.scenarios ?? ""; const errors: string[] = []; - if (jobs && scenarios) { - errors.push("Use either scenarios or jobs, not both"); - } - if (scenarios && !SELECTOR_PATTERN.test(scenarios)) { - errors.push("Invalid scenario input"); - } if (jobs && !SELECTOR_PATTERN.test(jobs)) { errors.push("Invalid jobs input"); } @@ -105,40 +95,12 @@ export function evaluateE2eVitestWorkflowDispatchSelectors(input: { }; } - if (!jobs && !scenarios) { - return { - valid: true, - errors: [], - selectedFreeStandingJobs: [...ALLOWED_FREE_STANDING_JOBS].sort(), - registryScenarios: [], - liveScenariosRuns: true, - }; - } - - if (jobs) { - return { - valid: true, - errors: [], - selectedFreeStandingJobs: splitSelector(jobs).sort(), - registryScenarios: [], - liveScenariosRuns: false, - }; - } - - const selectedFreeStandingJobs = new Set(); - const registryScenarios: string[] = []; - for (const scenario of splitSelector(scenarios)) { - const job = FREE_STANDING_SCENARIO_JOBS.get(scenario); - if (job) selectedFreeStandingJobs.add(job); - else registryScenarios.push(scenario); - } - return { valid: true, errors: [], - selectedFreeStandingJobs: [...selectedFreeStandingJobs].sort(), - registryScenarios, - liveScenariosRuns: registryScenarios.length > 0, + selectedFreeStandingJobs: jobs ? splitSelector(jobs).sort() : [...ALLOWED_FREE_STANDING_JOBS].sort(), + registryScenarios: [], + liveScenariosRuns: !jobs, }; } @@ -231,29 +193,20 @@ function requireNoDispatchInputInterpolation( } } -function freeStandingJobIf(jobName: string, scenarioName?: string): string { - const scenarioSelector = scenarioName - ? ` || contains(format(',{0},', inputs.scenarios), ',${scenarioName},')` - : ""; - return `\${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',${jobName},')${scenarioSelector} }}`; +function freeStandingJobIf(jobName: string): string { + return `\${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',${jobName},') }}`; } function validateFreeStandingJobSelector( errors: string[], jobs: WorkflowRecord, jobName: string, - scenarioName?: string, ): void { const job = asRecord(jobs[jobName]); - if (scenarioName) { - const needs = Array.isArray(job.needs) ? job.needs : []; - if (!needs.includes("validate-jobs") || !needs.includes("generate-matrix")) { - errors.push(`${jobName} job must depend on validate-jobs and generate-matrix`); - } - } else if (job.needs !== "validate-jobs") { + if (job.needs !== "validate-jobs") { errors.push(`${jobName} job must depend on validate-jobs`); } - if (job.if !== freeStandingJobIf(jobName, scenarioName)) { + if (job.if !== freeStandingJobIf(jobName)) { errors.push(`${jobName} job must use the shared jobs selector condition`); } } @@ -274,11 +227,6 @@ function validateJobsSelector(errors: string[], jobs: WorkflowRecord): void { if (env.JOBS !== "${{ inputs.jobs }}") { errors.push("validate-jobs step must pass jobs through JOBS env"); } - if (env.SCENARIOS !== "${{ inputs.scenarios }}") { - errors.push("validate-jobs step must pass scenarios through SCENARIOS env"); - } - requireRunContains(errors, validate, "Use either scenarios or jobs, not both"); - requireRunContains(errors, validate, "Invalid scenario input; use comma-separated scenario ids"); requireRunContains(errors, validate, "allowed_jobs="); requireRunContains(errors, validate, "openshell-version-pin-vitest"); requireRunContains(errors, validate, "onboard-negative-paths-vitest"); @@ -311,7 +259,7 @@ function validateOpenShellVersionPinVitestJob(errors: string[], jobs: WorkflowRe if (job["runs-on"] !== "ubuntu-latest") { errors.push("openshell-version-pin-vitest job must run on ubuntu-latest"); } - validateFreeStandingJobSelector(errors, jobs, jobName, "openshell-version-pin"); + validateFreeStandingJobSelector(errors, jobs, jobName); const jobEnv = asRecord(job.env); if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { @@ -391,12 +339,11 @@ function validateNetworkPolicyVitestJob(errors: string[], jobs: WorkflowRecord): if (job["runs-on"] !== "ubuntu-latest") { errors.push("network-policy-vitest job must run on ubuntu-latest"); } - const needs = Array.isArray(job.needs) ? job.needs : []; - if (!needs.includes("validate-jobs") || !needs.includes("generate-matrix")) { - errors.push("network-policy-vitest job must depend on validate-jobs and generate-matrix"); + if (job.needs !== "validate-jobs") { + errors.push("network-policy-vitest job must depend on validate-jobs"); } - if (job.if !== freeStandingJobIf(jobName, "network-policy")) { - errors.push("network-policy-vitest job must map scenarios=network-policy to the network-policy job"); + if (job.if !== freeStandingJobIf(jobName)) { + errors.push("network-policy-vitest job must use the shared jobs selector condition"); } const jobEnv = asRecord(job.env); @@ -526,7 +473,7 @@ function validateRebuildOpenClawVitestJob(errors: string[], jobs: WorkflowRecord if (job["runs-on"] !== "ubuntu-latest") { errors.push("rebuild-openclaw-vitest job must run on ubuntu-latest"); } - validateFreeStandingJobSelector(errors, jobs, jobName, "rebuild-openclaw"); + validateFreeStandingJobSelector(errors, jobs, jobName); if (job["timeout-minutes"] !== 130) { errors.push("rebuild-openclaw-vitest job must keep the legacy 130 minute timeout"); } @@ -630,7 +577,7 @@ function validateTokenRotationVitestJob(errors: string[], jobs: WorkflowRecord): if (job["runs-on"] !== "ubuntu-latest") { errors.push("token-rotation-vitest job must run on ubuntu-latest"); } - validateFreeStandingJobSelector(errors, jobs, jobName, "token-rotation"); + validateFreeStandingJobSelector(errors, jobs, jobName); if (job["timeout-minutes"] !== 45) { errors.push("token-rotation-vitest job must keep the legacy 45 minute timeout"); } @@ -749,7 +696,7 @@ function validateOnboardNegativePathsVitestJob(errors: string[], jobs: WorkflowR if (job["runs-on"] !== "ubuntu-latest") { errors.push("onboard-negative-paths-vitest job must run on ubuntu-latest"); } - validateFreeStandingJobSelector(errors, jobs, jobName, "onboard-negative-paths"); + validateFreeStandingJobSelector(errors, jobs, jobName); const jobEnv = asRecord(job.env); if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { @@ -953,7 +900,7 @@ function validateDoubleOnboardVitestJob(errors: string[], jobs: WorkflowRecord): if (job["runs-on"] !== "ubuntu-latest") { errors.push("runtime-overrides-vitest job must run on ubuntu-latest"); } - validateFreeStandingJobSelector(errors, jobs, jobName, "runtime-overrides"); + validateFreeStandingJobSelector(errors, jobs, jobName); const jobEnv = asRecord(job.env); if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { @@ -1025,15 +972,11 @@ function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): voi if (job["runs-on"] !== "ubuntu-latest") { errors.push("hermes-e2e-vitest job must run on ubuntu-latest"); } - const needs = Array.isArray(job.needs) ? job.needs : []; - if (!needs.includes("validate-jobs") || !needs.includes("generate-matrix")) { - errors.push("hermes-e2e-vitest job must depend on validate-jobs and generate-matrix validation"); - } - if (job.if !== "${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}") { - errors.push("hermes-e2e-vitest job must use validated hermes_selected output"); + if (job.needs !== "validate-jobs") { + errors.push("hermes-e2e-vitest job must depend on validate-jobs"); } - if (stringValue(job.if).includes("inputs.scenarios")) { - errors.push("hermes-e2e-vitest job must not inspect raw workflow dispatch scenarios"); + if (job.if !== freeStandingJobIf(jobName)) { + errors.push("hermes-e2e-vitest job must use the shared jobs selector condition"); } const jobEnv = asRecord(job.env); @@ -1129,17 +1072,14 @@ function validateHermesRootEntrypointSmokeVitestJob( if (job["runs-on"] !== "ubuntu-latest") { errors.push("hermes-root-entrypoint-smoke-vitest job must run on ubuntu-latest"); } - const needs = Array.isArray(job.needs) ? job.needs : []; - if (!needs.includes("validate-jobs") || !needs.includes("generate-matrix")) { - errors.push( - "hermes-root-entrypoint-smoke-vitest job must depend on validate-jobs and generate-matrix", - ); + if (job.needs !== "validate-jobs") { + errors.push("hermes-root-entrypoint-smoke-vitest job must depend on validate-jobs"); } const expectedIf = - "${{ needs.generate-matrix.result == 'success' && ((inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-vitest,') || contains(format(',{0},', inputs.scenarios), ',hermes-root-entrypoint-smoke,')) }}"; + "${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-vitest,') }}"; if (job.if !== expectedIf) { errors.push( - "hermes-root-entrypoint-smoke-vitest job must gate on generate-matrix and the shared selector condition", + "hermes-root-entrypoint-smoke-vitest job must use the shared jobs selector condition", ); } if (job["timeout-minutes"] !== 45) { @@ -1284,7 +1224,6 @@ function validateModelRouterProviderRoutedInferenceVitestJob( jobs: WorkflowRecord, ): void { const jobName = "model-router-provider-routed-inference-vitest"; - const scenarioName = "model-router-provider-routed-inference"; const job = asRecord(jobs[jobName]); if (Object.keys(job).length === 0) { errors.push("workflow missing model-router-provider-routed-inference-vitest job"); @@ -1294,7 +1233,7 @@ function validateModelRouterProviderRoutedInferenceVitestJob( if (job["runs-on"] !== "ubuntu-latest") { errors.push("model-router-provider-routed-inference-vitest job must run on ubuntu-latest"); } - validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + validateFreeStandingJobSelector(errors, jobs, jobName); const jobEnv = asRecord(job.env); if ( @@ -1472,7 +1411,6 @@ export function validateE2eVitestScenariosWorkflowBoundary( rejectAutomaticTriggers(errors, triggers); const dispatchInputs = asRecord(workflowDispatch.inputs); - requireInput(errors, dispatchInputs, "scenarios"); requireInput(errors, dispatchInputs, "jobs"); if (Object.hasOwn(dispatchInputs, "test_filter")) { errors.push("workflow_dispatch must not expose legacy test_filter input"); @@ -1493,9 +1431,6 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (generateOutputs.matrix !== "${{ steps.matrix.outputs.matrix }}") { errors.push("generate-matrix job must expose matrix output"); } - if (generateOutputs.hermes_selected !== "${{ steps.matrix.outputs.hermes_selected }}") { - errors.push("generate-matrix job must expose hermes_selected output"); - } const generateSteps = asSteps(generateMatrix.steps); requireNoDispatchInputInterpolation(errors, generateSteps); const generateCheckout = generateSteps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); @@ -1512,20 +1447,13 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (generateEnv.JOBS !== "${{ inputs.jobs }}") { errors.push("matrix generation step must pass jobs through JOBS env"); } - if (generateEnv.SCENARIOS !== "${{ inputs.scenarios }}") { - errors.push("matrix generation step must pass scenarios through SCENARIOS env"); - } requireRunContains(errors, generate, "allowed_jobs="); - requireRunContains(errors, generate, "Use either scenarios or jobs, not both"); requireRunContains(errors, generate, "Unknown free-standing Vitest job"); requireRunContains(errors, generate, "inference-routing-vitest"); - requireRunContains(errors, generate, "inference-routing"); requireRunContains(errors, generate, "runtime-overrides-vitest"); - requireRunContains(errors, generate, "runtime-overrides"); requireRunContains(errors, generate, "double-onboard-vitest"); requireRunContains(errors, generate, "hermes-e2e-vitest"); requireRunContains(errors, generate, "hermes-root-entrypoint-smoke-vitest"); - requireRunContains(errors, generate, "hermes-root-entrypoint-smoke"); requireRunContains(errors, generate, "network-policy-vitest"); requireRunContains(errors, generate, "rebuild-openclaw-vitest"); requireRunContains(errors, generate, "token-rotation-vitest"); @@ -1534,16 +1462,10 @@ export function validateE2eVitestScenariosWorkflowBoundary( requireRunContains(errors, generate, 'matrix="[]"'); requireRunContains(errors, generate, "npx tsx test/e2e-scenario/scenarios/run.ts"); requireRunContains(errors, generate, "--emit-live-matrix"); - requireRunContains(errors, generate, "--scenarios"); requireRunContains(errors, generate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$"); - requireRunContains(errors, generate, "Invalid scenario input; use comma-separated scenario ids"); requireRunContains(errors, generate, "Invalid jobs input; use comma-separated job ids"); requireRunDoesNotContain(errors, generate, "Invalid jobs input: ${JOBS}"); - requireRunDoesNotContain(errors, generate, "Invalid scenario input: ${SCENARIOS}"); requireRunDoesNotContain(errors, generate, "^[A-Za-z0-9._-]+"); - requireRunContains(errors, generate, "hermes_selected=false"); - requireRunContains(errors, generate, "hermes_selected=true"); - requireRunContains(errors, generate, 'echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT"'); requireRunContains(errors, generate, "## Vitest E2E Scenario Matrix"); requireRunContains(errors, generate, "| Scenario | Runner | Label |"); @@ -1684,7 +1606,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateOpenShellVersionPinVitestJob(errors, jobs); validateOnboardNegativePathsVitestJob(errors, jobs); validateFreeStandingJobSelector(errors, jobs, "credential-migration-vitest"); - validateFreeStandingJobSelector(errors, jobs, "inference-routing-vitest", "inference-routing"); + validateFreeStandingJobSelector(errors, jobs, "inference-routing-vitest"); validateRuntimeOverridesVitestJob(errors, jobs); validateDoubleOnboardVitestJob(errors, jobs); validateHermesE2EVitestJob(errors, jobs); @@ -1692,19 +1614,9 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateNetworkPolicyVitestJob(errors, jobs); validateRebuildOpenClawVitestJob(errors, jobs); validateTokenRotationVitestJob(errors, jobs); - validateFreeStandingJobSelector( - errors, - jobs, - "openclaw-tui-chat-correlation-vitest", - "openclaw-tui-chat-correlation", - ); + validateFreeStandingJobSelector(errors, jobs, "openclaw-tui-chat-correlation-vitest"); validateFreeStandingJobSelector(errors, jobs, "gateway-guard-recovery"); - validateFreeStandingJobSelector( - errors, - jobs, - "issue-4434-tui-unreachable-inference-vitest", - "issue-4434-tui-unreachable-inference", - ); + validateFreeStandingJobSelector(errors, jobs, "issue-4434-tui-unreachable-inference-vitest"); validateModelRouterProviderRoutedInferenceVitestJob(errors, jobs); const reportToPr = asRecord(jobs["report-to-pr"]); @@ -1743,32 +1655,20 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (reportEnv.JOB_PR_NUMBER !== "${{ inputs.pr_number }}") { errors.push("report-to-pr step must pass pr_number through JOB_PR_NUMBER env"); } - if (reportEnv.JOB_SCENARIOS !== "${{ inputs.scenarios }}") { - errors.push("report-to-pr step must pass scenarios through JOB_SCENARIOS env"); - } const reportScript = stringValue(asRecord(report?.with).script ?? report?.run); if (!reportScript.includes("process.env.JOBS")) { errors.push("step 'Post Vitest scenario results to PR' run script must include process.env.JOBS"); } - if (!reportScript.includes("process.env.JOB_SCENARIOS")) { - errors.push("step 'Post Vitest scenario results to PR' run script must include process.env.JOB_SCENARIOS"); - } if (!reportScript.includes("selectorValidationPassed")) { errors.push("step 'Post Vitest scenario results to PR' run script must check validate-jobs before echoing selectors"); } if (!reportScript.includes("jobsRejected")) { errors.push("step 'Post Vitest scenario results to PR' run script must omit rejected job selectors"); } - if (!reportScript.includes("scenariosRejected")) { - errors.push("step 'Post Vitest scenario results to PR' run script must omit rejected scenario selectors"); - } if (!reportScript.includes("**Requested jobs:**")) { errors.push("step 'Post Vitest scenario results to PR' run script must include **Requested jobs:**"); } - if (!reportScript.includes("**Requested scenarios:**")) { - errors.push("step 'Post Vitest scenario results to PR' run script must include **Requested scenarios:**"); - } - for (const forbidden of ["toJSON(inputs.pr_number)", "toJSON(inputs.scenarios)"]) { + for (const forbidden of ["toJSON(inputs.pr_number)"]) { if (reportScript.includes(forbidden)) { errors.push( `step 'Post Vitest scenario results to PR' run script must not include ${forbidden}`, From 67cf895d802d18c8f49c702c66a6fb3c3cc6ff8c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 10:12:56 -0400 Subject: [PATCH 2/6] test(e2e): tighten jobs selector assertions --- .../e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 0e8aaf3328..a7bfbfe32c 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -68,9 +68,10 @@ describe("e2e-vitest-scenarios workflow boundary", () => { it("evaluates jobs-only dispatch selector behavior before secret-bearing jobs run", () => { expect( - evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "network-policy,../escape" }), + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "network-policy-vitest,../escape" }), ).toMatchObject({ valid: false, + errors: ["Invalid jobs input"], liveScenariosRuns: false, selectedFreeStandingJobs: [], }); @@ -489,6 +490,7 @@ jobs: "report-to-pr job must wait for live-scenarios", "report-to-pr job must wait for double-onboard-vitest", "report-to-pr step must pass pr_number through JOB_PR_NUMBER env", + "report-to-pr step must pass jobs through JOBS env", "step 'Post Vitest scenario results to PR' run script must include process.env.JOBS", "step 'Post Vitest scenario results to PR' run script must check validate-jobs before echoing selectors", "step 'Post Vitest scenario results to PR' run script must omit rejected job selectors", From d4684a36eb2a682c8470e2fa5e3901a9758aa473 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 10:24:25 -0400 Subject: [PATCH 3/6] fix(e2e): update scenario advisor jobs selector --- test/e2e-scenario-advisor.test.ts | 16 +++++++++------- tools/e2e-advisor/scenarios.mts | 7 ++++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/test/e2e-scenario-advisor.test.ts b/test/e2e-scenario-advisor.test.ts index 04b3994726..4ce96daf64 100644 --- a/test/e2e-scenario-advisor.test.ts +++ b/test/e2e-scenario-advisor.test.ts @@ -61,9 +61,11 @@ describe("Vitest E2E scenario advisor — prompt construction", () => { expect(systemPrompt).toContain("trusted advisor checkout"); expect(systemPrompt).toContain("recommend the `e2e-scenarios-all` fan-out"); expect(systemPrompt).toContain("single NemoClaw E2E system"); + expect(systemPrompt).toContain("workflow only accepts the optional `jobs` selector"); expect(systemPrompt).not.toContain("non-scenario E2E"); expect(systemPrompt).not.toContain("e2e-scenarios-all.yaml"); expect(systemPrompt).not.toContain("e2e-scenarios.yaml"); + expect(systemPrompt).not.toContain("--field scenarios"); }); it("exports the Vitest scenario workflow for both targeted and fan-out recommendations", () => { @@ -116,12 +118,12 @@ describe("Vitest E2E scenario advisor — normalization contract", () => { expect(normalized.optional[0]?.dispatchCommand).toBe( canonicalDispatchCommand(VITEST_SCENARIO_WORKFLOW, "ubuntu-repo-cloud-openclaw"), ); - // Canonical fan-out command must not contain a scenarios field. + // The jobs-only workflow has no scenario selector; typed scenario + // recommendations dispatch the default fan-out while preserving scenario + // metadata for reviewers. expect(normalized.required[0]?.dispatchCommand).not.toContain("--field scenarios="); - // Canonical single-scenario command must use plural --field scenarios= - // and must never contain the legacy suite_filter input. - expect(normalized.optional[0]?.dispatchCommand).toContain( - "--field scenarios=ubuntu-repo-cloud-openclaw", + expect(normalized.optional[0]?.dispatchCommand).toBe( + "gh workflow run e2e-vitest-scenarios.yaml --ref ", ); expect(normalized.optional[0]?.dispatchCommand).not.toContain("suite_filter"); }); @@ -361,7 +363,7 @@ jobs: steps: - run: npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/registry-scenarios.test.ts token-rotation-vitest: - if: \${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') }} + if: \${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') }} steps: - run: npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/token-rotation.test.ts `), @@ -399,7 +401,7 @@ jobs: vitestWorkflowText: String.raw` jobs: token-rotation-vitest: - if: \${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') }} + if: \${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') }} steps: - run: npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/token-rotation.test.ts `, diff --git a/tools/e2e-advisor/scenarios.mts b/tools/e2e-advisor/scenarios.mts index b95a93bbc0..93694339e8 100755 --- a/tools/e2e-advisor/scenarios.mts +++ b/tools/e2e-advisor/scenarios.mts @@ -73,7 +73,8 @@ export function canonicalDispatchCommand( if (!JOB_ID_PATTERN.test(id)) throw new Error(`Invalid Vitest job id: ${id}`); return `gh workflow run ${SCENARIO_WORKFLOW} --ref --field jobs=${id}`; } - return `gh workflow run ${SCENARIO_WORKFLOW} --ref --field scenarios=${id}`; + if (!SCENARIO_ID_PATTERN.test(id)) throw new Error(`Invalid Vitest scenario id: ${id}`); + return `gh workflow run ${SCENARIO_WORKFLOW} --ref `; } type ArtifactPaths = AdvisorArtifactPaths; @@ -266,7 +267,7 @@ export function buildSystemPrompt(schema: AdvisorSchema): string { "", "Decision policy:", "- Required (all scenarios): changes to scenario registry, matrix emission, expected-state metadata, live support classification, shared fixtures, or the shared Vitest scenario workflow machinery. Recommend the `e2e-scenarios-all` fan-out through `e2e-vitest-scenarios.yaml`.", - "- Required (targeted): fixture, live test, manifest, runtime-support, or scenario changes that affect a specific subset. Recommend the smallest set of live-supported typed scenario IDs that exercises the changed surface.", + "- Required (targeted): fixture, live test, manifest, runtime-support, or scenario changes that affect a specific subset. Recommend the smallest set of live-supported typed scenario IDs that exercises the changed surface, but dispatch them through the default jobs-empty fan-out because `.github/workflows/e2e-vitest-scenarios.yaml` no longer exposes an operator-facing `scenarios` selector.", "- Required (free-standing job): if a PR wires or changes a discrete live Vitest job in `.github/workflows/e2e-vitest-scenarios.yaml` for a specific `test/e2e-scenario/live/*.test.ts`, prefer that job over `e2e-scenarios-all`. Use selectorType=`job`, id=``, workflow=`e2e-vitest-scenarios.yaml`, and dispatchCommand exactly `gh workflow run e2e-vitest-scenarios.yaml --ref --field jobs=`.", "- Missing wiring: if a PR adds or changes a free-standing live Vitest file under `test/e2e-scenario/live/*.test.ts` but that file is not referenced by `.github/workflows/e2e-vitest-scenarios.yaml` and is not `registry-scenarios.test.ts`, do not recommend the fan-out as proof. Return no required/optional recommendations and set `noScenarioE2eReason` to say the test must be wired into `e2e-vitest-scenarios.yaml` before it can be dispatched.", "- Optional: adjacent scenarios that exercise the same suite on a different platform/onboarding (e.g. macOS, WSL, GPU) but are not the primary target. Special-runner scenarios (`gpu-`, `macos-`, `wsl-`, `brev-`) should usually be optional unless they are the only path that exercises the change.", @@ -275,7 +276,7 @@ export function buildSystemPrompt(schema: AdvisorSchema): string { "Hard rules:", "- Only recommend live-supported typed scenario IDs that exist in the registry or the synthetic fan-out id `e2e-scenarios-all`. Do not invent IDs.", "- The only allowed workflow is `e2e-vitest-scenarios.yaml`.", - "- Each `dispatchCommand` for a single-scenario recommendation MUST be exactly: `gh workflow run e2e-vitest-scenarios.yaml --ref --field scenarios=`.", + "- Each `dispatchCommand` for a typed scenario recommendation MUST use the default fan-out exactly: `gh workflow run e2e-vitest-scenarios.yaml --ref `. Never use a scenarios field; the workflow only accepts the optional `jobs` selector.", "- Each `dispatchCommand` for a free-standing job recommendation MUST be exactly: `gh workflow run e2e-vitest-scenarios.yaml --ref --field jobs=`.", "- For the fan-out, use exactly: `gh workflow run e2e-vitest-scenarios.yaml --ref ` and set `id`/`workflow`/`selectorType` to `e2e-scenarios-all`/`e2e-vitest-scenarios.yaml`/`all`.", "- The normalizer validates targeted IDs against the trusted advisor checkout's registry/runtime-support modules, not PR-local TypeScript. If a PR adds or newly wires a typed registry scenario that is not live-supported on trusted `main` yet, recommend the `e2e-scenarios-all` fan-out rather than a targeted dispatch. This fallback does not apply to free-standing live test jobs.", From cf1fa4c7e8cd3dba061471795b2f37b1469ca3f6 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 10:37:16 -0400 Subject: [PATCH 4/6] fix(e2e): address jobs selector review gaps --- .github/workflows/e2e-vitest-scenarios.yaml | 5 ++- test/e2e-scenario-advisor.test.ts | 28 +++++++++++++- tools/e2e-advisor/scenarios.mts | 26 +++++++++++-- tools/e2e-scenarios/workflow-boundary.mts | 41 +++++++++------------ 4 files changed, 71 insertions(+), 29 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index cb380ea7ab..306a8540df 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -59,16 +59,19 @@ jobs: matrix: ${{ steps.matrix.outputs.matrix }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + if: ${{ inputs.jobs == '' }} with: persist-credentials: false - name: Set up Node + if: ${{ inputs.jobs == '' }} uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 with: node-version: 22 cache: npm - name: Install root dependencies + if: ${{ inputs.jobs == '' }} run: npm ci --ignore-scripts - id: matrix @@ -1060,7 +1063,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: - DOCKER_CONFIG: ${{ runner.temp }}/docker-config-model-router-provider-routed-inference + DOCKER_CONFIG: /tmp/docker-config-model-router-provider-routed-inference E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/model-router-provider-routed-inference NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" diff --git a/test/e2e-scenario-advisor.test.ts b/test/e2e-scenario-advisor.test.ts index 4ce96daf64..fee4b36329 100644 --- a/test/e2e-scenario-advisor.test.ts +++ b/test/e2e-scenario-advisor.test.ts @@ -373,6 +373,16 @@ jobs: liveTestFiles: ["test/e2e-scenario/live/token-rotation.test.ts"], }, ]); + + expect( + extractFreeStandingVitestJobs(String.raw` +jobs: + token-rotation-vitest: + if: \${{ contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') }} + steps: + - run: npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/token-rotation.test.ts +`), + ).toEqual([]); }); it("prefers a focused free-standing job over fan-out once workflow wiring is present", () => { @@ -438,7 +448,7 @@ jobs: vitestWorkflowText: String.raw` jobs: token-rotation-vitest: - if: \${{ contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') }} + if: \${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') }} steps: - run: npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/token-rotation.test.ts `, @@ -559,6 +569,22 @@ describe("Vitest E2E scenario advisor — summary and comment rendering", () => ); }); + it("renders duplicate fan-out dispatches as one run-once command", () => { + const result = sampleResult(); + result.required.push({ + id: "ubuntu-repo-cloud-openclaw", + workflow: VITEST_SCENARIO_WORKFLOW, + selectorType: "scenario", + required: true, + reason: "targeted scenario metadata", + dispatchCommand: canonicalDispatchCommand(VITEST_SCENARIO_WORKFLOW, "ubuntu-repo-cloud-openclaw"), + }); + + const summary = renderScenarioSummary(result); + expect(summary.match(/gh workflow run e2e-vitest-scenarios.yaml --ref /g)).toHaveLength(1); + expect(summary).toContain("covered by the shared fan-out command above"); + }); + it("builds a sticky scenario comment with the marker and run url", () => { const result = sampleResult(); const summary = renderScenarioSummary(result); diff --git a/tools/e2e-advisor/scenarios.mts b/tools/e2e-advisor/scenarios.mts index 93694339e8..674d5c17aa 100755 --- a/tools/e2e-advisor/scenarios.mts +++ b/tools/e2e-advisor/scenarios.mts @@ -420,7 +420,7 @@ export function extractFreeStandingVitestJobs(workflowText: string): VitestWorkf bodyLines.push(lines[bodyIndex]); } const body = bodyLines.join("\n"); - if (!body.includes("inputs.jobs") || !body.includes(`,${id},`)) continue; + if (!hasSharedJobsSelector(body, id)) continue; const liveTestFiles = uniqueStrings( [...body.matchAll(/test\/e2e-scenario\/live\/[A-Za-z0-9._-]+\.test\.ts/g)].map( (item) => item[0], @@ -432,6 +432,14 @@ export function extractFreeStandingVitestJobs(workflowText: string): VitestWorkf return jobs.sort((a, b) => a.id.localeCompare(b.id)); } +function hasSharedJobsSelector(body: string, id: string): boolean { + return ( + body.includes("inputs.jobs == ''") && + body.includes("contains(format(',{0},', inputs.jobs)") && + body.includes(`,${id},`) + ); +} + function findUnwiredFreeStandingLiveTests( changedFiles: string[], vitestWorkflowText = readVitestWorkflowText(), @@ -619,9 +627,15 @@ export function renderScenarioSummary(result: ScenarioAdvisorResult): string { if (result.required.length === 0) { lines.push(`- _None._ ${result.noScenarioE2eReason || ""}`.trim()); } else { + const seenDispatches = new Set(); for (const recommendation of result.required) { lines.push(`- **${recommendation.id}**: ${recommendation.reason}`); - lines.push(` - Dispatch: \`${recommendation.dispatchCommand}\``); + if (seenDispatches.has(recommendation.dispatchCommand)) { + lines.push(" - Dispatch: covered by the shared fan-out command above."); + } else { + seenDispatches.add(recommendation.dispatchCommand); + lines.push(` - Dispatch: \`${recommendation.dispatchCommand}\``); + } } } lines.push(""); @@ -629,9 +643,15 @@ export function renderScenarioSummary(result: ScenarioAdvisorResult): string { if (result.optional.length === 0) { lines.push("- _None._"); } else { + const seenDispatches = new Set(); for (const recommendation of result.optional) { lines.push(`- **${recommendation.id}**: ${recommendation.reason}`); - lines.push(` - Dispatch: \`${recommendation.dispatchCommand}\``); + if (seenDispatches.has(recommendation.dispatchCommand)) { + lines.push(" - Dispatch: covered by the shared fan-out command above."); + } else { + seenDispatches.add(recommendation.dispatchCommand); + lines.push(` - Dispatch: \`${recommendation.dispatchCommand}\``); + } } } lines.push(""); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index d1268b74bb..767a0a588e 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -339,12 +339,7 @@ function validateNetworkPolicyVitestJob(errors: string[], jobs: WorkflowRecord): if (job["runs-on"] !== "ubuntu-latest") { errors.push("network-policy-vitest job must run on ubuntu-latest"); } - if (job.needs !== "validate-jobs") { - errors.push("network-policy-vitest job must depend on validate-jobs"); - } - if (job.if !== freeStandingJobIf(jobName)) { - errors.push("network-policy-vitest job must use the shared jobs selector condition"); - } + validateFreeStandingJobSelector(errors, jobs, jobName); const jobEnv = asRecord(job.env); if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { @@ -972,12 +967,7 @@ function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): voi if (job["runs-on"] !== "ubuntu-latest") { errors.push("hermes-e2e-vitest job must run on ubuntu-latest"); } - if (job.needs !== "validate-jobs") { - errors.push("hermes-e2e-vitest job must depend on validate-jobs"); - } - if (job.if !== freeStandingJobIf(jobName)) { - errors.push("hermes-e2e-vitest job must use the shared jobs selector condition"); - } + validateFreeStandingJobSelector(errors, jobs, jobName); const jobEnv = asRecord(job.env); if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { @@ -1072,16 +1062,7 @@ function validateHermesRootEntrypointSmokeVitestJob( if (job["runs-on"] !== "ubuntu-latest") { errors.push("hermes-root-entrypoint-smoke-vitest job must run on ubuntu-latest"); } - if (job.needs !== "validate-jobs") { - errors.push("hermes-root-entrypoint-smoke-vitest job must depend on validate-jobs"); - } - const expectedIf = - "${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-vitest,') }}"; - if (job.if !== expectedIf) { - errors.push( - "hermes-root-entrypoint-smoke-vitest job must use the shared jobs selector condition", - ); - } + validateFreeStandingJobSelector(errors, jobs, jobName); if (job["timeout-minutes"] !== 45) { errors.push("hermes-root-entrypoint-smoke-vitest job must keep the 45 minute timeout"); } @@ -1238,10 +1219,10 @@ function validateModelRouterProviderRoutedInferenceVitestJob( const jobEnv = asRecord(job.env); if ( jobEnv.DOCKER_CONFIG !== - "${{ runner.temp }}/docker-config-model-router-provider-routed-inference" + "/tmp/docker-config-model-router-provider-routed-inference" ) { errors.push( - "model-router-provider-routed-inference-vitest job must isolate Docker auth with DOCKER_CONFIG under runner.temp", + "model-router-provider-routed-inference-vitest job must isolate Docker auth with a job-safe DOCKER_CONFIG path", ); } if ( @@ -1439,9 +1420,21 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (asRecord(generateCheckout?.with)["persist-credentials"] !== false) { errors.push("generate-matrix checkout step must set persist-credentials=false"); } + if (generateCheckout?.if !== "${{ inputs.jobs == '' }}") { + errors.push("generate-matrix checkout step must skip when jobs selector is supplied"); + } const generateSetupNode = namedStep(generateSteps, "Set up Node"); if (!generateSetupNode) errors.push("generate-matrix job missing step: Set up Node"); requireFullShaAction(errors, generateSetupNode, "generate-matrix setup-node"); + if (generateSetupNode?.if !== "${{ inputs.jobs == '' }}") { + errors.push("generate-matrix setup-node step must skip when jobs selector is supplied"); + } + const generateInstall = namedStep(generateSteps, "Install root dependencies"); + if (!generateInstall) errors.push("generate-matrix job missing step: Install root dependencies"); + if (generateInstall?.if !== "${{ inputs.jobs == '' }}") { + errors.push("generate-matrix install step must skip when jobs selector is supplied"); + } + requireRunContains(errors, generateInstall, "npm ci --ignore-scripts"); const generate = requireStep(errors, generateSteps, "Generate Vitest scenario matrix"); const generateEnv = asRecord(generate?.env); if (generateEnv.JOBS !== "${{ inputs.jobs }}") { From 25ca0405948ae811e64412212160000af0c064f4 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 10:49:22 -0400 Subject: [PATCH 5/6] style(e2e): apply static formatting --- test/e2e-scenario-advisor.test.ts | 9 +++++++-- .../support-tests/e2e-scenarios-workflow.test.ts | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario-advisor.test.ts b/test/e2e-scenario-advisor.test.ts index fee4b36329..629c1689b2 100644 --- a/test/e2e-scenario-advisor.test.ts +++ b/test/e2e-scenario-advisor.test.ts @@ -577,11 +577,16 @@ describe("Vitest E2E scenario advisor — summary and comment rendering", () => selectorType: "scenario", required: true, reason: "targeted scenario metadata", - dispatchCommand: canonicalDispatchCommand(VITEST_SCENARIO_WORKFLOW, "ubuntu-repo-cloud-openclaw"), + dispatchCommand: canonicalDispatchCommand( + VITEST_SCENARIO_WORKFLOW, + "ubuntu-repo-cloud-openclaw", + ), }); const summary = renderScenarioSummary(result); - expect(summary.match(/gh workflow run e2e-vitest-scenarios.yaml --ref /g)).toHaveLength(1); + expect( + summary.match(/gh workflow run e2e-vitest-scenarios.yaml --ref /g), + ).toHaveLength(1); expect(summary).toContain("covered by the shared fan-out command above"); }); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index b39198efa7..c1dac5265e 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -106,7 +106,9 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["model-router-provider-routed-inference-vitest"], registryScenarios: [], }); - expect(evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "skill-agent-vitest" })).toMatchObject({ + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "skill-agent-vitest" }), + ).toMatchObject({ valid: true, liveScenariosRuns: false, selectedFreeStandingJobs: ["skill-agent-vitest"], From 4df0a328b305b1a6a53f57e03b7a616bf042ffbc Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 12 Jun 2026 11:05:36 -0400 Subject: [PATCH 6/6] fix(e2e): clarify scenario fan-out dispatch --- test/e2e-scenario-advisor.test.ts | 5 ++++- tools/e2e-advisor/scenarios.mts | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario-advisor.test.ts b/test/e2e-scenario-advisor.test.ts index 629c1689b2..7c885b9145 100644 --- a/test/e2e-scenario-advisor.test.ts +++ b/test/e2e-scenario-advisor.test.ts @@ -569,7 +569,7 @@ describe("Vitest E2E scenario advisor — summary and comment rendering", () => ); }); - it("renders duplicate fan-out dispatches as one run-once command", () => { + it("renders duplicate fan-out dispatches as one explicit full-fan-out command", () => { const result = sampleResult(); result.required.push({ id: "ubuntu-repo-cloud-openclaw", @@ -588,6 +588,9 @@ describe("Vitest E2E scenario advisor — summary and comment rendering", () => summary.match(/gh workflow run e2e-vitest-scenarios.yaml --ref /g), ).toHaveLength(1); expect(summary).toContain("covered by the shared fan-out command above"); + expect(summary).toContain("Typed scenario IDs are metadata-only"); + expect(summary).toContain("full default fan-out"); + expect(summary).toContain("not only this scenario"); }); it("builds a sticky scenario comment with the marker and run url", () => { diff --git a/tools/e2e-advisor/scenarios.mts b/tools/e2e-advisor/scenarios.mts index 674d5c17aa..70db54f16d 100755 --- a/tools/e2e-advisor/scenarios.mts +++ b/tools/e2e-advisor/scenarios.mts @@ -615,6 +615,14 @@ function stringArrayWithinChanged(value: unknown, changedFiles: string[]): strin return value.filter((file): file is string => typeof file === "string" && allowed.has(file)); } +function scenarioFanoutNote(recommendation: ScenarioRecommendation): string | undefined { + if (recommendation.selectorType !== "scenario") return undefined; + if (recommendation.dispatchCommand !== `gh workflow run ${SCENARIO_WORKFLOW} --ref `) { + return undefined; + } + return "Typed scenario IDs are metadata-only in this jobs-only workflow: this dispatch runs the full default fan-out (all supported registry scenarios plus all free-standing Vitest jobs), not only this scenario."; +} + export function renderScenarioSummary(result: ScenarioAdvisorResult): string { const lines: string[] = []; lines.push("# Vitest E2E Scenario Advisor"); @@ -636,6 +644,8 @@ export function renderScenarioSummary(result: ScenarioAdvisorResult): string { seenDispatches.add(recommendation.dispatchCommand); lines.push(` - Dispatch: \`${recommendation.dispatchCommand}\``); } + const fanoutNote = scenarioFanoutNote(recommendation); + if (fanoutNote) lines.push(` - Note: ${fanoutNote}`); } } lines.push(""); @@ -652,6 +662,8 @@ export function renderScenarioSummary(result: ScenarioAdvisorResult): string { seenDispatches.add(recommendation.dispatchCommand); lines.push(` - Dispatch: \`${recommendation.dispatchCommand}\``); } + const fanoutNote = scenarioFanoutNote(recommendation); + if (fanoutNote) lines.push(` - Note: ${fanoutNote}`); } } lines.push("");