Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 64 additions & 4 deletions .github/workflows/e2e-vitest-scenarios.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ on:
required: false
default: ""
type: string
jobs:
description: "Optional comma-separated free-standing live Vitest job ids. Empty runs all jobs only when scenarios is also empty."
required: false
default: ""
type: string
pr_number:
description: Optional PR number for selective-dispatch result comments.
required: false
Expand All @@ -21,10 +26,43 @@ permissions:
contents: read

concurrency:
group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }}
group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }}-${{ inputs.jobs || 'all-jobs' }}
cancel-in-progress: false

jobs:
validate-jobs:
runs-on: ubuntu-latest
steps:
- id: validate
name: Validate free-standing job selector
env:
JOBS: ${{ inputs.jobs }}
SCENARIOS: ${{ inputs.scenarios }}
run: |
set -euo pipefail
allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery"
if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then
echo "::error::Use either scenarios or jobs, not both." >&2
exit 1
fi
if [ -z "${JOBS}" ]; then
echo "No free-standing job selector supplied; default workflow behavior applies."
exit 0
fi
if [[ ! "${JOBS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then
echo "::error::Invalid jobs input; use comma-separated job ids containing only letters, numbers, underscores, and hyphens." >&2
exit 1
fi
IFS=',' read -ra requested <<< "${JOBS}"
for job in "${requested[@]}"; do
if [[ ",${allowed_jobs}," != *",${job},"* ]]; then
echo "::error::Unknown free-standing Vitest job: ${job}" >&2
echo "::error::Allowed jobs: ${allowed_jobs}" >&2
exit 1
fi
done
printf 'Validated free-standing Vitest jobs: `%s`\n' "${JOBS}" >> "$GITHUB_STEP_SUMMARY"

generate-matrix:
runs-on: ubuntu-latest
outputs:
Expand Down Expand Up @@ -74,6 +112,7 @@ jobs:

live-scenarios:
needs: generate-matrix
if: ${{ inputs.jobs == '' }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 45
strategy:
Expand Down Expand Up @@ -174,6 +213,8 @@ jobs:
# because the matrix above only runs registry-scenarios.test.ts. Modeled on
# #5049's free-standing pattern.
openshell-version-pin-vitest:
needs: validate-jobs
if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin-vitest,') }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
Expand Down Expand Up @@ -213,6 +254,8 @@ jobs:
retention-days: 14

onboard-negative-paths-vitest:
needs: validate-jobs
if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths-vitest,') }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
Expand Down Expand Up @@ -259,7 +302,8 @@ jobs:
# protocol/history contract. The retained legacy bash lane remains the
# source for full closeout until a later PR proves replacement and deletes it.
openclaw-tui-chat-correlation-vitest:
if: ${{ inputs.scenarios == '' }}
needs: validate-jobs
if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',openclaw-tui-chat-correlation-vitest,') }}
runs-on: ubuntu-latest
timeout-minutes: 75
env:
Expand Down Expand Up @@ -338,6 +382,8 @@ jobs:
# restore the /tmp guard chain after pod recreate). Will fail on `main`
# until the #2701 fix lands; flips green afterwards.
gateway-guard-recovery:
needs: validate-jobs
if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',gateway-guard-recovery,') }}
runs-on: ubuntu-latest
timeout-minutes: 45
env:
Expand Down Expand Up @@ -430,6 +476,7 @@ jobs:
runs-on: ubuntu-latest
needs:
[
validate-jobs,
generate-matrix,
live-scenarios,
openshell-version-pin-vitest,
Expand All @@ -444,13 +491,21 @@ jobs:
steps:
- name: Post Vitest scenario results to PR
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
JOB_PR_NUMBER: ${{ inputs.pr_number }}
JOB_SCENARIOS: ${{ inputs.scenarios }}
JOBS: ${{ inputs.jobs }}
with:
script: |
const needs = ${{ toJSON(needs) }};
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const workflowBranch = context.ref.replace('refs/heads/', '');
const prNumberInput = ${{ toJSON(inputs.pr_number) }} || '';
const requestedScenarios = ${{ toJSON(inputs.scenarios) }} || '';
const prNumberInput = process.env.JOB_PR_NUMBER || '';
const requestedScenarios = process.env.JOB_SCENARIOS || '';
const rawRequestedJobs = process.env.JOBS || '';
const jobsValidationPassed = needs['validate-jobs']?.result === 'success';
const requestedJobs = jobsValidationPassed ? rawRequestedJobs : '';
const jobsRejected = rawRequestedJobs && !jobsValidationPassed;

let prNumber = prNumberInput ? Number.parseInt(prNumberInput, 10) : undefined;
if (!prNumber) {
Expand Down Expand Up @@ -491,6 +546,11 @@ jobs:
requestedScenarios
? `**Requested scenarios:** \`${requestedScenarios}\``
: '**Requested scenarios:** _(default — all supported)_',
jobsRejected
? '**Requested jobs:** _(selector rejected by validate-jobs)_'
: requestedJobs
? `**Requested jobs:** \`${requestedJobs}\``
: '**Requested jobs:** _(default — all free-standing when no scenarios are requested)_',
`**Summary:** ${passed.length} passed, ${failed.length} failed, ${skipped.length} skipped`,
'',
'| Job | Result |',
Expand Down
46 changes: 42 additions & 4 deletions test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => {
permissions:
contents: read
jobs:
validate-jobs:
runs-on: macos-latest
steps:
- name: Validate free-standing job selector
env:
JOBS: bad
run: |
echo "::error::Invalid jobs input: \${JOBS}"
report-to-pr:
runs-on: ubuntu-latest
needs: [generate-matrix]
steps:
- name: Post Vitest scenario results to PR
env:
JOBS: bad
run: echo "\${{ inputs.pr_number }} \${{ inputs.scenarios }}"
live-scenarios:
runs-on: ubuntu-latest
env:
Expand Down Expand Up @@ -120,11 +136,21 @@ jobs:
expect(errors).toEqual(
expect.arrayContaining([
"workflow_dispatch missing input: scenarios",
"workflow_dispatch missing input: jobs",
"workflow_dispatch must not expose legacy test_filter input",
"validate-jobs job must run on ubuntu-latest",
"validate-jobs step must pass jobs through JOBS env",
"validate-jobs step must pass scenarios through SCENARIOS env",
"step 'Validate free-standing job selector' run script must include Use either scenarios or jobs, not both",
"step 'Validate free-standing job selector' run script must include allowed_jobs=",
"step 'Validate free-standing job selector' run script must include Invalid jobs input; use comma-separated job ids",
"step 'Validate free-standing job selector' run script must not include Invalid jobs input: ${JOBS}",
"step 'Validate free-standing job selector' run script must include Unknown free-standing Vitest job",
"workflow missing generate-matrix job",
"generate-matrix job must run on ubuntu-latest",
"live-scenarios job must run on the matrix runner",
"live-scenarios job must depend on generate-matrix",
"live-scenarios job must not run when a free-standing jobs selector is supplied",
"live-scenarios strategy.fail-fast must be false",
"live-scenarios matrix.include must come from generate-matrix output",
"live-scenarios job must write artifacts under e2e-artifacts/vitest",
Expand Down Expand Up @@ -153,8 +179,8 @@ jobs:
"artifact upload path must include e2e-artifacts/vitest/${{ matrix.id }}/shell/",
"artifact upload retention-days must be 14",
"upload-artifact action must be pinned to a full commit SHA",
"openshell-version-pin-vitest job must run independently of generate-matrix",
"openshell-version-pin-vitest job must run independently of workflow dispatch scenario filters",
"openshell-version-pin-vitest job must depend on validate-jobs",
"openshell-version-pin-vitest job must use the shared jobs selector condition",
"openshell-version-pin-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1",
"openshell-version-pin-vitest job must write artifacts under e2e-artifacts/vitest/openshell-version-pin",
"openshell-version-pin-vitest job env must not include NVIDIA_API_KEY",
Expand All @@ -172,8 +198,8 @@ jobs:
"openshell-version-pin-vitest artifact upload must set include-hidden-files: false",
"openshell-version-pin-vitest artifact upload must ignore missing fixture artifacts",
"openshell-version-pin-vitest artifact upload retention-days must be 14",
"onboard-negative-paths-vitest job must run independently of generate-matrix",
"onboard-negative-paths-vitest job must run independently of workflow dispatch scenario filters",
"onboard-negative-paths-vitest job must depend on validate-jobs",
"onboard-negative-paths-vitest job must use the shared jobs selector condition",
"onboard-negative-paths-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1",
"onboard-negative-paths-vitest job must write artifacts under e2e-artifacts/vitest/onboard-negative-paths",
"onboard-negative-paths-vitest job env must not include NVIDIA_API_KEY",
Expand All @@ -191,6 +217,18 @@ jobs:
"onboard-negative-paths-vitest artifact upload must set include-hidden-files: false",
"onboard-negative-paths-vitest artifact upload must ignore missing fixture artifacts",
"onboard-negative-paths-vitest artifact upload retention-days must be 14",
"openclaw-tui-chat-correlation-vitest job must depend on validate-jobs",
"openclaw-tui-chat-correlation-vitest job must use the shared jobs selector condition",
"gateway-guard-recovery job must depend on validate-jobs",
"gateway-guard-recovery job must use the shared jobs selector condition",
"report-to-pr job must wait for validate-jobs",
"report-to-pr job must wait for live-scenarios",
"report-to-pr step must pass pr_number through JOB_PR_NUMBER env",
"report-to-pr step must pass scenarios through JOB_SCENARIOS env",
"step 'Post Vitest scenario results to PR' run script must include process.env.JOBS",
"step 'Post Vitest scenario results to PR' run script must check validate-jobs before echoing jobs",
"step 'Post Vitest scenario results to PR' run script must omit rejected job selectors",
"step 'Post Vitest scenario results to PR' run script must include **Requested jobs:**",
]),
);
} finally {
Expand Down
Loading
Loading