diff --git a/.github/actions/setup-node-cache/action.yml b/.github/actions/setup-node-cache/action.yml index 33497d1..b335b11 100644 --- a/.github/actions/setup-node-cache/action.yml +++ b/.github/actions/setup-node-cache/action.yml @@ -29,7 +29,7 @@ runs: using: composite steps: - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ inputs.node-version }} cache: npm @@ -46,10 +46,10 @@ runs: npm --version npm sbom --help > /dev/null - - name: Check lockfile drift - shell: bash - run: npm run lockfile:check - + # Lockfile drift is validated once per workflow by a dedicated diagnostic gate + # (quality `lockfile-drift`, peer-matrix `lockfile`), not before every `npm ci`. + # `npm ci` already fails closed when package.json and package-lock.json diverge, + # so deterministic install is preserved without re-running the dry-run per job. - name: Install dependencies shell: bash run: npm ci @@ -67,14 +67,16 @@ runs: # ~114 MB; downloading it every run makes the job hostage to apt mirror speed # (a slow draw has cancelled whole shards at the job timeout). Caching the # archives makes the download a one-time cost -- later runs install from disk. - # run_id in the key means every run refreshes the cache; restore-keys reuses - # the most recent one, so package drift only ever costs an incremental fetch. + # Key is content-derived and bounded (lockfile + Playwright config hash), not + # per-run: `github.run_id` would create an unbounded entry every run and cause + # same-run matrix save collisions. restore-keys falls back to the newest + # lane entry, so apt-version drift only ever costs an incremental fetch. - name: Cache Playwright apt dependencies if: ${{ inputs.install-browsers == 'true' }} uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/playwright-apt-debs - key: ${{ runner.os }}-pw-aptdebs-${{ inputs.cache-version }}-${{ inputs.browser-name }}-${{ github.run_id }} + key: ${{ runner.os }}-pw-aptdebs-${{ inputs.cache-version }}-${{ inputs.browser-name }}-${{ hashFiles('package-lock.json', 'configs/playwright.config.ts') }} restore-keys: | ${{ runner.os }}-pw-aptdebs-${{ inputs.cache-version }}-${{ inputs.browser-name }}- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7eb131..8e5472a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,12 @@ name: E2E Matrix +# Exhaustive daily cross-browser E2E validation feeding flakiness and SLO trends. +# Post-merge Chrome coverage is provided once by the Quality Gates `E2E (Chrome)` +# lane; this workflow runs the full browser matrix on a daily schedule (and manual +# dispatch) rather than on every main push. Trigger it manually after a risky merge +# if cross-browser confirmation is needed before the next daily run. + on: - push: - branches: [main] schedule: - cron: '0 3 * * *' workflow_dispatch: @@ -19,43 +23,41 @@ env: jobs: e2e: - name: Full E2E (${{ matrix.project }} shard ${{ matrix.shard }}) + name: Full E2E (${{ matrix.lane }}) runs-on: ubuntu-latest - # 30 min (was 20): webkit --with-deps can pull ~114 MB from a slow apt mirror; - # the setup action now retries+resumes that download, which needs headroom - # beyond the test run itself. + # 30 min: webkit --with-deps can pull ~114 MB from a slow apt mirror; the setup + # action retries+resumes that download, which needs headroom beyond the tests. timeout-minutes: 30 strategy: fail-fast: false - max-parallel: 6 + max-parallel: 5 + # One job per installed browser binary (Playwright is already fully parallel + # across a project's tests, so browser/dependency setup -- not test time -- + # dominates a job). Desktop + mobile WebKit share one webkit install in a + # single job without losing per-project results. Two-way sharding was removed: + # measured shard wall-clock was setup, not tests, so it only doubled setup. matrix: - project: - - Google Chrome - - Firefox - - Safari - - Microsoft Edge - - Mobile Chrome - - Mobile Safari - shard: [1, 2] include: - - project: Google Chrome + - lane: Google Chrome install_args: chrome - project_slug: google-chrome - - project: Firefox + slug: google-chrome + projects: 'Google Chrome' + - lane: Firefox install_args: firefox - project_slug: firefox - - project: Safari - install_args: webkit - project_slug: safari - - project: Microsoft Edge + slug: firefox + projects: 'Firefox' + - lane: Microsoft Edge install_args: msedge - project_slug: microsoft-edge - - project: Mobile Chrome + slug: microsoft-edge + projects: 'Microsoft Edge' + - lane: Mobile Chrome install_args: chromium - project_slug: mobile-chrome - - project: Mobile Safari + slug: mobile-chrome + projects: 'Mobile Chrome' + - lane: 'WebKit (Safari + Mobile Safari)' install_args: webkit - project_slug: mobile-safari + slug: webkit + projects: 'Safari;Mobile Safari' steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -73,14 +75,23 @@ jobs: - name: Run full E2E suite env: - PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/playwright-results-${{ matrix.project_slug }}-shard-${{ matrix.shard }}.json - run: npx playwright test --config=configs/playwright.config.ts --project="${{ matrix.project }}" --shard=${{ matrix.shard }}/2 + PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/playwright-results-${{ matrix.slug }}.json + PLAYWRIGHT_PROJECTS: ${{ matrix.projects }} + # Split the semicolon-delimited project list into safely-quoted --project + # flags so multi-project lanes (WebKit) handle names with spaces correctly. + run: | + IFS=';' read -ra projects <<< "${PLAYWRIGHT_PROJECTS}" + args=() + for project in "${projects[@]}"; do + args+=(--project "${project}") + done + npx playwright test --config=configs/playwright.config.ts "${args[@]}" - name: Upload E2E artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: - name: e2e-matrix-artifacts-${{ matrix.project }}-shard-${{ matrix.shard }} + name: e2e-matrix-artifacts-${{ matrix.slug }} retention-days: 14 path: | test-results/ diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml deleted file mode 100644 index fe86953..0000000 --- a/.github/workflows/examples.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Examples - -on: - pull_request: - branches: [main] - push: - branches: [main] - -concurrency: - group: examples-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -permissions: - contents: read - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' - -jobs: - preflight: - name: Examples Preflight - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - outputs: - run_examples: ${{ steps.paths-filter.outputs.examples }} - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Detect examples-relevant changes - id: paths-filter - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 - with: - filters: | - examples: - - 'examples/**' - - 'tests/suites/e2e/examples/**' - - 'configs/playwright.config.ts' - - 'package.json' - - 'package-lock.json' - - '.github/workflows/examples.yml' - - examples: - name: Examples - runs-on: ubuntu-latest - needs: preflight - timeout-minutes: 8 - permissions: - contents: read - env: - SHOULD_RUN_EXAMPLES: ${{ github.ref == 'refs/heads/main' || needs.preflight.outputs.run_examples == 'true' }} - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup locked Node.js dependencies and Playwright browser - if: env.SHOULD_RUN_EXAMPLES == 'true' - uses: ./.github/actions/setup-node-cache - with: - node-version: '22' - install-browsers: 'true' - browser-name: chrome - cache-namespace: examples-chrome - cache-version: v1 - - - name: Run examples suite - if: env.SHOULD_RUN_EXAMPLES == 'true' - run: npm run test:examples - - - name: Upload examples artifacts - if: env.SHOULD_RUN_EXAMPLES == 'true' && always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: examples-artifacts - retention-days: 7 - path: | - test-results/ - test-reports/ - - - name: Skip examples lane when no relevant changes - if: env.SHOULD_RUN_EXAMPLES != 'true' - run: echo "No examples-relevant changes detected; examples lane completed as no-op." diff --git a/.github/workflows/observability-full-stack.yml b/.github/workflows/observability-full-stack.yml new file mode 100644 index 0000000..73b8a3c --- /dev/null +++ b/.github/workflows/observability-full-stack.yml @@ -0,0 +1,148 @@ +name: Observability Full Stack + +# Weekly reference-stack validation of the seven-service local observability stack +# (collector, Prometheus, Grafana, Jaeger, Elasticsearch, Logstash, Kibana). This +# validates local/reference infrastructure, dashboards, provisioning, and backend +# semantics -- NOT a supported production service. It is intentionally never a PR +# gate: with only schedule + manual triggers, no runner is allocated on pull +# requests. See docs/operations/observability-support-tiers.md. + +on: + schedule: + - cron: '0 6 * * 0' + workflow_dispatch: + +concurrency: + group: observability-full-stack-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + full-stack-smoke: + name: Full Stack Smoke + runs-on: ubuntu-latest + if: vars.AURORAFLOW_OBSERVABILITY_FULL_STACK_CI_ENABLED != 'false' + timeout-minutes: 15 + permissions: + contents: read + env: + OBSERVABILITY_DIAGNOSTICS_DIR: observability-output/full-stack + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup locked Node.js dependencies + uses: ./.github/actions/setup-node-cache + with: + node-version: '22' + cache-namespace: observability-full-stack + cache-version: v1 + + - name: Validate full-stack Compose config + run: docker compose -f docker-compose.yml -f docker-compose.observability.yml config + + - name: Start full observability stack + run: | + mkdir -p logs test-results/self-healing "${OBSERVABILITY_DIAGNOSTICS_DIR}" + printf '{"timestamp":"%s","level":"info","message":"Observability full-stack smoke log seed.","component":"observability-ci","runId":"%s","workflow":"%s","branch":"%s","commit":"%s"}\n' \ + "$(date --utc +%Y-%m-%dT%H:%M:%SZ)" \ + "${GITHUB_RUN_ID}" \ + "${GITHUB_WORKFLOW}" \ + "${GITHUB_REF_NAME}" \ + "${GITHUB_SHA}" > logs/auroraflow.ndjson + docker compose -f docker-compose.yml -f docker-compose.observability.yml up -d + + - name: Wait for full-stack backends + run: | + npm run observability:validate -- \ + --mode readiness \ + --output-dir "${OBSERVABILITY_DIAGNOSTICS_DIR}" \ + --max-attempts 60 \ + --poll-interval-ms 3000 \ + --timeout-ms 15000 + + - name: Provision Elasticsearch and Kibana assets + run: | + curl --fail --silent --show-error -X PUT http://localhost:9200/_ilm/policy/auroraflow-local-retention \ + -H 'Content-Type: application/json' \ + --data-binary @observability/elastic/ilm/auroraflow-local-retention.json + + for template in observability/elastic/index-templates/*.json; do + name="$(basename "${template}" .json)" + curl --fail --silent --show-error -X PUT "http://localhost:9200/_index_template/${name}" \ + -H 'Content-Type: application/json' \ + --data-binary "@${template}" + done + + curl --fail --silent --show-error -X POST http://localhost:5601/api/saved_objects/_import?overwrite=true \ + -H 'kbn-xsrf: auroraflow' \ + --form file=@observability/kibana/saved-objects/auroraflow-log-exploration.ndjson + + - name: Emit full-stack smoke telemetry + env: + AURORAFLOW_OBSERVABILITY_DIAGNOSTICS_DIR: ${{ env.OBSERVABILITY_DIAGNOSTICS_DIR }} + AURORAFLOW_OBSERVABILITY_ENABLED: 'true' + AURORAFLOW_OBSERVABILITY_ENVIRONMENT: ci + OTEL_EXPORTER_OTLP_ENDPOINT: http://localhost:4318 + run: npm run observability:ci:smoke + + - name: Emit full-stack smoke log through Logstash + run: | + curl --fail --silent --show-error -X POST http://localhost:8080 \ + -H 'Content-Type: application/json' \ + --data-binary @- < "${OBSERVABILITY_DIAGNOSTICS_DIR}/compose-ps.txt" || true + docker compose -f docker-compose.yml -f docker-compose.observability.yml logs --no-color > "${OBSERVABILITY_DIAGNOSTICS_DIR}/compose.log" || true + + - name: Stop full observability stack + if: always() + run: docker compose -f docker-compose.yml -f docker-compose.observability.yml down --remove-orphans --volumes + + - name: Upload full-stack diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: observability-full-stack-diagnostics + retention-days: 7 + path: | + observability-output/full-stack/ + logs/auroraflow.ndjson diff --git a/.github/workflows/observability-lite.yml b/.github/workflows/observability-lite.yml new file mode 100644 index 0000000..9c4f9e3 --- /dev/null +++ b/.github/workflows/observability-lite.yml @@ -0,0 +1,119 @@ +name: Observability Lite Smoke + +# Collector-only OTLP/HTTP Lite smoke. Proves real metric/trace export, optional +# OTLP log emission, collector health, and a run-scoped collector receipt against +# the local reference endpoint http://localhost:4318 (never a guessed remote). +# +# Runs: called by Quality Gates on observability-relevant changes (workflow_call), +# on a daily scheduled floor, and on manual dispatch. It is intentionally NOT a +# routine gate on unrelated pushes. Remote export requires an owned OTLP backend, +# secrets, and receipt proof; see docs/operations/observability-production.md. + +on: + workflow_call: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +concurrency: + group: observability-lite-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + collector-lite-smoke: + name: Collector Lite Smoke + runs-on: ubuntu-latest + # Non-secret enable variable escape hatch (defaults on when unset). + if: vars.AURORAFLOW_OBSERVABILITY_CI_ENABLED != 'false' + timeout-minutes: 8 + permissions: + contents: read + env: + OBSERVABILITY_DIAGNOSTICS_DIR: observability-output/ci + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup locked Node.js dependencies + uses: ./.github/actions/setup-node-cache + with: + node-version: '22' + cache-namespace: observability-lite + cache-version: v1 + + - name: Validate Lite collector-only Compose config + run: docker compose -f docker-compose.observability-ci.yml config + + - name: Start Lite collector-only stack + run: docker compose -f docker-compose.observability-ci.yml up -d + + - name: Wait for collector health + run: | + for attempt in $(seq 1 30); do + if curl --fail --silent --show-error http://localhost:13133/ >/dev/null; then + exit 0 + fi + echo "Waiting for collector health (${attempt}/30)" + sleep 2 + done + docker compose -f docker-compose.observability-ci.yml logs otel-collector + exit 1 + + - name: Emit smoke telemetry + env: + AURORAFLOW_OBSERVABILITY_DIAGNOSTICS_DIR: ${{ env.OBSERVABILITY_DIAGNOSTICS_DIR }} + AURORAFLOW_OBSERVABILITY_EMIT_OTLP_LOG: 'true' + AURORAFLOW_OBSERVABILITY_ENABLED: 'true' + AURORAFLOW_OBSERVABILITY_ENVIRONMENT: ci + AURORAFLOW_OBSERVABILITY_STRICT: 'true' + OTEL_EXPORTER_OTLP_ENDPOINT: http://localhost:4318 + run: npm run observability:ci:smoke + + - name: Assert Lite collector receipt + run: | + mkdir -p "${OBSERVABILITY_DIAGNOSTICS_DIR}" + for attempt in $(seq 1 15); do + curl --fail --silent --show-error http://localhost:9464/metrics > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector-metrics.txt" || true + docker compose -f docker-compose.observability-ci.yml logs --no-color otel-collector > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector.log" || true + if npm run observability:collector-receipt -- \ + --metrics-path "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector-metrics.txt" \ + --collector-log-path "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector.log" \ + --output-dir "${OBSERVABILITY_DIAGNOSTICS_DIR}"; then + exit 0 + fi + echo "Waiting for Collector receipt evidence (${attempt}/15)" + sleep 2 + done + echo "Collector receipt evidence did not become complete." + exit 1 + + - name: Capture observability diagnostics + if: always() + run: | + mkdir -p "${OBSERVABILITY_DIAGNOSTICS_DIR}" + curl --silent --show-error http://localhost:13133/ > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector-health.txt" || true + curl --silent --show-error http://localhost:9464/metrics > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector-metrics.txt" || true + docker compose -f docker-compose.observability-ci.yml ps > "${OBSERVABILITY_DIAGNOSTICS_DIR}/compose-ps.txt" || true + docker compose -f docker-compose.observability-ci.yml logs --no-color otel-collector > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector.log" || true + + - name: Stop Lite collector-only stack + if: always() + run: docker compose -f docker-compose.observability-ci.yml down --remove-orphans + + - name: Upload observability diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: observability-collector-diagnostics + retention-days: 7 + path: | + observability-output/ci/ + logs/auroraflow.ndjson diff --git a/.github/workflows/playwright-peer-matrix.yml b/.github/workflows/playwright-peer-matrix.yml index 070f9f3..760a041 100644 --- a/.github/workflows/playwright-peer-matrix.yml +++ b/.github/workflows/playwright-peer-matrix.yml @@ -17,9 +17,30 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' jobs: + lockfile: + name: Lockfile Drift + runs-on: ubuntu-latest + timeout-minutes: 3 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + + - name: Check lockfile drift + run: npm run lockfile:check + peer-matrix: name: Playwright ${{ matrix.lane }} runs-on: ubuntu-latest + needs: lockfile timeout-minutes: 20 strategy: fail-fast: false @@ -44,9 +65,8 @@ jobs: node-version: '22' cache: npm - - name: Check lockfile drift - run: npm run lockfile:check - + # Lockfile drift is validated once in the `lockfile` pre-job before fan-out, + # not per matrix lane. npm ci still fails closed on any lock/manifest drift. - name: Install locked dependencies run: npm ci diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 06c9876..a38a4bf 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -27,17 +27,16 @@ jobs: contents: read pull-requests: read outputs: + run_browser_e2e: ${{ steps.paths-filter.outputs.browser_e2e }} + run_observability: ${{ steps.paths-filter.outputs.observability }} run_lockfile_check: ${{ steps.paths-filter.outputs.lockfile_check }} - run_observability_stack: ${{ steps.paths-filter.outputs.observability_stack }} - run_risk_e2e: ${{ steps.paths-filter.outputs.risk_e2e }} - run_smoke: ${{ steps.paths-filter.outputs.smoke }} steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - name: Detect smoke-relevant changes + - name: Detect change scopes id: paths-filter if: github.event_name == 'pull_request' || github.event_name == 'push' uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 @@ -50,27 +49,20 @@ jobs: - '.github/actions/setup-node-cache/action.yml' - 'scripts/check-lockfile-drift.mjs' - '.github/workflows/quality.yml' - smoke: + browser_e2e: - 'src/**' - 'tests/suites/e2e/**' - 'tests/fixtures/e2e-app/**' + - 'examples/**' - 'configs/playwright.config.ts' - 'scripts/e2e-fixture-server.mjs' - 'package.json' - 'package-lock.json' - '.github/workflows/quality.yml' - risk_e2e: - - 'src/pageObjects/**' - - 'src/framework/selfHealing/**' - - 'tests/suites/e2e/**' - - 'tests/fixtures/e2e-app/**' - - 'configs/playwright.config.ts' - - 'scripts/e2e-fixture-server.mjs' - - 'package.json' - - 'package-lock.json' - - '.github/workflows/quality.yml' - observability_stack: + - '.github/actions/setup-node-cache/action.yml' + observability: - '.github/workflows/quality.yml' + - '.github/workflows/observability-lite.yml' - 'docker-compose.observability*.yml' - 'docs/adr/0007-durable-trend-export.md' - 'docs/architecture/observability-stack.md' @@ -81,21 +73,17 @@ jobs: - 'scripts/observability-*.ts' - 'src/framework/observability/**' - 'tests/suites/unit/framework/observability/**' + - 'tests/suites/contracts/observability/**' - 'package.json' - 'package-lock.json' - - 'scripts/observability-ci-smoke.ts' - - 'scripts/observability-collector-receipt.ts' - - 'src/framework/observability/**' - - 'tests/suites/contracts/observability/**' - - - name: Skip path filtering for scheduled or manual runs - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - run: echo "Scheduled/manual run executes repository and heavy observability gates directly." lockfile-drift: name: Lockfile Drift runs-on: ubuntu-latest needs: preflight + # The single explicit lockfile diagnostic gate for this workflow. The shared + # setup action no longer dry-runs the lockfile before every `npm ci`; this + # clearer-diagnostic gate runs once when the lockfile could have moved. if: github.ref == 'refs/heads/main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.preflight.outputs.run_lockfile_check == 'true' || startsWith(github.head_ref, 'dependabot/') timeout-minutes: 3 permissions: @@ -114,39 +102,8 @@ jobs: - name: Check package lockfile drift run: npm run lockfile:check - verify: - name: Node Compatibility (Node ${{ matrix.node-version }}) - runs-on: ubuntu-latest - permissions: - contents: read - timeout-minutes: 5 - needs: preflight - strategy: - fail-fast: false - max-parallel: 2 - matrix: - node-version: [20, 22, 24] - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup locked Node.js dependencies - uses: ./.github/actions/setup-node-cache - with: - node-version: ${{ matrix.node-version }} - cache-namespace: node-${{ matrix.node-version }} - cache-version: v1 - - - name: Run Node compatibility static gates - run: npm run lint && npm run typecheck - - - name: Run unit tests - run: npm test - - repository-gates: - name: Repository Gates (Node 22) + static-analysis: + name: Static Analysis (Node 22) runs-on: ubuntu-latest permissions: contents: read @@ -162,7 +119,7 @@ jobs: uses: ./.github/actions/setup-node-cache with: node-version: '22' - cache-namespace: repository-gates + cache-namespace: static-analysis cache-version: v1 - name: Install shellcheck @@ -174,6 +131,12 @@ jobs: - name: Run format check run: npm run format:check + - name: Run lint + run: npm run lint + + - name: Run typecheck + run: npm run typecheck + - name: Run contract tests run: npm run test:contracts @@ -186,10 +149,42 @@ jobs: run: npm run schemas:check - name: Run shell and workflow lint - run: npm run shellcheck && npm run workflows:lint + # workflows:lint:check reuses the actionlint installed above instead of + # reinstalling it (workflows:lint would re-bootstrap the binary). + run: npm run shellcheck && npm run workflows:lint:check + + node-compat: + name: Node Compatibility (Node ${{ matrix.node-version }}) + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 5 + needs: preflight + strategy: + fail-fast: false + max-parallel: 3 + matrix: + node-version: [20, 22, 24] + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup locked Node.js dependencies + uses: ./.github/actions/setup-node-cache + with: + node-version: ${{ matrix.node-version }} + cache-namespace: node-${{ matrix.node-version }} + cache-version: v1 + + - name: Run unit tests + # Runtime-sensitive signal only. Static gates (lint/typecheck/format/schema/ + # workflow lint) run once on Node 22 in static-analysis, not per Node version. + run: npm test coverage: - name: Coverage (Critical + Global) + name: Coverage (Unit Floors) runs-on: ubuntu-latest needs: preflight timeout-minutes: 8 @@ -208,7 +203,10 @@ jobs: cache-namespace: coverage cache-version: v1 - - name: Enforce critical and global coverage thresholds + - name: Enforce global and risk-weighted coverage thresholds + # Single authoritative coverage-floor run over the complete unit suite. + # Global aggregate + every risk-weighted per-file floor is enforced here; + # the previous critical-only pass was a strict subset with identical floors. run: | set -o pipefail mkdir -p coverage @@ -222,7 +220,8 @@ jobs: echo echo "Command: \`npm run test:coverage\`" echo - echo "Critical-module and global coverage thresholds are enforced by this required job." + echo "One complete unit coverage run enforces the global aggregate floor and every" + echo "risk-weighted per-file floor (configs/vitest.coverage-global.mts)." echo echo "\`\`\`text" if [ -f coverage/coverage-gate.log ]; then @@ -267,380 +266,16 @@ jobs: retention-days: 14 path: mutation-output/baseline-check.txt - guarded-self-heal: - name: Guarded Self-Heal Proof (Chrome) - runs-on: ubuntu-latest - needs: [preflight, verify, repository-gates] - timeout-minutes: 8 - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup locked Node.js dependencies and Playwright browser - uses: ./.github/actions/setup-node-cache - with: - node-version: '22' - install-browsers: 'true' - browser-name: chrome - cache-namespace: guarded-chrome - cache-version: v1 - - - name: Prove guarded self-heal at default gate - run: npm run test:e2e:guarded - - - name: Upload guarded self-heal artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: guarded-self-heal-proof - retention-days: 7 - path: | - test-results/ - test-reports/ - - risk-e2e: - name: Risk-Triggered E2E (Chrome) - runs-on: ubuntu-latest - needs: [preflight, verify, repository-gates] - timeout-minutes: 20 - permissions: - contents: read - env: - SHOULD_RUN_RISK_E2E: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.preflight.outputs.run_risk_e2e == 'true' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-e2e')) || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'risk:e2e')) }} - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup locked Node.js dependencies and Playwright browser - if: env.SHOULD_RUN_RISK_E2E == 'true' - uses: ./.github/actions/setup-node-cache - with: - node-version: '22' - install-browsers: 'true' - browser-name: chrome - cache-namespace: risk-chrome - cache-version: v1 - - - name: Run full Chrome E2E suite - if: env.SHOULD_RUN_RISK_E2E == 'true' - run: npm run test:e2e -- --project='Google Chrome' - - - name: Upload risk E2E artifacts - if: env.SHOULD_RUN_RISK_E2E == 'true' && always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: risk-e2e-artifacts - retention-days: 7 - path: | - test-results/ - test-reports/ - - - name: Skip risk-triggered E2E when not requested - if: env.SHOULD_RUN_RISK_E2E != 'true' - run: echo "No risky browser/runtime changes or full-e2e label detected; risk E2E completed as no-op." - - observability-stack: - name: Observability Lite Smoke - runs-on: ubuntu-latest - needs: [preflight, verify, repository-gates] - timeout-minutes: 8 - permissions: - contents: read - env: - SHOULD_RUN_OBSERVABILITY: ${{ github.ref == 'refs/heads/main' || (needs.preflight.outputs.run_observability_stack == 'true' && vars.AURORAFLOW_OBSERVABILITY_CI_ENABLED != 'false') }} - OBSERVABILITY_DIAGNOSTICS_DIR: observability-output/ci - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup locked Node.js dependencies - if: env.SHOULD_RUN_OBSERVABILITY == 'true' - uses: ./.github/actions/setup-node-cache - with: - node-version: '22' - cache-namespace: observability-lite - cache-version: v1 - - - name: Validate Lite collector-only Compose config - if: env.SHOULD_RUN_OBSERVABILITY == 'true' - run: docker compose -f docker-compose.observability-ci.yml config - - - name: Start Lite collector-only stack - if: env.SHOULD_RUN_OBSERVABILITY == 'true' - run: docker compose -f docker-compose.observability-ci.yml up -d - - - name: Wait for collector health - if: env.SHOULD_RUN_OBSERVABILITY == 'true' - run: | - for attempt in $(seq 1 30); do - if curl --fail --silent --show-error http://localhost:13133/ >/dev/null; then - exit 0 - fi - echo "Waiting for collector health (${attempt}/30)" - sleep 2 - done - docker compose -f docker-compose.observability-ci.yml logs otel-collector - exit 1 - - - name: Emit smoke telemetry - if: env.SHOULD_RUN_OBSERVABILITY == 'true' - env: - AURORAFLOW_OBSERVABILITY_DIAGNOSTICS_DIR: ${{ env.OBSERVABILITY_DIAGNOSTICS_DIR }} - AURORAFLOW_OBSERVABILITY_EMIT_OTLP_LOG: 'true' - AURORAFLOW_OBSERVABILITY_ENABLED: 'true' - AURORAFLOW_OBSERVABILITY_ENVIRONMENT: ci - AURORAFLOW_OBSERVABILITY_STRICT: 'true' - OTEL_EXPORTER_OTLP_ENDPOINT: http://localhost:4318 - run: npm run observability:ci:smoke - - - name: Assert Lite collector receipt - if: env.SHOULD_RUN_OBSERVABILITY == 'true' - run: | - mkdir -p "${OBSERVABILITY_DIAGNOSTICS_DIR}" - for attempt in $(seq 1 15); do - curl --fail --silent --show-error http://localhost:9464/metrics > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector-metrics.txt" || true - docker compose -f docker-compose.observability-ci.yml logs --no-color otel-collector > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector.log" || true - if npm run observability:collector-receipt -- \ - --metrics-path "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector-metrics.txt" \ - --collector-log-path "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector.log" \ - --output-dir "${OBSERVABILITY_DIAGNOSTICS_DIR}"; then - exit 0 - fi - echo "Waiting for Collector receipt evidence (${attempt}/15)" - sleep 2 - done - echo "Collector receipt evidence did not become complete." - exit 1 - - - name: Capture observability diagnostics - if: env.SHOULD_RUN_OBSERVABILITY == 'true' && always() - run: | - mkdir -p "${OBSERVABILITY_DIAGNOSTICS_DIR}" - curl --silent --show-error http://localhost:13133/ > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector-health.txt" || true - curl --silent --show-error http://localhost:9464/metrics > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector-metrics.txt" || true - docker compose -f docker-compose.observability-ci.yml ps > "${OBSERVABILITY_DIAGNOSTICS_DIR}/compose-ps.txt" || true - docker compose -f docker-compose.observability-ci.yml logs --no-color otel-collector > "${OBSERVABILITY_DIAGNOSTICS_DIR}/collector.log" || true - - - name: Stop Lite collector-only stack - if: env.SHOULD_RUN_OBSERVABILITY == 'true' && always() - run: docker compose -f docker-compose.observability-ci.yml down --remove-orphans - - - name: Upload observability diagnostics - if: env.SHOULD_RUN_OBSERVABILITY == 'true' && always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: observability-collector-diagnostics - retention-days: 7 - path: | - observability-output/ci/ - logs/auroraflow.ndjson - - - name: Skip observability collector smoke when not requested - if: env.SHOULD_RUN_OBSERVABILITY != 'true' - run: echo "No observability-relevant changes detected; Lite collector smoke completed as no-op." - - observability-full-stack: - name: Observability Full Stack Smoke - runs-on: ubuntu-latest - needs: [preflight, verify, repository-gates] - timeout-minutes: 15 - permissions: - contents: read - env: - AURORAFLOW_OBSERVABILITY_FULL_STACK_CI_ENABLED: 'true' - SHOULD_RUN_OBSERVABILITY_FULL_STACK: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} - OBSERVABILITY_DIAGNOSTICS_DIR: observability-output/full-stack - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup locked Node.js dependencies - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK == 'true' - uses: ./.github/actions/setup-node-cache - with: - node-version: '22' - cache-namespace: observability-full-stack - cache-version: v1 - - - name: Validate full-stack Compose config - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK == 'true' - run: docker compose -f docker-compose.yml -f docker-compose.observability.yml config - - - name: Start full observability stack - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK == 'true' - run: | - mkdir -p logs test-results/self-healing "${OBSERVABILITY_DIAGNOSTICS_DIR}" - printf '{"timestamp":"%s","level":"info","message":"Observability full-stack smoke log seed.","component":"observability-ci","runId":"%s","workflow":"%s","branch":"%s","commit":"%s"}\n' \ - "$(date --utc +%Y-%m-%dT%H:%M:%SZ)" \ - "${GITHUB_RUN_ID}" \ - "${GITHUB_WORKFLOW}" \ - "${GITHUB_REF_NAME}" \ - "${GITHUB_SHA}" > logs/auroraflow.ndjson - docker compose -f docker-compose.yml -f docker-compose.observability.yml up -d - - - name: Wait for full-stack backends - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK == 'true' - run: | - npm run observability:validate -- \ - --mode readiness \ - --output-dir "${OBSERVABILITY_DIAGNOSTICS_DIR}" \ - --max-attempts 60 \ - --poll-interval-ms 3000 \ - --timeout-ms 15000 - - - name: Provision Elasticsearch and Kibana assets - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK == 'true' - run: | - curl --fail --silent --show-error -X PUT http://localhost:9200/_ilm/policy/auroraflow-local-retention \ - -H 'Content-Type: application/json' \ - --data-binary @observability/elastic/ilm/auroraflow-local-retention.json - - for template in observability/elastic/index-templates/*.json; do - name="$(basename "${template}" .json)" - curl --fail --silent --show-error -X PUT "http://localhost:9200/_index_template/${name}" \ - -H 'Content-Type: application/json' \ - --data-binary "@${template}" - done - - curl --fail --silent --show-error -X POST http://localhost:5601/api/saved_objects/_import?overwrite=true \ - -H 'kbn-xsrf: auroraflow' \ - --form file=@observability/kibana/saved-objects/auroraflow-log-exploration.ndjson - - - name: Emit full-stack smoke telemetry - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK == 'true' - env: - AURORAFLOW_OBSERVABILITY_DIAGNOSTICS_DIR: ${{ env.OBSERVABILITY_DIAGNOSTICS_DIR }} - AURORAFLOW_OBSERVABILITY_ENABLED: 'true' - AURORAFLOW_OBSERVABILITY_ENVIRONMENT: ci - OTEL_EXPORTER_OTLP_ENDPOINT: http://localhost:4318 - run: npm run observability:ci:smoke - - - name: Emit full-stack smoke log through Logstash - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK == 'true' - run: | - curl --fail --silent --show-error -X POST http://localhost:8080 \ - -H 'Content-Type: application/json' \ - --data-binary @- < "${OBSERVABILITY_DIAGNOSTICS_DIR}/compose-ps.txt" || true - docker compose -f docker-compose.yml -f docker-compose.observability.yml logs --no-color > "${OBSERVABILITY_DIAGNOSTICS_DIR}/compose.log" || true - - - name: Stop full observability stack - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK == 'true' && always() - run: docker compose -f docker-compose.yml -f docker-compose.observability.yml down --remove-orphans --volumes - - - name: Upload full-stack diagnostics - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK == 'true' && always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: observability-full-stack-diagnostics - retention-days: 7 - path: | - observability-output/full-stack/ - logs/auroraflow.ndjson - - - name: Skip full-stack smoke when not requested - if: env.SHOULD_RUN_OBSERVABILITY_FULL_STACK != 'true' - run: echo "No observability-relevant changes detected; full-stack smoke completed as no-op." - - observability-remote-export: - name: Observability Remote Export Smoke + e2e-chrome: + name: E2E (Chrome) runs-on: ubuntu-latest - needs: [preflight, verify, repository-gates] - timeout-minutes: 8 - permissions: - contents: read - env: - AURORAFLOW_OBSERVABILITY_REMOTE_EXPORT_ENABLED: 'true' - SHOULD_RUN_REMOTE_EXPORT: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} - OBSERVABILITY_DIAGNOSTICS_DIR: observability-output/remote-export - OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }} - OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup locked Node.js dependencies - if: env.SHOULD_RUN_REMOTE_EXPORT == 'true' && env.OTEL_EXPORTER_OTLP_ENDPOINT != '' - uses: ./.github/actions/setup-node-cache - with: - node-version: '22' - cache-namespace: observability-remote-export - cache-version: v1 - - - name: Emit remote smoke telemetry - if: env.SHOULD_RUN_REMOTE_EXPORT == 'true' && env.OTEL_EXPORTER_OTLP_ENDPOINT != '' - env: - AURORAFLOW_OBSERVABILITY_DIAGNOSTICS_DIR: ${{ env.OBSERVABILITY_DIAGNOSTICS_DIR }} - AURORAFLOW_OBSERVABILITY_ENABLED: 'true' - AURORAFLOW_OBSERVABILITY_ENVIRONMENT: ci - AURORAFLOW_OBSERVABILITY_STRICT: 'true' - run: npm run observability:ci:smoke - - - name: Upload remote export diagnostics - if: env.SHOULD_RUN_REMOTE_EXPORT == 'true' && env.OTEL_EXPORTER_OTLP_ENDPOINT != '' && always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: observability-remote-export-diagnostics - retention-days: 7 - path: | - observability-output/remote-export/smoke-result.json - logs/auroraflow.ndjson - - - name: Skip remote export when not configured - if: env.SHOULD_RUN_REMOTE_EXPORT != 'true' || env.OTEL_EXPORTER_OTLP_ENDPOINT == '' - run: echo "Remote observability export smoke skipped because no observability-relevant changes were detected or the OTLP endpoint secret is not configured." - - smoke-e2e: - name: Smoke E2E (Chrome) - runs-on: ubuntu-latest - needs: [preflight, verify, repository-gates] - timeout-minutes: 8 + needs: [preflight, static-analysis, node-compat] + timeout-minutes: 12 + # One event-aware Chrome lane. The full Chrome project (14 tests) is the union + # of the old smoke (@smoke), guarded self-heal, examples, and risk subsets, so + # running it once means no Playwright test ID executes twice on the same event. + # Job-level gate: no runner is allocated for irrelevant changes. + if: needs.preflight.outputs.run_browser_e2e == 'true' || github.ref == 'refs/heads/main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-e2e')) || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'risk:e2e')) outputs: self_heal_triage_required: ${{ steps.self-heal-governance.outputs.triage_required }} self_heal_acknowledged: ${{ steps.self-heal-governance.outputs.acknowledged }} @@ -649,8 +284,6 @@ jobs: permissions: contents: read issues: write - env: - SHOULD_RUN_SMOKE: ${{ github.ref == 'refs/heads/main' || needs.preflight.outputs.run_smoke == 'true' }} steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -658,22 +291,21 @@ jobs: persist-credentials: false - name: Setup locked Node.js dependencies and Playwright browser - if: env.SHOULD_RUN_SMOKE == 'true' uses: ./.github/actions/setup-node-cache with: node-version: '22' install-browsers: 'true' browser-name: chrome - cache-namespace: smoke-chrome + cache-namespace: e2e-chrome cache-version: v1 - - name: Run smoke test - if: env.SHOULD_RUN_SMOKE == 'true' - run: npm run test:smoke + - name: Run full Chrome E2E suite + # Full Chrome project: smoke + guarded self-heal + examples + SAT, each ID once. + run: npm run test:e2e -- --project='Google Chrome' - name: Evaluate self-healing governance id: self-heal-governance - if: env.SHOULD_RUN_SMOKE == 'true' && always() + if: always() env: SELF_HEAL_ARTIFACTS_DIR: test-results/self-healing SELF_HEAL_REQUIRE_ACK_FOR_ACCEPTED: 'true' @@ -683,7 +315,7 @@ jobs: run: npm run self-heal:governance - name: Auto-open self-healing triage issue - if: env.SHOULD_RUN_SMOKE == 'true' && always() && github.ref == 'refs/heads/main' && vars.SELF_HEAL_AUTO_OPEN_TRIAGE_ISSUE == 'true' && steps.self-heal-governance.outputs.triage_required == 'true' && steps.self-heal-governance.outputs.acknowledged != 'true' + if: always() && github.ref == 'refs/heads/main' && vars.SELF_HEAL_AUTO_OPEN_TRIAGE_ISSUE == 'true' && steps.self-heal-governance.outputs.triage_required == 'true' && steps.self-heal-governance.outputs.acknowledged != 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} @@ -707,16 +339,122 @@ jobs: --title "Self-healing guarded acceptance review (${GITHUB_SHA::7})" \ --body-file "${SUMMARY_MARKDOWN_PATH}" - - name: Upload smoke artifacts - if: env.SHOULD_RUN_SMOKE == 'true' && always() + - name: Upload E2E artifacts + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: smoke-e2e-artifacts + name: e2e-chrome-artifacts retention-days: 7 path: | test-results/ test-reports/ - - name: Skip smoke lane when no relevant changes - if: env.SHOULD_RUN_SMOKE != 'true' - run: echo "No smoke-relevant changes detected; smoke lane completed as no-op." + observability-lite: + name: Observability Lite Smoke + needs: [preflight, static-analysis] + # Runs only for observability-relevant changes. A scheduled daily floor and + # manual dispatch live in the reusable observability-lite workflow itself, so + # unrelated main pushes no longer force the collector stack. + if: needs.preflight.outputs.run_observability == 'true' + permissions: + contents: read + uses: ./.github/workflows/observability-lite.yml + + quality-gate: + name: Quality Gate + runs-on: ubuntu-latest + if: always() + needs: + - preflight + - lockfile-drift + - static-analysis + - node-compat + - coverage + - e2e-chrome + permissions: + contents: read + timeout-minutes: 3 + steps: + - name: Enforce upstream quality lane results + env: + EVENT_NAME: ${{ github.event_name }} + IS_MAIN: ${{ github.ref == 'refs/heads/main' }} + IS_DEPENDABOT: ${{ startsWith(github.head_ref, 'dependabot/') }} + RUN_BROWSER_E2E: ${{ needs.preflight.outputs.run_browser_e2e }} + RUN_LOCKFILE_CHECK: ${{ needs.preflight.outputs.run_lockfile_check }} + HAS_FULL_E2E_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full-e2e') }} + HAS_RISK_E2E_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'risk:e2e') }} + PREFLIGHT_RESULT: ${{ needs.preflight.result }} + LOCKFILE_RESULT: ${{ needs.lockfile-drift.result }} + STATIC_ANALYSIS_RESULT: ${{ needs.static-analysis.result }} + NODE_COMPAT_RESULT: ${{ needs.node-compat.result }} + COVERAGE_RESULT: ${{ needs.coverage.result }} + E2E_CHROME_RESULT: ${{ needs.e2e-chrome.result }} + run: | + set -u + status=0 + + require_success() { + label="$1"; result="$2" + if [ "$result" != "success" ]; then + echo "::error::${label} expected success but was ${result:-missing}" + status=1 + fi + } + + require_skipped() { + label="$1"; result="$2" + if [ "$result" != "skipped" ]; then + echo "::error::${label} expected skipped for ${EVENT_NAME} but was ${result:-missing}" + status=1 + fi + } + + # Expected result depends on event + change scope so the gate fails closed + # on a lane that was cancelled, errored, or unexpectedly skipped. + lockfile_expected=skipped + if [ "$IS_MAIN" = "true" ] || [ "$EVENT_NAME" = "schedule" ] || [ "$EVENT_NAME" = "workflow_dispatch" ] || [ "$RUN_LOCKFILE_CHECK" = "true" ] || [ "$IS_DEPENDABOT" = "true" ]; then + lockfile_expected=success + fi + + e2e_expected=skipped + if [ "$IS_MAIN" = "true" ] || [ "$EVENT_NAME" = "schedule" ] || [ "$EVENT_NAME" = "workflow_dispatch" ] || [ "$RUN_BROWSER_E2E" = "true" ] || [ "$HAS_FULL_E2E_LABEL" = "true" ] || [ "$HAS_RISK_E2E_LABEL" = "true" ]; then + e2e_expected=success + fi + + { + echo "### Quality Gate results" + echo + echo "| Lane | Expected | Actual |" + echo "| --- | --- | --- |" + echo "| Preflight | success | ${PREFLIGHT_RESULT:-missing} |" + echo "| Static Analysis (Node 22) | success | ${STATIC_ANALYSIS_RESULT:-missing} |" + echo "| Node Compatibility (20/22/24) | success | ${NODE_COMPAT_RESULT:-missing} |" + echo "| Coverage (Unit Floors) | success | ${COVERAGE_RESULT:-missing} |" + echo "| Lockfile Drift | ${lockfile_expected} | ${LOCKFILE_RESULT:-missing} |" + echo "| E2E (Chrome) | ${e2e_expected} | ${E2E_CHROME_RESULT:-missing} |" + } >> "$GITHUB_STEP_SUMMARY" + + require_success "Preflight" "$PREFLIGHT_RESULT" + require_success "Static Analysis (Node 22)" "$STATIC_ANALYSIS_RESULT" + require_success "Node Compatibility (20/22/24)" "$NODE_COMPAT_RESULT" + require_success "Coverage (Unit Floors)" "$COVERAGE_RESULT" + + if [ "$lockfile_expected" = "success" ]; then + require_success "Lockfile Drift" "$LOCKFILE_RESULT" + else + require_skipped "Lockfile Drift" "$LOCKFILE_RESULT" + fi + + if [ "$e2e_expected" = "success" ]; then + require_success "E2E (Chrome)" "$E2E_CHROME_RESULT" + else + require_skipped "E2E (Chrome)" "$E2E_CHROME_RESULT" + fi + + if [ "$status" -ne 0 ]; then + echo "Quality Gate failed closed because one or more required quality lanes had an unexpected result." + exit "$status" + fi + + echo "Quality Gate passed with the expected event-aware result map." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 34051cc..b381a51 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,21 +68,25 @@ jobs: mkdir -p release-evidence npm run schemas:check 2>&1 | tee release-evidence/schema-validation.txt - - name: Build package - run: npm run build - - name: Pack package tarball + # One intentional build + one local pack. `npm pack` runs the `prepack` + # build (npm run build) once and emits the real tarball; its JSON report + # proves the package-content contract, so the previous standalone + # `npm run build` and `npm run pack:dry-run` were duplicate build/pack work. run: | mkdir -p release-evidence - npm run pack:dry-run npm pack --json --pack-destination release-evidence --loglevel warn > release-evidence/pack-report.json - name: Validate package consumer install + # Reuses the single tarball packed above; no additional pack. run: | set -o pipefail npm run package:consumer-smoke -- --pack-report release-evidence/pack-report.json 2>&1 | tee release-evidence/consumer-smoke.txt - name: Run package publish validators + # publint reads the already-built dist. `attw --pack` performs its own + # internal repack to inspect the published type surface -- that repack is + # tool-owned and unavoidable; every other build/pack in this job is deduped. run: | set -o pipefail npm run package:publint 2>&1 | tee release-evidence/publint.txt diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 1b49de0..0bf8bd4 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -19,6 +19,34 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' jobs: + preflight: + name: Security Preflight + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + run_effect_proof: ${{ steps.paths-filter.outputs.effect_proof }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Detect security-governing changes + id: paths-filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 + with: + # The gitleaks version/checksum, synthetic credential, allowlist, and + # scanner invocation all live inline in security.yml, so its change is + # what governs whether the effect-proof meta-test must re-run on a PR. + filters: | + effect_proof: + - '.github/workflows/security.yml' + - '.github/zizmor.yml' + - 'scripts/dependency-review.sh' + dependency-review: name: Dependency Review if: github.event_name == 'pull_request' @@ -133,6 +161,11 @@ jobs: secret-scan-effect-proof: name: Secret Scan Effect Proof runs-on: ubuntu-latest + needs: preflight + # Scanner-effectiveness meta-test. It runs on push/schedule/dispatch and on + # pull requests only when the scanner configuration itself changed, so it no + # longer consumes a runner on every ordinary PR (spec section 8). + if: github.event_name != 'pull_request' || needs.preflight.outputs.run_effect_proof == 'true' timeout-minutes: 6 permissions: contents: read @@ -176,6 +209,7 @@ jobs: runs-on: ubuntu-latest if: always() needs: + - preflight - dependency-review - npm-audit - codeql @@ -194,6 +228,7 @@ jobs: SECRET_SCAN_RESULT: ${{ needs.secret-scan.result }} SECRET_SCAN_EFFECT_PROOF_RESULT: ${{ needs.secret-scan-effect-proof.result }} WORKFLOW_SECURITY_RESULT: ${{ needs.workflow-security.result }} + RUN_EFFECT_PROOF: ${{ needs.preflight.outputs.run_effect_proof }} run: | set -u status=0 @@ -244,13 +279,22 @@ jobs: require_success "CodeQL" "$CODEQL_RESULT" require_success "Secret Scan" "$SECRET_SCAN_RESULT" - require_success "Secret Scan Effect Proof" "$SECRET_SCAN_EFFECT_PROOF_RESULT" require_success "Workflow Security" "$WORKFLOW_SECURITY_RESULT" record_result "CodeQL" "success" "$CODEQL_RESULT" record_result "Secret Scan" "success" "$SECRET_SCAN_RESULT" - record_result "Secret Scan Effect Proof" "success" "$SECRET_SCAN_EFFECT_PROOF_RESULT" record_result "Workflow Security" "success" "$WORKFLOW_SECURITY_RESULT" + # Effect proof runs on push/schedule/dispatch, and on pull requests only + # when scanner-governing files changed; otherwise it is intentionally + # skipped and the gate must confirm that skip rather than a silent drop. + if [ "${{ github.event_name }}" != "pull_request" ] || [ "$RUN_EFFECT_PROOF" = "true" ]; then + require_success "Secret Scan Effect Proof" "$SECRET_SCAN_EFFECT_PROOF_RESULT" + record_result "Secret Scan Effect Proof" "success" "$SECRET_SCAN_EFFECT_PROOF_RESULT" + else + require_skipped "Secret Scan Effect Proof" "$SECRET_SCAN_EFFECT_PROOF_RESULT" + record_result "Secret Scan Effect Proof" "skipped" "$SECRET_SCAN_EFFECT_PROOF_RESULT" + fi + if [ "$status" -ne 0 ]; then echo "Security Gate failed closed because one or more required security jobs had an unexpected result." exit "$status" diff --git a/README.md b/README.md index b310b07..3efee77 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # AuroraFlow -[![Quality Gates](https://github.com/jsugg/auroraflow/actions/workflows/quality.yml/badge.svg)](https://github.com/jsugg/auroraflow/actions/workflows/quality.yml) [![Examples](https://github.com/jsugg/auroraflow/actions/workflows/examples.yml/badge.svg)](https://github.com/jsugg/auroraflow/actions/workflows/examples.yml) [![Security Checks](https://github.com/jsugg/auroraflow/actions/workflows/security.yml/badge.svg)](https://github.com/jsugg/auroraflow/actions/workflows/security.yml) [![E2E Matrix](https://github.com/jsugg/auroraflow/actions/workflows/ci.yml/badge.svg)](https://github.com/jsugg/auroraflow/actions/workflows/ci.yml) +[![Quality Gates](https://github.com/jsugg/auroraflow/actions/workflows/quality.yml/badge.svg)](https://github.com/jsugg/auroraflow/actions/workflows/quality.yml) [![Security Checks](https://github.com/jsugg/auroraflow/actions/workflows/security.yml/badge.svg)](https://github.com/jsugg/auroraflow/actions/workflows/security.yml) [![E2E Matrix](https://github.com/jsugg/auroraflow/actions/workflows/ci.yml/badge.svg)](https://github.com/jsugg/auroraflow/actions/workflows/ci.yml) ![AuroraFlow Logo](https://github.com/jsugg/auroraflow/blob/main/.github/assets/auroraflow-logo.png?raw=true) @@ -38,7 +38,7 @@ Observability deployment references: | Self-healing diagnostics | `off`, `suggest`, and `guarded` modes are parsed and enforced. Invalid `SELF_HEAL_*` values warn with the applied fallback by default and throw in opt-in strict mode (`AURORAFLOW_CONFIG_STRICT=true`); diagnostics never echo received values. Failure capture can include ranked suggestions, DOM-derived candidates, registry-backed history, guarded validation, one guarded retry for click/type/read/wait, history observations, reviewable pending promotion records, and audited approve/reject/rollback workflows. | Reviewed promotion scope is registry mutation only; no source-code rewrites or blind unreviewed selector mutation occur. | `src/framework/selfHealing/config.ts`, `src/framework/selfHealing/analyzer.ts`, `src/framework/selfHealing/guardedValidation.ts`, `src/framework/selfHealing/promotionWorkflow.ts`, `docs/architecture/self-healing.md` | | Redis data layer | Runtime config validation, namespaced keys, bounded retry with jitter, SCAN-based listing, batched reads, selector record validation, CAS, page/action indexes, TTL-capable stores, SAT history records, pending promotions, audited selector updates, Testcontainers coverage, and operator-owned production runbook guidance are implemented. | Redis remains consumer/operator-owned; prefixes are namespace hygiene, not authorization. | `src/utils/redisClient.ts`, `src/data/selectors/selectorRegistry.ts`, `docs/architecture/data-layer.md`, `docs/operations/redis-production-runbook.md` | | Observability | Artifact-only/no-op is the supported default. Opt-in Lite collector smoke is best effort; the Full local stack and production manifests are reference-only. Dashboard and alert labels are asserted against live Prometheus label/series/query/rule snapshots. | Live telemetry is never enabled implicitly. Shared or production deployment remains consumer/operator-owned and requires credentials, storage, DNS, TLS, capacity, retention, and network controls. | `src/framework/observability/*`, `docs/operations/observability-support-tiers.md`, `docs/operations/observability-contract.md`, `docs/architecture/observability-stack.md`, `observability/README.md` | -| CI and security | Pull requests run quality and security gates. Example and smoke lanes are path-filtered. The full E2E matrix runs on `main`, schedule, and manual dispatch. | Some optional observability and remote-export paths need repository variables/secrets and enough runner capacity. | `.github/workflows/quality.yml`, `.github/workflows/examples.yml`, `.github/workflows/security.yml`, `.github/workflows/ci.yml` | +| CI and security | Pull requests run the aggregate `Quality Gate` and `Security Gate`. One path-filtered `E2E (Chrome)` lane runs the full Chrome suite on browser-relevant changes; the exhaustive cross-browser matrix runs on a daily schedule and manual dispatch. | Optional observability paths need repository variables and enough runner capacity; remote export is operator-owned and documented, not a standing CI job. | `.github/workflows/quality.yml`, `.github/workflows/security.yml`, `.github/workflows/ci.yml`, `.github/workflows/observability-lite.yml`, `.github/workflows/observability-full-stack.yml` | ## Getting started diff --git a/docs/development.md b/docs/development.md index c975ad4..098e0b2 100644 --- a/docs/development.md +++ b/docs/development.md @@ -107,10 +107,10 @@ Contract specs are semantic-first: | `npm run test:contracts` | Static/contract | Package, workflow, infrastructure, and docs contracts. | | `npm run test:integration` | Real integration | Redis/Testcontainers and OTLP export. | | `AURORAFLOW_REDIS_INTEGRATION_REQUIRED=true npm run test:integration` | Blocking real integration | Same Redis/OTLP integration suite, but Redis startup/connect failures fail instead of skip. | -| `npm run test:coverage` | Coverage | Critical-module thresholds plus global `src/**` coverage. | +| `npm run test:coverage` | Coverage | One complete unit coverage run enforcing the global `src/**` floor and every risk-weighted per-file floor. | | `npm run test:e2e` | Browser | Playwright browser projects. | | `npm run test:e2e:guarded` | Guarded browser proof | Parallel Chrome proof for guarded self-heal at the default gate. | -| `npm run verify` | Full local gate | Static checks, unit, contracts, integration, schemas, ShellCheck, and workflow lint. | +| `npm run verify` | Full local gate | Static checks, unit, contracts, integration, ShellCheck, and workflow lint. Schema validation is a CI/release evidence gate (`npm run schemas:check`), run once outside `verify`. | | `npm run test:mutation` / `test:mutation:check` | Scheduled/manual advisory | Scoped mutation baseline for calibration-critical code; killed mutants that survive or become inapplicable fail the evidence lane, but it remains outside `verify`. See [mutation & property baseline](quality/mutation-property-baseline.md). | | `npm run benchmark:failure-path` | Manual/browser | Warning-only safe-action failure, DOM snapshot, SAT extraction, and artifact-write timing; not part of `verify`. See [failure-path performance baseline](quality/failure-path-performance-baseline.md). | @@ -130,15 +130,16 @@ Calibration-critical code (scoring, config, guarded validation, retry, Redis CAS ### CI gate topology +Branch protection depends on two stable aggregate gates, `Quality Gate` and `Security Gate`, not on individual matrix-leg or internal job names. Each aggregate runs with `if: always()`, verifies the expected success-or-skipped result of every upstream lane for the current event, fails closed on a missing/cancelled/unexpectedly-skipped lane, and publishes a result table to the job summary. + | Gate | Cost tier | Runs | Responsibility | | --- | --- | --- | --- | -| `Lockfile Drift` | Fast dependency check | Manifest/dependabot changes, Dependabot PRs, `main`, scheduled, and manual quality workflow runs | Fails early with clear remediation when `package-lock.json` is not synchronized with `package.json`. | -| `Node Compatibility (Node 20/22/24)` | Fast matrix | Pull requests, `main`, scheduled, and manual quality workflow runs | `npm ci`, lint, typecheck, and unit tests only; no Docker, Redis, OTLP collector, or browser install. | -| `Repository Gates (Node 22)` | Static + Docker integration | Pull requests, `main`, scheduled, and manual quality workflow runs | Format, contracts, Redis/OTLP integration with `AURORAFLOW_REDIS_INTEGRATION_REQUIRED=true`, schemas, ShellCheck, and workflow lint. | -| `Coverage (Critical + Global)` | Coverage | Pull requests, `main`, scheduled, and manual quality workflow runs | Enforces critical-module thresholds, global `src/**` coverage, and risk-weighted per-file floors for high-risk modules once on Node 22. | -| `Guarded Self-Heal Proof (Chrome)` | Guarded browser proof | Pull requests, `main`, scheduled, and manual quality workflow runs | Preserves Chrome proof for guarded self-heal at the shipped default confidence gate. | -| `Risk-Triggered E2E (Chrome)` | Browser heavy | `main`, scheduled/manual runs, risky browser/runtime paths, or `full-e2e`/`risk:e2e` PR labels | Runs the full Chrome E2E project outside the Node compatibility matrix. | -| Observability smoke jobs | Docker/remote optional | Path-triggered, `main`, scheduled, or manual runs depending on the job | Keeps collector/full-stack/remote export evidence separate from fast compatibility gates. | +| `Lockfile Drift` | Fast dependency check | Manifest/dependabot changes, Dependabot PRs, `main`, scheduled, and manual quality workflow runs | Fails early with clear remediation when `package-lock.json` is not synchronized with `package.json`; the one explicit lockfile gate (the shared setup no longer dry-runs it before every `npm ci`). | +| `Node Compatibility (Node 20/22/24)` | Fast matrix | Pull requests, `main`, scheduled, and manual quality workflow runs | `npm ci` and unit tests only — runtime-sensitive behavior across supported Node versions; no lint/typecheck/schema (those are runtime-insensitive and run once), no Docker, Redis, OTLP collector, or browser install. | +| `Static Analysis (Node 22)` | Static + Docker integration | Pull requests, `main`, scheduled, and manual quality workflow runs | Format, lint, typecheck, contracts, Redis/OTLP integration with `AURORAFLOW_REDIS_INTEGRATION_REQUIRED=true`, schemas, ShellCheck, and workflow lint — the canonical runtime-insensitive gates, run once. actionlint is installed once and reused. | +| `Coverage (Unit Floors)` | Coverage | Pull requests, `main`, scheduled, and manual quality workflow runs | One complete unit coverage run enforcing the global `src/**` floor and risk-weighted per-file floors for high-risk modules once on Node 22. The former critical-only coverage pass was a strict subset with identical floors and was removed. | +| `E2E (Chrome)` | Browser heavy | Browser-relevant changes, `main`, scheduled/manual runs, or `full-e2e`/`risk:e2e` PR labels; skipped (no runner) otherwise | Runs the full Chrome project once — the union of the former smoke, guarded self-heal, examples, and risk subsets, so no Playwright test ID runs twice on the same event — then evaluates self-healing governance on that run's artifacts. | +| Observability smoke jobs | Docker optional | `Observability Lite` on observability-relevant changes plus a daily floor; `Observability Full Stack` weekly/manual only | Keeps collector-only Lite receipt evidence separate from the weekly full seven-service reference-stack validation; neither is a routine gate on unrelated pushes. | ### Browser suites @@ -148,7 +149,7 @@ npm run test:examples npm run test:e2e ``` -The full E2E workflow shards across desktop and mobile browser projects on `main`, schedule, and manual dispatch. The quality workflow also has a risk-triggered full Chrome E2E lane for risky browser/runtime paths or `full-e2e`/`risk:e2e` PR labels. Pull-request smoke and examples lanes remain path-filtered by workflow configuration. +The exhaustive `E2E Matrix` workflow runs one job per installed browser binary (desktop and mobile WebKit share a single `webkit` install in one job) on a daily schedule and manual dispatch — no per-run sharding, because Playwright is already fully parallel and browser/dependency setup, not test time, dominates a job. Post-merge Chrome coverage comes once from the Quality Gates `E2E (Chrome)` lane rather than the full matrix on every push; run `E2E Matrix` manually after a risky merge if cross-browser confirmation is needed before the next daily run. The `E2E (Chrome)` lane is path-filtered: it runs the full Chrome project on browser-relevant changes and is skipped (no runner) otherwise. If a constrained local machine times out during accessibility smoke checks, reproduce with a larger Playwright timeout before changing source: @@ -180,7 +181,7 @@ npm run infra:redis:logs npm run infra:redis:down ``` -Default local Redis behavior is skip-friendly: if Docker/Testcontainers cannot start Redis, the Redis integration spec reports an explicit skip so non-Redis contributors are not blocked. Required mode is blocking: `AURORAFLOW_REDIS_INTEGRATION_REQUIRED=true npm run test:integration` fails on Redis startup, connection, or evidence failures and is the CI/release mode used by `Repository Gates (Node 22)` and the release dry-run. External mode is opt-in: `AURORAFLOW_REDIS_INTEGRATION_EXTERNAL=true AURORAFLOW_REDIS_HOST=127.0.0.1 AURORAFLOW_REDIS_PORT=6379 npm run test:integration` reuses an existing Redis with a per-run key prefix. +Default local Redis behavior is skip-friendly: if Docker/Testcontainers cannot start Redis, the Redis integration spec reports an explicit skip so non-Redis contributors are not blocked. Required mode is blocking: `AURORAFLOW_REDIS_INTEGRATION_REQUIRED=true npm run test:integration` fails on Redis startup, connection, or evidence failures and is the CI/release mode used by `Static Analysis (Node 22)` and the release dry-run. External mode is opt-in: `AURORAFLOW_REDIS_INTEGRATION_EXTERNAL=true AURORAFLOW_REDIS_HOST=127.0.0.1 AURORAFLOW_REDIS_PORT=6379 npm run test:integration` reuses an existing Redis with a per-run key prefix. Production Redis use is outside local development ownership: operators own TLS, auth, ACLs, backups, restore drills, no-eviction capacity planning, retention, and incident response. Keep the [Redis production runbook](operations/redis-production-runbook.md) aligned with selector-store behavior when changing Redis keys, commands, TTLs, or cleanup workflows. diff --git a/docs/operations/flakiness-analytics.md b/docs/operations/flakiness-analytics.md index f7eea45..1361a49 100644 --- a/docs/operations/flakiness-analytics.md +++ b/docs/operations/flakiness-analytics.md @@ -69,9 +69,9 @@ Policy fields: The triage report groups, per owner, the failing tests, flaky tests, quarantined tests (kept visible even when the run passed), and repeated flakes (failed attempts at or over the threshold), each with an actionable next step. Quarantined and repeated flakes always appear in the report so they cannot silently disappear. -### PR risk E2E lane +### PR E2E lane -Default pull requests stay fast: only smoke and path-filtered lanes run. Risky pull requests can run the full Chrome E2E suite through the `Risk-Triggered E2E` job in `quality.yml`, triggered by the `risk:e2e` or `full-e2e` label, by changes under risk paths (`src/pageObjects/**`, `src/framework/selfHealing/**`, E2E specs and fixtures), or by a manual `workflow_dispatch` run. +Default pull requests stay fast: the single path-filtered `E2E (Chrome)` job in `quality.yml` runs the full Chrome project only on browser-relevant changes and is skipped (no runner) otherwise. Because the full Chrome suite is the union of the former smoke, guarded self-heal, examples, and risk subsets, no Playwright test ID runs twice on a pull request. It also runs on `main`, on scheduled/manual dispatch, and can be forced on an otherwise-skipped PR with the `risk:e2e` or `full-e2e` label. Exhaustive cross-browser coverage runs on the daily `E2E Matrix`. ## Downstream SLO and Alerting diff --git a/docs/operations/observability-production.md b/docs/operations/observability-production.md index 3b1446a..7b35d4d 100644 --- a/docs/operations/observability-production.md +++ b/docs/operations/observability-production.md @@ -12,12 +12,45 @@ AuroraFlow production observability environments must be deployed from hardened, ## Remote Export -CI remote export runs for observability-related changes and on `main` when endpoint secrets are configured: +There is **no standing remote-export CI job**. AuroraFlow does not own an OTLP/HTTP backend, so a permanently-configured lane would never do real work (its endpoint secret is unset) and could only ever be a runner-consuming no-op. It was removed; the local Lite and Full lanes validate real export against `http://localhost:4318`. -- Secret: `OTEL_EXPORTER_OTLP_ENDPOINT` -- Optional secret: `OTEL_EXPORTER_OTLP_HEADERS` +An operator who owns an OTLP/HTTP backend can add the lane below. Because it has no backend-receipt query proving the run-scoped marker arrived, it is a **Remote Export Connectivity Smoke**, not end-to-end validation. Before enabling it you must have all of: a named owner; a selected OTLP/HTTP-compatible backend; the repository variable `AURORAFLOW_OBSERVABILITY_REMOTE_EXPORT_ENABLED=true`; endpoint and header secrets; and documented cost, retention, privacy, and incident ownership. Never invent an endpoint — use the provider- or operator-issued OTLP/HTTP base endpoint. -The workflow sets `AURORAFLOW_OBSERVABILITY_REMOTE_EXPORT_ENABLED=true`, passes these values directly to OpenTelemetry environment variables, and uploads only smoke diagnostics. Do not echo headers or endpoint credentials in workflow logs. +```yaml +# Operator-owned workflow example (not shipped in .github/workflows). +name: Observability Remote Export Connectivity Smoke +on: + workflow_dispatch: + schedule: + - cron: '0 7 * * 1' +permissions: + contents: read +jobs: + remote-export-connectivity-smoke: + name: Remote Export Connectivity Smoke + runs-on: ubuntu-latest + # Fail closed: enable variable must be set AND the endpoint secret present. + if: vars.AURORAFLOW_OBSERVABILITY_REMOTE_EXPORT_ENABLED == 'true' + timeout-minutes: 8 + env: + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }} + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Fail closed when the endpoint secret is absent + run: test -n "${OTEL_EXPORTER_OTLP_ENDPOINT}" || { echo "endpoint secret missing"; exit 1; } + # ... setup Node, then: + - name: Emit remote smoke telemetry + env: + AURORAFLOW_OBSERVABILITY_ENABLED: 'true' + AURORAFLOW_OBSERVABILITY_ENVIRONMENT: ci + AURORAFLOW_OBSERVABILITY_STRICT: 'true' + run: npm run observability:ci:smoke +``` + +Pass the endpoint/header secrets directly to the OpenTelemetry environment variables and upload only smoke diagnostics. Do not echo headers or endpoint credentials in workflow logs. Promote it to end-to-end validation only once a backend receipt query proves the run-scoped marker arrived. ## Storage Budgets diff --git a/examples/README.md b/examples/README.md index b9dde7b..b0e7186 100644 --- a/examples/README.md +++ b/examples/README.md @@ -34,6 +34,6 @@ npm run security:check ## Notes -- Examples are validated in CI by `.github/workflows/examples.yml`. +- Examples are validated in CI by the `E2E (Chrome)` lane in `.github/workflows/quality.yml` (the full Chrome suite includes every example on browser-relevant changes) and by the daily cross-browser `.github/workflows/ci.yml` matrix. - Example tests are organized under `tests/suites/e2e/examples/`. - Examples are documentation and test fixtures. They are not production integrations and must not depend on live external websites. diff --git a/examples/ci/quality.workflow.example.yml b/examples/ci/quality.workflow.example.yml index 4a7b0b6..3c932a7 100644 --- a/examples/ci/quality.workflow.example.yml +++ b/examples/ci/quality.workflow.example.yml @@ -17,7 +17,9 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' jobs: - verify: + # Runtime-sensitive signal only: unit tests across supported Node versions. + # Runtime-insensitive gates (lint/typecheck/format/schema) run once below. + node-compat: name: Node Compatibility (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest timeout-minutes: 10 @@ -35,11 +37,11 @@ jobs: cache: npm - name: Install dependencies run: npm ci - - name: Run Node compatibility gates - run: npm run lint && npm run typecheck && npm test + - name: Run unit tests + run: npm test - repository-gates: - name: Repository Gates (Node 22) + static-analysis: + name: Static Analysis (Node 22) runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -52,7 +54,7 @@ jobs: cache: npm - name: Install dependencies run: npm ci - - name: Run repository gates + - name: Run static analysis and repository gates env: AURORAFLOW_REDIS_INTEGRATION_REQUIRED: 'true' run: npm run verify diff --git a/package.json b/package.json index 67ec7ac..16c8690 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "typecheck": "tsc --noEmit --incremental --tsBuildInfoFile .cache/tsconfig.tsbuildinfo", "test": "npm run test:unit", "test:unit": "vitest run tests/suites/unit --pool=threads --no-isolate --testTimeout=30000", - "test:coverage": "npm run test:coverage:critical && npm run test:coverage:global", + "test:coverage": "npm run test:coverage:global", "test:contracts": "vitest run tests/suites/contracts --config=configs/vitest.contracts.mts", "test:integration": "vitest run tests/suites/integration", "test:integration:all": "npm run test:integration && npm run test:contracts", @@ -93,9 +93,8 @@ "security:workflows": "pipx run --spec zizmor==1.26.1 zizmor .github/workflows", "security:all": "npm run security:audit && npm run security:workflows", "security:check": "npm run security:all", - "verify": "npm run verify:tools && npm run format:check && npm run lint && npm run typecheck && npm run test:unit && npm run test:contracts && npm run test:integration && npm run schemas:check && npm run shellcheck && npm run workflows:lint:check", + "verify": "npm run verify:tools && npm run format:check && npm run lint && npm run typecheck && npm run test:unit && npm run test:contracts && npm run test:integration && npm run shellcheck && npm run workflows:lint:check", "verify:smoke": "npm run verify && npm run test:smoke", - "test:coverage:critical": "vitest run tests/suites/unit/framework/selfHealing/artifactPrivacy.spec.ts tests/suites/unit/framework/selfHealing/config.spec.ts tests/suites/unit/framework/selfHealing/guardedValidation.spec.ts tests/suites/unit/framework/selfHealing/historyRepository.spec.ts tests/suites/unit/framework/selfHealing/promotionWorkflow.spec.ts tests/suites/unit/framework/selfHealing/registryRuntime.spec.ts tests/suites/unit/framework/observability/trends.spec.ts --coverage", "test:coverage:global": "vitest run tests/suites/unit --config=configs/vitest.coverage-global.mts --coverage", "test:mutation": "node scripts/mutation-baseline.mjs", "test:mutation:check": "node scripts/mutation-baseline.mjs --check", diff --git a/tests/suites/contracts/observability/observabilityContract.spec.ts b/tests/suites/contracts/observability/observabilityContract.spec.ts index 6f76bd0..a4ea696 100644 --- a/tests/suites/contracts/observability/observabilityContract.spec.ts +++ b/tests/suites/contracts/observability/observabilityContract.spec.ts @@ -155,7 +155,8 @@ describe('observability contract documentation', () => { const supportTiers = readFileSync(SUPPORT_TIERS_PATH, 'utf8'); const liteCompose = readComposeModel('docker-compose.observability-ci.yml'); const fullCompose = readComposeModel('docker-compose.observability.yml'); - const workflow = readWorkflowModel('.github/workflows/quality.yml'); + const liteWorkflow = readWorkflowModel('.github/workflows/observability-lite.yml'); + const fullStackWorkflow = readWorkflowModel('.github/workflows/observability-full-stack.yml'); const packageJson = readPackageJson(); expect([...liteCompose.services.keys()]).toEqual(['otel-collector']); @@ -170,10 +171,8 @@ describe('observability contract documentation', () => { 'prometheus', ].sort(), ); - expect(getWorkflowJob(workflow, 'observability-stack').name).toBe('Observability Lite Smoke'); - expect(getWorkflowJob(workflow, 'observability-full-stack').name).toBe( - 'Observability Full Stack Smoke', - ); + expect(getWorkflowJob(liteWorkflow, 'collector-lite-smoke').name).toBe('Collector Lite Smoke'); + expect(getWorkflowJob(fullStackWorkflow, 'full-stack-smoke').name).toBe('Full Stack Smoke'); expect(packageJson.scripts).toEqual( expect.objectContaining({ 'observability:lite:up': 'docker compose -f docker-compose.observability-ci.yml up -d', @@ -290,9 +289,10 @@ describe('observability contract documentation', () => { it('provides a collector-only Lite CI smoke lane with diagnostics', () => { const ciCompose = readComposeModel('docker-compose.observability-ci.yml'); - const workflow = readWorkflowModel('.github/workflows/quality.yml'); + const qualityWorkflow = readWorkflowModel('.github/workflows/quality.yml'); + const liteWorkflow = readWorkflowModel('.github/workflows/observability-lite.yml'); const packageJson = readPackageJson(); - const collectorOnlyJob = getWorkflowJob(workflow, 'observability-stack'); + const collectorOnlyJob = getWorkflowJob(liteWorkflow, 'collector-lite-smoke'); const collectorConfig = readFileSync( path.join(process.cwd(), 'observability', 'otel-collector', 'ci-config.yaml'), 'utf8', @@ -302,21 +302,34 @@ describe('observability contract documentation', () => { expect(getComposeService(ciCompose, 'otel-collector').volumes).toEqual([ './observability/otel-collector/ci-config.yaml:/etc/otelcol-contrib/config.yaml:ro', ]); + // Quality gates call the Lite lane only on observability-relevant changes; the + // reusable workflow owns the daily scheduled floor so unrelated pushes do not + // force the collector stack. + const liteCallJob = getWorkflowJob(qualityWorkflow, 'observability-lite'); + expect(liteCallJob.uses).toBe('./.github/workflows/observability-lite.yml'); + expectTextIncludes(liteCallJob.if ?? '', { + text: "needs.preflight.outputs.run_observability == 'true'", + rationale: 'Quality gates must call the Lite lane only for observability-relevant changes.', + }); + expect( + [...liteWorkflow.triggers].sort(), + 'Lite lane must run on relevant-change calls, a scheduled floor, and manual dispatch.', + ).toEqual(['schedule', 'workflow_call', 'workflow_dispatch']); const pathFilters = getWorkflowStep( - getWorkflowJob(workflow, 'preflight'), - 'Detect smoke-relevant changes', + getWorkflowJob(qualityWorkflow, 'preflight'), + 'Detect change scopes', ).with.get('filters') ?? ''; expectTextIncludes(pathFilters, { - text: 'observability_stack:', - rationale: 'Quality preflight must expose observability_stack path filter.', + text: 'observability:', + rationale: 'Quality preflight must expose the observability path filter.', }); expectTextIncludes(pathFilters, { - text: 'scripts/observability-collector-receipt.ts', - rationale: 'Collector receipt changes must trigger the Lite evidence lane.', + text: 'scripts/observability-*.ts', + rationale: 'Collector/receipt script changes must trigger the Lite evidence lane.', }); - expectTextIncludes(collectorOnlyJob.env.get('SHOULD_RUN_OBSERVABILITY') ?? '', { - text: 'AURORAFLOW_OBSERVABILITY_CI_ENABLED', + expectTextIncludes(collectorOnlyJob.if ?? '', { + text: "vars.AURORAFLOW_OBSERVABILITY_CI_ENABLED != 'false'", rationale: 'Collector smoke lane must support repository-variable opt-out.', }); expect(collectorOnlyJob.env.get('OBSERVABILITY_DIAGNOSTICS_DIR')).toBe( @@ -365,16 +378,21 @@ describe('observability contract documentation', () => { ); }); - it('provides full-stack and secret-gated remote-export CI observability lanes', () => { - const workflow = readWorkflowModel('.github/workflows/quality.yml'); - const fullStackJob = getWorkflowJob(workflow, 'observability-full-stack'); - const remoteExportJob = getWorkflowJob(workflow, 'observability-remote-export'); + it('runs full-stack validation weekly and keeps remote export as operator docs only', () => { + const fullStackWorkflow = readWorkflowModel('.github/workflows/observability-full-stack.yml'); + const fullStackJob = getWorkflowJob(fullStackWorkflow, 'full-stack-smoke'); - expect(fullStackJob.name).toBe('Observability Full Stack Smoke'); - expect(fullStackJob.env.get('AURORAFLOW_OBSERVABILITY_FULL_STACK_CI_ENABLED')).toBe('true'); - expect(fullStackJob.env.get('SHOULD_RUN_OBSERVABILITY_FULL_STACK')).toBe( - "${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}", - ); + // Full Stack is scheduled/manual reference-asset validation, never a PR gate: + // with only schedule + dispatch triggers no runner is allocated on pull requests. + expect(fullStackJob.name).toBe('Full Stack Smoke'); + expect( + [...fullStackWorkflow.triggers].sort(), + 'Full-stack reference validation must be weekly/manual only, never a PR gate.', + ).toEqual(['schedule', 'workflow_dispatch']); + expectTextIncludes(fullStackJob.if ?? '', { + text: "vars.AURORAFLOW_OBSERVABILITY_FULL_STACK_CI_ENABLED != 'false'", + rationale: 'Full-stack lane must support repository-variable opt-out.', + }); expect(fullStackJob.env.get('OBSERVABILITY_DIAGNOSTICS_DIR')).toBe( 'observability-output/full-stack', ); @@ -400,20 +418,27 @@ describe('observability contract documentation', () => { rationale: 'Full-stack workflow must upload validator JSON diagnostics.', }); - expect(remoteExportJob.name).toBe('Observability Remote Export Smoke'); - expect(remoteExportJob.env.get('AURORAFLOW_OBSERVABILITY_REMOTE_EXPORT_ENABLED')).toBe('true'); - expect(remoteExportJob.env.get('SHOULD_RUN_REMOTE_EXPORT')).toBe( - "${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}", - ); - expect(remoteExportJob.env.get('OTEL_EXPORTER_OTLP_ENDPOINT')).toBe( - '${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }}', - ); - expect(remoteExportJob.env.get('OTEL_EXPORTER_OTLP_HEADERS')).toBe( - '${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }}', + // No owned OTLP backend / endpoint secret exists, so the standing remote-export + // CI job is removed (spec section 7.3); the operator workflow stays documented. + for (const workflowPath of [ + '.github/workflows/quality.yml', + '.github/workflows/observability-lite.yml', + '.github/workflows/observability-full-stack.yml', + ]) { + expectInvariant( + !readWorkflowModel(workflowPath).jobs.has('observability-remote-export'), + `Remote export must not remain a standing CI job in ${workflowPath} without an owned backend.`, + ); + } + const productionDoc = readFileSync( + path.join(DOCS_OPERATIONS_ROOT, 'observability-production.md'), + 'utf8', ); - expect( - getWorkflowStep(remoteExportJob, 'Upload remote export diagnostics').with.get('name'), - ).toBe('observability-remote-export-diagnostics'); + expectTextIncludes(productionDoc, { + text: 'OTEL_EXPORTER_OTLP_ENDPOINT', + rationale: + 'Operator docs must retain the remote-export configuration example after CI removal.', + }); }); it('uses snapshot-proven Prometheus labels in dashboards and rules', () => { diff --git a/tests/suites/contracts/workflows/actions-runtime.contract.spec.ts b/tests/suites/contracts/workflows/actions-runtime.contract.spec.ts index 08b6953..5ac7658 100644 --- a/tests/suites/contracts/workflows/actions-runtime.contract.spec.ts +++ b/tests/suites/contracts/workflows/actions-runtime.contract.spec.ts @@ -7,18 +7,20 @@ import { getWorkflowActionReferences, readWorkflowModel } from '../../../helpers const WORKFLOW_FILES = [ '.github/workflows/ci.yml', '.github/workflows/quality.yml', - '.github/workflows/examples.yml', '.github/workflows/security.yml', + '.github/workflows/observability-lite.yml', + '.github/workflows/observability-full-stack.yml', '.github/workflows/playwright-peer-matrix.yml', '.github/workflows/release.yml', ] as const; const PR_AND_EVIDENCE_WORKFLOW_FILES = [ '.github/workflows/quality.yml', - '.github/workflows/examples.yml', '.github/workflows/security.yml', ] as const; const EVIDENCE_ONLY_WORKFLOW_FILES = [ '.github/workflows/ci.yml', + '.github/workflows/observability-lite.yml', + '.github/workflows/observability-full-stack.yml', '.github/workflows/playwright-peer-matrix.yml', '.github/workflows/release.yml', ] as const; @@ -123,15 +125,16 @@ describe('workflow actions runtime contract', () => { expect(workflowsUsingComposite.sort()).toEqual([ '.github/workflows/ci.yml', - '.github/workflows/examples.yml', + '.github/workflows/observability-full-stack.yml', + '.github/workflows/observability-lite.yml', '.github/workflows/quality.yml', '.github/workflows/release.yml', '.github/workflows/security.yml', ]); - expectTextIncludes(compositeAction, { - text: 'npm run lockfile:check', - rationale: 'Composite locked install must fail early on package-lock drift.', - }); + expectInvariant( + !compositeAction.includes('npm run lockfile:check'), + 'Composite locked install must not dry-run the lockfile before every npm ci; the explicit lockfile gate runs once per workflow.', + ); expectTextIncludes(compositeAction, { text: 'npm ci', rationale: 'Composite locked install must keep reproducible npm ci semantics.', diff --git a/tests/suites/contracts/workflows/ci-browser-install.contract.spec.ts b/tests/suites/contracts/workflows/ci-browser-install.contract.spec.ts index e89e0dd..e193672 100644 --- a/tests/suites/contracts/workflows/ci-browser-install.contract.spec.ts +++ b/tests/suites/contracts/workflows/ci-browser-install.contract.spec.ts @@ -6,27 +6,7 @@ const ciWorkflow = readWorkflowModel('.github/workflows/ci.yml'); const lockedInstallActionPath = './.github/actions/setup-node-cache'; describe('ci.yml browser provisioning contract', () => { - it('defines install_args for each E2E matrix project', () => { - const e2eJob = getWorkflowJob(ciWorkflow, 'e2e'); - const expectedProjectMappings = [ - { project: 'Google Chrome', installArgs: 'chrome' }, - { project: 'Firefox', installArgs: 'firefox' }, - { project: 'Safari', installArgs: 'webkit' }, - { project: 'Microsoft Edge', installArgs: 'msedge' }, - { project: 'Mobile Chrome', installArgs: 'chromium' }, - { project: 'Mobile Safari', installArgs: 'webkit' }, - ]; - - expect( - e2eJob.strategy?.include.map((entry) => ({ - project: entry.get('project'), - installArgs: entry.get('install_args'), - })), - 'E2E matrix must map every browser project to matching Playwright install arguments.', - ).toEqual(expectedProjectMappings); - }); - - it('uses a browser cache key scoped by install_args', () => { + it('provisions exactly one browser binary per matrix lane through the composite action', () => { const setupStep = getWorkflowStep( getWorkflowJob(ciWorkflow, 'e2e'), 'Setup locked Node.js dependencies and Playwright browser', @@ -38,6 +18,30 @@ describe('ci.yml browser provisioning contract', () => { expect(setupStep.with.get('cache-namespace')).toBe('e2e-${{ matrix.install_args }}'); }); + it('covers every configured Playwright project exactly once across the matrix', () => { + const laneProjects = (getWorkflowJob(ciWorkflow, 'e2e').strategy?.include ?? []).flatMap( + (entry) => (entry.get('projects') ?? '').split(';'), + ); + + expect( + [...laneProjects].sort(), + 'E2E matrix must run each Playwright project exactly once, WebKit projects grouped by binary.', + ).toEqual( + [ + 'Firefox', + 'Google Chrome', + 'Microsoft Edge', + 'Mobile Chrome', + 'Mobile Safari', + 'Safari', + ].sort(), + ); + expectInvariant( + new Set(laneProjects).size === laneProjects.length, + 'No Playwright project may appear in more than one matrix lane on the same event.', + ); + }); + it('always installs the required browser even when cache is restored', () => { const setupStep = getWorkflowStep( getWorkflowJob(ciWorkflow, 'e2e'), diff --git a/tests/suites/contracts/workflows/dependabot.contract.spec.ts b/tests/suites/contracts/workflows/dependabot.contract.spec.ts index 0cbb3e5..e6d1a72 100644 --- a/tests/suites/contracts/workflows/dependabot.contract.spec.ts +++ b/tests/suites/contracts/workflows/dependabot.contract.spec.ts @@ -203,9 +203,7 @@ describe('dependabot configuration contract', () => { it('runs an early lockfile drift guard for dependency automation changes', () => { const preflightJob = getWorkflowJob(qualityWorkflow, 'preflight'); const lockfileJob = getWorkflowJob(qualityWorkflow, 'lockfile-drift'); - const filters = getWorkflowStep(preflightJob, 'Detect smoke-relevant changes').with.get( - 'filters', - ); + const filters = getWorkflowStep(preflightJob, 'Detect change scopes').with.get('filters'); expect(packageJson.scripts?.['lockfile:check']).toBe('node scripts/check-lockfile-drift.mjs'); expect(preflightJob.outputs.get('run_lockfile_check')).toBe( diff --git a/tests/suites/contracts/workflows/e2e-sharding.contract.spec.ts b/tests/suites/contracts/workflows/e2e-sharding.contract.spec.ts index 7606092..b6395e8 100644 --- a/tests/suites/contracts/workflows/e2e-sharding.contract.spec.ts +++ b/tests/suites/contracts/workflows/e2e-sharding.contract.spec.ts @@ -1,34 +1,98 @@ import { describe, expect, it } from 'vitest'; -import { - getWorkflowJob, - getWorkflowMatrixValues, - getWorkflowStep, - readWorkflowModel, -} from '../../../helpers/workflowModel'; +import { expectInvariant, expectTextIncludes } from '../../../helpers/contractAssertions'; +import { getWorkflowJob, getWorkflowStep, readWorkflowModel } from '../../../helpers/workflowModel'; const ciWorkflow = readWorkflowModel('.github/workflows/ci.yml'); -describe('ci.yml e2e sharding contract', () => { - it('defines a two-shard matrix for the full E2E job', () => { +describe('ci.yml e2e matrix topology contract', () => { + it('runs one job per installed browser binary with no per-run sharding', () => { + const e2eJob = getWorkflowJob(ciWorkflow, 'e2e'); + + // Two-way sharding was removed: measured shard wall-clock was dominated by + // browser/dependency setup, not test time, so sharding only doubled setup. + expectInvariant( + e2eJob.strategy?.matrix.get('shard') === undefined, + 'Full E2E matrix must not shard: Playwright is already parallel and setup dominates.', + ); + expect( + getWorkflowStep(e2eJob, 'Run full E2E suite').run?.includes('--shard'), + 'Full E2E runner must not pass a Playwright shard argument.', + ).toBeFalsy(); + + const lanes = (e2eJob.strategy?.include ?? []).map((entry) => ({ + lane: entry.get('lane'), + installArgs: entry.get('install_args'), + slug: entry.get('slug'), + projects: entry.get('projects'), + })); expect( - getWorkflowMatrixValues(getWorkflowJob(ciWorkflow, 'e2e'), 'shard'), - 'Full E2E matrix must split coverage across exactly two shards.', - ).toEqual(['1', '2']); + lanes, + 'Full E2E matrix must map one job per browser binary and group desktop+mobile WebKit.', + ).toEqual([ + { + lane: 'Google Chrome', + installArgs: 'chrome', + slug: 'google-chrome', + projects: 'Google Chrome', + }, + { lane: 'Firefox', installArgs: 'firefox', slug: 'firefox', projects: 'Firefox' }, + { + lane: 'Microsoft Edge', + installArgs: 'msedge', + slug: 'microsoft-edge', + projects: 'Microsoft Edge', + }, + { + lane: 'Mobile Chrome', + installArgs: 'chromium', + slug: 'mobile-chrome', + projects: 'Mobile Chrome', + }, + { + lane: 'WebKit (Safari + Mobile Safari)', + installArgs: 'webkit', + slug: 'webkit', + projects: 'Safari;Mobile Safari', + }, + ]); }); - it('runs each matrix cell with explicit Playwright shard arguments', () => { + it('groups the two WebKit projects into a single webkit install without losing project results', () => { + const runStep = getWorkflowStep(getWorkflowJob(ciWorkflow, 'e2e'), 'Run full E2E suite'); + + expectTextIncludes(runStep.run ?? '', { + text: "IFS=';' read -ra projects", + rationale: + 'Multi-project WebKit lane must split its project list safely for names with spaces.', + }); + expectTextIncludes(runStep.run ?? '', { + text: 'args+=(--project "${project}")', + rationale: + 'Each configured project must be passed as its own quoted Playwright --project flag.', + }); + }); + + it('uses binary-scoped artifact and JSON names to avoid collisions', () => { + const e2eJob = getWorkflowJob(ciWorkflow, 'e2e'); + expect( - getWorkflowStep(getWorkflowJob(ciWorkflow, 'e2e'), 'Run full E2E suite').run, - 'Full E2E runner must pass matrix shard into Playwright.', - ).toBe( - 'npx playwright test --config=configs/playwright.config.ts --project="${{ matrix.project }}" --shard=${{ matrix.shard }}/2', - ); + getWorkflowStep(e2eJob, 'Run full E2E suite').env.get('PLAYWRIGHT_JSON_OUTPUT_FILE'), + 'E2E matrix must emit binary-scoped JSON for flakiness aggregation.', + ).toBe('test-results/playwright-results-${{ matrix.slug }}.json'); + expect( + getWorkflowStep(e2eJob, 'Upload E2E artifacts').with.get('name'), + 'Uploaded E2E artifacts must be binary-scoped to avoid matrix collisions.', + ).toBe('e2e-matrix-artifacts-${{ matrix.slug }}'); }); - it('uses shard-specific artifact names to avoid collisions', () => { + it('runs the exhaustive matrix on a daily schedule and dispatch, not on every main push', () => { expect( - getWorkflowStep(getWorkflowJob(ciWorkflow, 'e2e'), 'Upload E2E artifacts').with.get('name'), - 'Uploaded E2E artifacts must include project and shard to avoid matrix collisions.', - ).toBe('e2e-matrix-artifacts-${{ matrix.project }}-shard-${{ matrix.shard }}'); + [...ciWorkflow.triggers].sort(), + 'E2E matrix must be a scheduled/manual daily workflow, not a per-push gate.', + ).toEqual(['schedule', 'workflow_dispatch']); + expectInvariant( + !ciWorkflow.triggers.has('push'), + 'Full cross-browser matrix must not run on every main push; post-merge Chrome runs in Quality Gates.', + ); }); }); diff --git a/tests/suites/contracts/workflows/example-workflow-templates.contract.spec.ts b/tests/suites/contracts/workflows/example-workflow-templates.contract.spec.ts index 411bf37..c15e050 100644 --- a/tests/suites/contracts/workflows/example-workflow-templates.contract.spec.ts +++ b/tests/suites/contracts/workflows/example-workflow-templates.contract.spec.ts @@ -60,24 +60,30 @@ describe('example workflow template contract', () => { } }); - it('keeps quality template gates aligned with repository gate topology', () => { + it('keeps quality template gates aligned with the static/Node-matrix split topology', () => { const workflow = readWorkflowModel('examples/ci/quality.workflow.example.yml'); - const verifyJob = getWorkflowJob(workflow, 'verify'); - const repositoryGatesJob = getWorkflowJob(workflow, 'repository-gates'); + const nodeCompatJob = getWorkflowJob(workflow, 'node-compat'); + const staticJob = getWorkflowJob(workflow, 'static-analysis'); - expect(verifyJob.name).toBe('Node Compatibility (Node ${{ matrix.node-version }})'); - expect(repositoryGatesJob.name).toBe('Repository Gates (Node 22)'); - expect(getWorkflowStep(verifyJob, 'Run Node compatibility gates').run).toBe( - 'npm run lint && npm run typecheck && npm test', + expect(nodeCompatJob.name).toBe('Node Compatibility (Node ${{ matrix.node-version }})'); + expect(staticJob.name).toBe('Static Analysis (Node 22)'); + // The Node matrix exercises runtime-sensitive behavior only; runtime-insensitive + // gates run once via `npm run verify` on the canonical Node 22 lane. + expect(getWorkflowStep(nodeCompatJob, 'Run unit tests').run).toBe('npm test'); + expectInvariant( + nodeCompatJob.steps.every( + (step) => !step.run?.includes('lint') && !step.run?.includes('typecheck'), + ), + 'Quality template Node matrix must not repeat lint/typecheck per Node version.', ); expect( - getWorkflowStep(repositoryGatesJob, 'Run repository gates').env.get( + getWorkflowStep(staticJob, 'Run static analysis and repository gates').env.get( 'AURORAFLOW_REDIS_INTEGRATION_REQUIRED', ), ).toBe('true'); expectInvariant( - repositoryGatesJob.steps.every((step) => step.name !== 'Run verification contract'), - 'Quality template must use current repository gate naming, not retired verification contract prose.', + staticJob.steps.every((step) => step.name !== 'Run verification contract'), + 'Quality template must use current static-analysis naming, not retired verification contract prose.', ); }); }); diff --git a/tests/suites/contracts/workflows/flakiness-report.contract.spec.ts b/tests/suites/contracts/workflows/flakiness-report.contract.spec.ts index 3968d95..1413038 100644 --- a/tests/suites/contracts/workflows/flakiness-report.contract.spec.ts +++ b/tests/suites/contracts/workflows/flakiness-report.contract.spec.ts @@ -5,15 +5,13 @@ import { getWorkflowJob, getWorkflowStep, readWorkflowModel } from '../../../hel const ciWorkflow = readWorkflowModel('.github/workflows/ci.yml'); describe('ci.yml flakiness report contract', () => { - it('emits shard-scoped Playwright JSON output files from matrix runs', () => { + it('emits binary-scoped Playwright JSON output files from matrix runs', () => { expect( getWorkflowStep(getWorkflowJob(ciWorkflow, 'e2e'), 'Run full E2E suite').env.get( 'PLAYWRIGHT_JSON_OUTPUT_FILE', ), - 'E2E matrix must emit shard-scoped JSON files for flakiness aggregation.', - ).toBe( - 'test-results/playwright-results-${{ matrix.project_slug }}-shard-${{ matrix.shard }}.json', - ); + 'E2E matrix must emit binary-scoped JSON files for flakiness aggregation.', + ).toBe('test-results/playwright-results-${{ matrix.slug }}.json'); }); it('defines a dedicated flakiness report job that aggregates matrix artifacts', () => { diff --git a/tests/suites/contracts/workflows/playwright-peer-matrix.contract.spec.ts b/tests/suites/contracts/workflows/playwright-peer-matrix.contract.spec.ts index 988763f..93f95f5 100644 --- a/tests/suites/contracts/workflows/playwright-peer-matrix.contract.spec.ts +++ b/tests/suites/contracts/workflows/playwright-peer-matrix.contract.spec.ts @@ -35,8 +35,13 @@ describe('Playwright peer matrix workflow', () => { 'Peer matrix must keep playwright, playwright-core, and @playwright/test aligned.', }, ); - expect(getWorkflowStep(peerMatrixJob, 'Check lockfile drift').run).toBe( - 'npm run lockfile:check', + // The lockfile gate runs once before matrix fan-out, not once per lane. + const lockfileJob = getWorkflowJob(workflow, 'lockfile'); + expect(getWorkflowStep(lockfileJob, 'Check lockfile drift').run).toBe('npm run lockfile:check'); + expect(peerMatrixJob.needs).toEqual(['lockfile']); + expectInvariant( + peerMatrixJob.steps.every((step) => step.run !== 'npm run lockfile:check'), + 'Peer matrix lanes must not repeat the lockfile dry-run; the pre-fan-out lockfile job owns it.', ); }); diff --git a/tests/suites/contracts/workflows/quality-node-compat.contract.spec.ts b/tests/suites/contracts/workflows/quality-node-compat.contract.spec.ts index e6f469d..821cf2b 100644 --- a/tests/suites/contracts/workflows/quality-node-compat.contract.spec.ts +++ b/tests/suites/contracts/workflows/quality-node-compat.contract.spec.ts @@ -17,88 +17,109 @@ const packageJson = JSON.parse(readFileSync(path.join(process.cwd(), 'package.js }; describe('quality workflow Node compatibility contract', () => { - it('includes Node 20, 22, and 24 in the Node compatibility matrix', () => { - const verifyJob = getWorkflowJob(qualityWorkflow, 'verify'); + it('runs the Node matrix for 20, 22, and 24 on runtime-sensitive behavior only', () => { + const nodeCompatJob = getWorkflowJob(qualityWorkflow, 'node-compat'); - expect(verifyJob.name).toBe('Node Compatibility (Node ${{ matrix.node-version }})'); + expect(nodeCompatJob.name).toBe('Node Compatibility (Node ${{ matrix.node-version }})'); expect( - getWorkflowMatrixValues(verifyJob, 'node-version'), + getWorkflowMatrixValues(nodeCompatJob, 'node-version'), 'Node compatibility matrix must cover supported Node floor/current/ceiling versions.', ).toEqual(['20', '22', '24']); + expect( + getWorkflowStep(nodeCompatJob, 'Run unit tests').run, + 'Node matrix must exercise runtime-sensitive unit behavior only.', + ).toBe('npm test'); }); it('declares engines range compatible with Node 24', () => { expect(packageJson.engines?.node).toBe('>=20 <25'); }); - it('keeps the Node matrix free of Docker and browser-heavy gates', () => { - const verifyJob = getWorkflowJob(qualityWorkflow, 'verify'); - const stepNames = verifyJob.steps.map((step) => step.name); + it('keeps runtime-insensitive static gates out of the Node matrix', () => { + const nodeCompatJob = getWorkflowJob(qualityWorkflow, 'node-compat'); + const stepNames = nodeCompatJob.steps.map((step) => step.name); - expect(getWorkflowStep(verifyJob, 'Run Node compatibility static gates').run).toBe( - 'npm run lint && npm run typecheck', - ); - expect(getWorkflowStep(verifyJob, 'Run unit tests').run).toBe('npm test'); - for (const blockedStepName of [ + for (const staticStepName of [ + 'Run format check', + 'Run lint', + 'Run typecheck', 'Run contract tests', 'Run Redis/OTLP integration tests', 'Validate artifact schemas', - 'Ensure Playwright Chrome is installed', + 'Run shell and workflow lint', + 'Run full Chrome E2E suite', ]) { expectInvariant( - !stepNames.includes(blockedStepName), - `Node compatibility job must not include heavy repository/browser gate: ${blockedStepName}.`, + !stepNames.includes(staticStepName), + `Node compatibility matrix must not repeat runtime-insensitive gate per Node version: ${staticStepName}.`, ); } + expectInvariant( + nodeCompatJob.steps.every( + (step) => !step.run?.includes('lint') && !step.run?.includes('typecheck'), + ), + 'Node matrix must not rerun lint/typecheck on every Node version.', + ); }); - it('runs contract, Redis required-mode, schema, shell, and workflow gates once on Node 22', () => { - const repositoryGatesJob = getWorkflowJob(qualityWorkflow, 'repository-gates'); - const stepNames = repositoryGatesJob.steps.map((step) => step.name); - const lockedInstallStep = getWorkflowStep( - repositoryGatesJob, - 'Setup locked Node.js dependencies', - ); + it('runs format, lint, typecheck, contracts, integration, schema, shell, and workflow gates once on Node 22', () => { + const staticJob = getWorkflowJob(qualityWorkflow, 'static-analysis'); + const lockedInstallStep = getWorkflowStep(staticJob, 'Setup locked Node.js dependencies'); - expect(repositoryGatesJob.name).toBe('Repository Gates (Node 22)'); + expect(staticJob.name).toBe('Static Analysis (Node 22)'); expect(lockedInstallStep.uses).toBe(lockedInstallActionPath); expect(lockedInstallStep.with.get('node-version')).toBe('22'); - expect(lockedInstallStep.with.get('cache-namespace')).toBe('repository-gates'); - expect(getWorkflowStep(repositoryGatesJob, 'Run format check').run).toBe( - 'npm run format:check', - ); - expect(getWorkflowStep(repositoryGatesJob, 'Run contract tests').run).toBe( - 'npm run test:contracts', - ); - expect(getWorkflowStep(repositoryGatesJob, 'Run Redis/OTLP integration tests').run).toBe( + expect(lockedInstallStep.with.get('cache-namespace')).toBe('static-analysis'); + expect(getWorkflowStep(staticJob, 'Run format check').run).toBe('npm run format:check'); + expect(getWorkflowStep(staticJob, 'Run lint').run).toBe('npm run lint'); + expect(getWorkflowStep(staticJob, 'Run typecheck').run).toBe('npm run typecheck'); + expect(getWorkflowStep(staticJob, 'Run contract tests').run).toBe('npm run test:contracts'); + expect(getWorkflowStep(staticJob, 'Run Redis/OTLP integration tests').run).toBe( 'npm run test:integration', ); expect( - getWorkflowStep(repositoryGatesJob, 'Run Redis/OTLP integration tests').env.get( + getWorkflowStep(staticJob, 'Run Redis/OTLP integration tests').env.get( 'AURORAFLOW_REDIS_INTEGRATION_REQUIRED', ), ).toBe('true'); - expect(getWorkflowStep(repositoryGatesJob, 'Validate artifact schemas').run).toBe( + expect(getWorkflowStep(staticJob, 'Validate artifact schemas').run).toBe( 'npm run schemas:check', ); - expect(getWorkflowStep(repositoryGatesJob, 'Run shell and workflow lint').run).toBe( - 'npm run shellcheck && npm run workflows:lint', + expect(getWorkflowStep(staticJob, 'Run shell and workflow lint').run).toBe( + 'npm run shellcheck && npm run workflows:lint:check', ); - expect(stepNames.indexOf('Run contract tests')).toBeLessThan( - stepNames.indexOf('Run Redis/OTLP integration tests'), + }); + + it('installs actionlint once and lints workflows through the non-installing command', () => { + const staticJob = getWorkflowJob(qualityWorkflow, 'static-analysis'); + const installSteps = staticJob.steps.filter((step) => step.run === 'npm run tools:actionlint'); + + expectInvariant( + installSteps.length === 1, + 'actionlint must be installed exactly once per static-analysis job.', + ); + const shellWorkflowStep = getWorkflowStep(staticJob, 'Run shell and workflow lint'); + expectTextIncludes(shellWorkflowStep.run ?? '', { + text: 'workflows:lint:check', + rationale: 'Workflow lint must reuse the installed actionlint, not reinstall it.', + }); + expectInvariant( + !(shellWorkflowStep.run ?? '').includes('npm run workflows:lint '), + 'Static-analysis must not invoke the reinstalling workflows:lint command.', ); }); - it('enforces critical and global coverage once on Node 22', () => { + it('enforces coverage floors in one complete unit coverage run on Node 22', () => { const coverageJob = getWorkflowJob(qualityWorkflow, 'coverage'); const setupNodeStep = getWorkflowStep(coverageJob, 'Setup locked Node.js dependencies'); - expect(coverageJob.name).toBe('Coverage (Critical + Global)'); + expect(coverageJob.name).toBe('Coverage (Unit Floors)'); expect(setupNodeStep.uses).toBe(lockedInstallActionPath); expect(setupNodeStep.with.get('node-version')).toBe('22'); expect(setupNodeStep.with.get('cache-namespace')).toBe('coverage'); expectTextIncludes( - getWorkflowStep(coverageJob, 'Enforce critical and global coverage thresholds').run ?? '', + getWorkflowStep(coverageJob, 'Enforce global and risk-weighted coverage thresholds').run ?? + '', { text: 'npm run test:coverage 2>&1 | tee coverage/coverage-gate.log', rationale: 'Coverage gate must preserve threshold output for actionable job summaries.', @@ -116,73 +137,69 @@ describe('quality workflow Node compatibility contract', () => { }); }); - it('makes Redis integration and guarded self-heal proof mandatory in quality gates', () => { - const repositoryGatesJob = getWorkflowJob(qualityWorkflow, 'repository-gates'); - const guardedSelfHealJob = getWorkflowJob(qualityWorkflow, 'guarded-self-heal'); - - expect( - getWorkflowStep(repositoryGatesJob, 'Run Redis/OTLP integration tests').env.get( - 'AURORAFLOW_REDIS_INTEGRATION_REQUIRED', - ), - ).toBe('true'); + it('keeps the required release verify path in Redis-required integration mode', () => { expect( getWorkflowStep( getWorkflowJob(releaseWorkflow, 'release-dry-run'), 'Run quality gates', ).env.get('AURORAFLOW_REDIS_INTEGRATION_REQUIRED'), ).toBe('true'); - expect(guardedSelfHealJob.name).toBe('Guarded Self-Heal Proof (Chrome)'); - expect(guardedSelfHealJob.needs).toEqual(['preflight', 'verify', 'repository-gates']); - expect(getWorkflowStep(guardedSelfHealJob, 'Prove guarded self-heal at default gate').run).toBe( - 'npm run test:e2e:guarded', - ); }); - it('adds path and label triggered full Chrome E2E without merging it into compatibility gates', () => { + it('consolidates smoke, guarded self-heal, examples, and risk into one event-aware Chrome lane', () => { const preflightJob = getWorkflowJob(qualityWorkflow, 'preflight'); - const riskE2eJob = getWorkflowJob(qualityWorkflow, 'risk-e2e'); - const pathsFilterStep = getWorkflowStep(preflightJob, 'Detect smoke-relevant changes'); - const shouldRunRiskE2e = riskE2eJob.env.get('SHOULD_RUN_RISK_E2E'); - if (shouldRunRiskE2e === undefined) { - throw new Error('risk-e2e job must declare SHOULD_RUN_RISK_E2E'); - } + const e2eJob = getWorkflowJob(qualityWorkflow, 'e2e-chrome'); + const pathsFilterStep = getWorkflowStep(preflightJob, 'Detect change scopes'); + expect(e2eJob.name).toBe('E2E (Chrome)'); + expect(e2eJob.needs).toEqual(['preflight', 'static-analysis', 'node-compat']); + expect(getWorkflowStep(e2eJob, 'Run full Chrome E2E suite').run).toBe( + "npm run test:e2e -- --project='Google Chrome'", + ); expect( - preflightJob.outputs.get('run_risk_e2e'), - 'Preflight job must expose path-filter output that gates risk-triggered E2E.', - ).toBe('${{ steps.paths-filter.outputs.risk_e2e }}'); + preflightJob.outputs.get('run_browser_e2e'), + 'Preflight must expose a browser-relevance output that gates the single Chrome lane.', + ).toBe('${{ steps.paths-filter.outputs.browser_e2e }}'); expectTextIncludes(pathsFilterStep.with.get('filters') ?? '', { - text: 'risk_e2e:', - rationale: 'Path filter config must include risk_e2e group for runtime/browser changes.', + text: 'browser_e2e:', + rationale: 'Path filter must define the browser_e2e scope for the consolidated Chrome lane.', }); - expect(riskE2eJob.name).toBe('Risk-Triggered E2E (Chrome)'); - expect(riskE2eJob.needs).toEqual(['preflight', 'verify', 'repository-gates']); + + const jobLevelGate = e2eJob.if ?? ''; for (const text of [ - "needs.preflight.outputs.run_risk_e2e == 'true'", + "needs.preflight.outputs.run_browser_e2e == 'true'", + "github.ref == 'refs/heads/main'", "'full-e2e'", "'risk:e2e'", ]) { - expectTextIncludes(shouldRunRiskE2e, { + expectTextIncludes(jobLevelGate, { text, - rationale: 'Risk E2E gate must respect paths plus explicit maintainer labels.', + rationale: 'Chrome lane must gate on browser relevance plus main and explicit labels.', }); } - const riskSetupStep = getWorkflowStep( - riskE2eJob, - 'Setup locked Node.js dependencies and Playwright browser', - ); - expect(riskSetupStep.uses).toBe(lockedInstallActionPath); - expect(riskSetupStep.with.get('install-browsers')).toBe('true'); - expect(riskSetupStep.with.get('browser-name')).toBe('chrome'); - expect(riskSetupStep.with.get('cache-namespace')).toBe('risk-chrome'); - expect(getWorkflowStep(riskE2eJob, 'Run full Chrome E2E suite').run).toBe( - "npm run test:e2e -- --project='Google Chrome'", - ); + + // The guarded self-heal proof and examples are folded into the full Chrome run, + // not separate jobs that would re-execute the same Playwright test IDs. + for (const retiredJobId of ['smoke-e2e', 'guarded-self-heal', 'risk-e2e']) { + expectInvariant( + !qualityWorkflow.jobs.has(retiredJobId), + `Retired duplicate Chrome lane must not remain: ${retiredJobId}.`, + ); + } expectInvariant( - getWorkflowJob(qualityWorkflow, 'verify').steps.every( + getWorkflowJob(qualityWorkflow, 'node-compat').steps.every( (step) => !step.run?.includes('test:e2e'), ), 'Node compatibility job must not run browser-heavy E2E commands.', ); }); + + it('replaces runner-consuming skip steps with job-level conditions', () => { + for (const job of qualityWorkflow.jobs.values()) { + expectInvariant( + job.steps.every((step) => !/completed as a no-op|no-op\.$/u.test(step.run ?? '')), + `Job ${job.id} must skip before runner allocation, not echo a no-op skip step.`, + ); + } + }); }); diff --git a/tests/suites/contracts/workflows/release-dry-run.contract.spec.ts b/tests/suites/contracts/workflows/release-dry-run.contract.spec.ts index ace1173..1ae8ca7 100644 --- a/tests/suites/contracts/workflows/release-dry-run.contract.spec.ts +++ b/tests/suites/contracts/workflows/release-dry-run.contract.spec.ts @@ -66,14 +66,14 @@ describe('release dry-run workflow contract', () => { 'Release workflow must not reference long-lived npm publish tokens.', ); const packCommand = getWorkflowStep(releaseJob, 'Pack package tarball').run ?? ''; - expectInvariant( - packCommand.includes('npm run pack:dry-run'), - 'Release workflow must keep pack dry-run evidence before creating the local validation tarball.', - ); expectInvariant( packCommand.includes('npm pack --json --pack-destination release-evidence'), 'Release workflow must emit a local tarball for consumer validation without publishing.', ); + expectInvariant( + !packCommand.includes('npm run pack:dry-run') && !packCommand.includes('npm pack --dry-run'), + 'Release workflow must perform one local pack; the dry-run pack is redundant with the real JSON pack report.', + ); }); it('pins every action to an immutable SHA and disables credential persistence', () => { @@ -96,6 +96,7 @@ describe('release dry-run workflow contract', () => { const setupStep = getWorkflowStep(releaseJob, 'Setup locked Node.js dependencies'); const consumerSmokeStep = getWorkflowStep(releaseJob, 'Validate package consumer install'); const validatorsStep = getWorkflowStep(releaseJob, 'Run package publish validators'); + const stepNames = releaseJob.steps.map((step) => step.name); expect(getWorkflowStep(releaseJob, 'Run quality gates').run).toBe('npm run verify'); expect(packageJson.packageManager).toBe('npm@11.17.0'); @@ -118,7 +119,12 @@ describe('release dry-run workflow contract', () => { ), 'Release dry-run evidence must tee schema validation output into the evidence bundle.', ); - expect(getWorkflowStep(releaseJob, 'Build package').run).toBe('npm run build'); + // One intentional build: the prepack build during `npm pack` is the single + // build, so no standalone `npm run build` step (which would rebuild) remains. + expectInvariant( + !stepNames.includes('Build package'), + 'Release must build once via the pack prepack hook, not a separate duplicate build step.', + ); expectInvariant( (consumerSmokeStep.run ?? '').includes( 'npm run package:consumer-smoke -- --pack-report release-evidence/pack-report.json 2>&1 | tee release-evidence/consumer-smoke.txt', diff --git a/tests/suites/contracts/workflows/security-secret-scan.contract.spec.ts b/tests/suites/contracts/workflows/security-secret-scan.contract.spec.ts index e65621d..7a208e4 100644 --- a/tests/suites/contracts/workflows/security-secret-scan.contract.spec.ts +++ b/tests/suites/contracts/workflows/security-secret-scan.contract.spec.ts @@ -155,6 +155,38 @@ describe('security workflow secret scanning contract', () => { }); }); + it('keeps the scanner effect-proof off ordinary pull requests and event-aware in the gate', () => { + const effectProofJob = getWorkflowJob(securityWorkflow, 'secret-scan-effect-proof'); + const preflightJob = getWorkflowJob(securityWorkflow, 'preflight'); + const securityGateRun = + getWorkflowStep( + getWorkflowJob(securityWorkflow, 'security-gate'), + 'Enforce upstream security job results', + ).run ?? ''; + + // Meta-test: runs on push/schedule/dispatch, and on PRs only when the scanner + // configuration itself changed -- not on every ordinary PR (spec section 8). + expect(effectProofJob.needs).toEqual(['preflight']); + for (const text of [ + "github.event_name != 'pull_request'", + "needs.preflight.outputs.run_effect_proof == 'true'", + ]) { + expectTextIncludes(effectProofJob.if ?? '', { + text, + rationale: 'Effect proof must be gated to non-PR events plus scanner-config PRs only.', + }); + } + expectTextIncludes(preflightJob.outputs.get('run_effect_proof') ?? '', { + text: 'steps.paths-filter.outputs.effect_proof', + rationale: 'Security preflight must expose the scanner-governing change signal.', + }); + expectTextIncludes(securityGateRun, { + text: 'require_skipped "Secret Scan Effect Proof" "$SECRET_SCAN_EFFECT_PROOF_RESULT"', + rationale: + 'Security gate must confirm the effect proof is intentionally skipped on ordinary PRs, not silently dropped.', + }); + }); + it('proves gitleaks catches a synthetic secret without weakening the pinned scan job', () => { const effectProofJob = getWorkflowJob(securityWorkflow, 'secret-scan-effect-proof'); const installStep = getWorkflowStep(effectProofJob, 'Install pinned gitleaks'); diff --git a/tests/suites/contracts/workflows/self-healing-governance.contract.spec.ts b/tests/suites/contracts/workflows/self-healing-governance.contract.spec.ts index f2a5734..2de9efa 100644 --- a/tests/suites/contracts/workflows/self-healing-governance.contract.spec.ts +++ b/tests/suites/contracts/workflows/self-healing-governance.contract.spec.ts @@ -12,7 +12,7 @@ const qualityWorkflow = readWorkflowModel('.github/workflows/quality.yml'); describe('self-healing governance workflow contract', () => { it('runs governance checks after smoke execution with explicit env controls', () => { const governanceStep = getWorkflowStep( - getWorkflowJob(qualityWorkflow, 'smoke-e2e'), + getWorkflowJob(qualityWorkflow, 'e2e-chrome'), 'Evaluate self-healing governance', ); @@ -23,7 +23,7 @@ describe('self-healing governance workflow contract', () => { }); it('exposes governance outputs for downstream triage handling', () => { - const smokeJob = getWorkflowJob(qualityWorkflow, 'smoke-e2e'); + const smokeJob = getWorkflowJob(qualityWorkflow, 'e2e-chrome'); expect(getWorkflowStepById(smokeJob, 'self-heal-governance').name).toBe( 'Evaluate self-healing governance', @@ -38,7 +38,7 @@ describe('self-healing governance workflow contract', () => { it('supports optional auto-triage issue creation with explicit opt-in', () => { const triageStep = getWorkflowStep( - getWorkflowJob(qualityWorkflow, 'smoke-e2e'), + getWorkflowJob(qualityWorkflow, 'e2e-chrome'), 'Auto-open self-healing triage issue', ); diff --git a/tests/suites/contracts/workflows/smoke-selection.contract.spec.ts b/tests/suites/contracts/workflows/smoke-selection.contract.spec.ts index 6fe8d84..34a7381 100644 --- a/tests/suites/contracts/workflows/smoke-selection.contract.spec.ts +++ b/tests/suites/contracts/workflows/smoke-selection.contract.spec.ts @@ -96,8 +96,7 @@ describe('smoke selection contract', () => { ), ).toBe('900919a84f2229bac68ca9cd4103ea297abc35e9689ebb842c6e34a3d1b01b0a'); expect( - getWorkflowStep(getWorkflowJob(qualityWorkflow, 'repository-gates'), 'Install actionlint') - .run, + getWorkflowStep(getWorkflowJob(qualityWorkflow, 'static-analysis'), 'Install actionlint').run, ).toBe('npm run tools:actionlint'); expect([...makeTargets]).toEqual(expect.arrayContaining(['tools', 'observability-smoke'])); }); @@ -146,6 +145,9 @@ describe('smoke selection contract', () => { 'tests/suites/e2e/examples/quickstart.spec.ts', 'tests/suites/e2e/examples/deterministic-network-mock.spec.ts', 'tests/suites/e2e/examples/retries-and-timeouts.spec.ts', + // The previously untagged deterministic SAT example is now part of smoke so + // no deterministic example test ID is left unselected (spec section 5.1). + 'tests/suites/e2e/examples/self-healing-sat.spec.ts', ] as const; for (const spec of exampleSpecs) { diff --git a/tests/suites/contracts/workflows/test-taxonomy.contract.spec.ts b/tests/suites/contracts/workflows/test-taxonomy.contract.spec.ts index 2770773..7a98387 100644 --- a/tests/suites/contracts/workflows/test-taxonomy.contract.spec.ts +++ b/tests/suites/contracts/workflows/test-taxonomy.contract.spec.ts @@ -144,7 +144,9 @@ describe('test script taxonomy contract', () => { } }); - it('requires contracts, Redis/OTLP integration, and schema validation in verify', () => { + it('requires contracts and Redis/OTLP integration in verify without a duplicate schema run', () => { + // Schema validation is captured once as CI/release evidence (npm run schemas:check), + // so verify intentionally excludes its own duplicate schema execution (spec section 4.4). expect(splitScriptSequence('verify')).toEqual([ 'npm run verify:tools', 'npm run format:check', @@ -153,7 +155,6 @@ describe('test script taxonomy contract', () => { 'npm run test:unit', 'npm run test:contracts', 'npm run test:integration', - 'npm run schemas:check', 'npm run shellcheck', 'npm run workflows:lint:check', ]); @@ -272,15 +273,15 @@ describe('test script taxonomy contract', () => { 'Same Redis/OTLP integration suite, but Redis startup/connect failures fail instead of skip.', }); expect(rows.get('`npm run test:coverage`')?.scope).toBe( - 'Critical-module thresholds plus global `src/**` coverage.', + 'One complete unit coverage run enforcing the global `src/**` floor and every risk-weighted per-file floor.', ); expect(rows.get('`npm run test:e2e:guarded`')?.scope).toBe( 'Parallel Chrome proof for guarded self-heal at the default gate.', ); for (const text of [ 'Node Compatibility (Node 20/22/24)', - 'Repository Gates (Node 22)', - 'Risk-Triggered E2E (Chrome)', + 'Static Analysis (Node 22)', + 'E2E (Chrome)', 'risk-weighted per-file floors for high-risk modules once on Node 22.', 'Default local Redis behavior is skip-friendly', ]) { diff --git a/tests/suites/e2e/examples/self-healing-sat.spec.ts b/tests/suites/e2e/examples/self-healing-sat.spec.ts index b74ee5f..4140254 100644 --- a/tests/suites/e2e/examples/self-healing-sat.spec.ts +++ b/tests/suites/e2e/examples/self-healing-sat.spec.ts @@ -27,7 +27,7 @@ class SelfHealingSatFixturePage extends PageObjectBase { } } -test('self-healing SAT enriches a deterministic failed page-object action', async ({ +test('@smoke self-healing SAT enriches a deterministic failed page-object action', async ({ page, }, testInfo) => { const artifactScope = await createPlaywrightSelfHealingArtifactScope(testInfo, {