diff --git a/.github/scripts/post-commit-status.sh b/.github/scripts/post-commit-status.sh deleted file mode 100755 index 55a1213f..00000000 --- a/.github/scripts/post-commit-status.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# Post a commit status to a PR head SHA. -# -# Usage: post-commit-status.sh [description] [target-url] -# state ∈ {pending, success, failure, error} -# -# The status context is fixed to "bench-ok" — the name branch protection -# requires. Splitting benchmark execution across two workflows (a fork-safe -# `pull_request` planner and a trusted `workflow_run` executor) means the real -# pass/fail verdict is posted here from the trusted run, while the check keeps -# the single stable name the required-checks list already knows. -# -# Requires GH_TOKEN with `statuses: write`. In the trusted workflow_run context -# the default GITHUB_TOKEN has it; in the fork `pull_request` run it can still -# write a status to its own head SHA (pending / short-circuit success). -# -# The SHA is fork-controlled (it arrives via a handoff artifact), so validate it -# is a full 40-hex commit id before using it in the API path — same defensive -# posture as the numeric PR-number guard in the workflow_run handoff. - -set -euo pipefail - -SHA="$1" -STATE="$2" -DESCRIPTION="${3:-}" -TARGET_URL="${4:-}" - -if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then - echo "ERROR: refusing to post status — '$SHA' is not a 40-hex commit SHA." >&2 - exit 1 -fi - -case "$STATE" in - pending | success | failure | error) ;; - *) - echo "ERROR: invalid state '$STATE' (want pending|success|failure|error)." >&2 - exit 1 - ;; -esac - -# GitHub caps status descriptions at 140 chars; trim defensively. -DESCRIPTION="${DESCRIPTION:0:140}" - -ARGS=( - --method POST - "repos/${GITHUB_REPOSITORY}/statuses/${SHA}" - -f "state=${STATE}" - -f "context=bench-ok" -) -[[ -n "$DESCRIPTION" ]] && ARGS+=(-f "description=${DESCRIPTION}") -[[ -n "$TARGET_URL" ]] && ARGS+=(-f "target_url=${TARGET_URL}") - -echo "Posting bench-ok=${STATE} to ${SHA}" -gh api "${ARGS[@]}" diff --git a/.github/scripts/pull-solver-images.py b/.github/scripts/pull-solver-images.py index 7dd67b3f..72dcf6c5 100644 --- a/.github/scripts/pull-solver-images.py +++ b/.github/scripts/pull-solver-images.py @@ -9,20 +9,25 @@ pulls the image from the registry and retags it to the local short name expected by ``Tesseract.from_image()``. -Images are pulled by commit-SHA tag, never ``:latest``. ``--tag`` is this -PR's HEAD (built by the build job for changed solvers); ``--fallback-tag`` is -the base/main commit (always built by the push-to-main run and immutable). -Both are immutable, so a plain ``docker pull`` of the tag is unambiguous — -unlike ``:latest``, a runner-cached or not-yet-propagated SHA tag can't point -at the wrong image. +Images are pulled by commit-SHA tag, never ``:latest``: an immutable SHA tag +makes ``docker pull`` unambiguous, whereas a runner-cached or not-yet-propagated +``:latest`` can point at the wrong image. + +In CI, PR runs build the *changed* solvers and hand them to the benchmark run +as image artifacts (``docker load``, not a registry pull), so this script only +fetches the *unchanged* solvers. Those are pulled at ``--fallback-tag`` (the +PR's base/main commit, always published by the push-to-main build). The +already-loaded solvers are passed via ``--skip-images`` so they are not +re-fetched at a SHA where they were never published. ``--tag`` remains for +direct/manual use (pull a specific SHA first, then fall back). Usage (in CI): python .github/scripts/pull-solver-images.py \ --registry ghcr.io/org/mosaic \ --problems all \ --hardware gpu \ - --tag \ - --fallback-tag + --fallback-tag \ + --skip-images , """ from __future__ import annotations @@ -71,6 +76,14 @@ def main() -> None: help="Comma-separated solver display names to restrict pulling to. " "When set, only matching solvers are pulled; others are skipped.", ) + parser.add_argument( + "--skip-images", + default=None, + help="Comma-separated image names (the tag's repository part, e.g. " + "'ins_navier_stokes_grid') to skip pulling. Used when those images are " + "already present locally — e.g. loaded from a build artifact on a PR — " + "so the registry pull only covers the unchanged solvers.", + ) args = parser.parse_args() registry = args.registry.lower().rstrip("/") @@ -83,6 +96,10 @@ def main() -> None: if args.solvers: solver_filter = {s.strip() for s in args.solvers.split(",") if s.strip()} + skip_images: set[str] = set() + if args.skip_images: + skip_images = {s.strip() for s in args.skip_images.split(",") if s.strip()} + seen: set[str] = set() failed: list[str] = [] pulled_count = 0 @@ -109,6 +126,13 @@ def main() -> None: local_tag = tag image_name = tag.rsplit(":", 1)[0] + # Already present locally (e.g. docker-loaded from a PR build + # artifact) → don't pull, it would only fail at a SHA where this + # changed solver was never published. + if image_name in skip_images: + print(f"Skipping {image_name} (already loaded locally)") + continue + # Pull by SHA: --tag (HEAD) first, then --fallback-tag (base/main). # Both are immutable commit SHAs, so the right image is unambiguous # — no :latest, hence no stale/racy pointer (the bug that made a diff --git a/.github/workflows/benchmark-execute.yml b/.github/workflows/benchmark-execute.yml deleted file mode 100644 index 204ea98d..00000000 --- a/.github/workflows/benchmark-execute.yml +++ /dev/null @@ -1,431 +0,0 @@ -name: Benchmark execute - -# Trusted half of the fork-safe benchmark flow. The Benchmark workflow runs on -# `pull_request` — on fork PRs it gets a read-only token and no secrets, so it -# can only plan and upload a metadata handoff. This workflow fills the gap. -# -# `workflow_run` always runs in the BASE repo's context, using the workflow -# definition from the default branch — trusted code with `packages: write` and -# full secret access — regardless of whether the triggering run came from a -# fork. It therefore does everything the planner couldn't: -# -# • build the PR's changed solver images and push them to (private) GHCR, -# • run the benchmarks (pulling the private images it just pushed, plus the -# immutable base-SHA images for unchanged solvers), -# • render the docs preview, and -# • post the `bench-ok` commit status + the PR comment / RTD preview. -# -# SECURITY: this run holds write scope and secrets, so it must never execute -# PR-authored *workflow/script* code. It checks out the base repo's default -# branch for its own scripts; the only PR-authored code it touches is the -# solver source it builds into a container and runs in the same sandboxed -# runners same-repo PRs already use. Every fork-controlled value (SHAs, PR -# number, label, matrices) arrives via the handoff artifact and is re-validated -# before use — same posture the retired docs-publish.yml used. - -on: - workflow_run: - workflows: ["Benchmark"] - types: [completed] - -permissions: - contents: read - packages: write # push/pull private solver images - statuses: write # post the bench-ok commit status - actions: read # download the handoff artifact from the triggering run - -concurrency: - # Serialize per triggering PR head; a newer push supersedes an in-flight run. - group: benchmark-execute-${{ github.event.workflow_run.head_sha }} - cancel-in-progress: true - -jobs: - # ── Recover + validate the handoff from the triggering PR run ────────── - recover-meta: - name: Recover handoff - # Only PR-triggered Benchmark runs that succeeded (a failed planner already - # posted bench-ok=failure and there's nothing to execute). - if: >- - github.event.workflow_run.event == 'pull_request' - && github.event.workflow_run.conclusion == 'success' - runs-on: ubuntu-latest - outputs: - found: ${{ steps.parse.outputs.found }} - should_run: ${{ steps.parse.outputs.should_run }} - pr_number: ${{ steps.parse.outputs.pr_number }} - head_sha: ${{ steps.parse.outputs.head_sha }} - base_sha: ${{ steps.parse.outputs.base_sha }} - label: ${{ steps.parse.outputs.label }} - is_release_pr: ${{ steps.parse.outputs.is_release_pr }} - problems: ${{ steps.parse.outputs.problems }} - suites: ${{ steps.parse.outputs.suites }} - solvers: ${{ steps.parse.outputs.solvers }} - solver_matrix: ${{ steps.parse.outputs.solver_matrix }} - run_matrix: ${{ steps.parse.outputs.run_matrix }} - steps: - - name: Download handoff - id: dl - uses: actions/download-artifact@v8 - continue-on-error: true - with: - pattern: bench-handoff-* - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - path: handoff/ - merge-multiple: true - - - name: Parse and validate handoff - id: parse - run: | - set -euo pipefail - META="handoff/handoff-meta.txt" - if [[ ! -f "$META" ]]; then - echo "No handoff metadata — nothing to execute." - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - get() { grep -oP "^$1=\K.*" "$META" || true; } - PR_NUMBER="$(get pr_number)" - HEAD_SHA="$(get head_sha)" - BASE_SHA="$(get base_sha)" - SHOULD_RUN="$(get should_run)" - LABEL="$(get label)" - IS_RELEASE_PR="$(get is_release_pr)" - PROBLEMS="$(get problems)" - SUITES="$(get suites)" - SOLVERS="$(get solvers)" - - # ── Validate every fork-controlled value before use ────────────── - fail() { echo "::warning::$1 — skipping."; echo "found=false" >> "$GITHUB_OUTPUT"; exit 0; } - [[ "$PR_NUMBER" =~ ^[0-9]+$ ]] || fail "PR number '$PR_NUMBER' not numeric" - [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || fail "head_sha '$HEAD_SHA' not a 40-hex SHA" - [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] || fail "base_sha '$BASE_SHA' not a 40-hex SHA" - case "$SHOULD_RUN" in true | false) ;; *) fail "should_run '$SHOULD_RUN' invalid" ;; esac - case "$LABEL" in none | solver | all | "") ;; *) fail "label '$LABEL' invalid" ;; esac - case "$IS_RELEASE_PR" in true | false) ;; *) fail "is_release_pr '$IS_RELEASE_PR' invalid" ;; esac - - # Matrices are JSON; validate they parse and re-emit compactly. - SOLVER_MATRIX="$(python3 -c 'import json,sys; print(json.dumps(json.load(open("handoff/solver-matrix.json"))))' 2>/dev/null || echo "INVALID")" - [[ "$SOLVER_MATRIX" != "INVALID" ]] || fail "solver-matrix.json not valid JSON" - # run-matrix is only present/needed when benchmarks run. - if [[ "$SHOULD_RUN" == "true" ]]; then - RUN_MATRIX="$(python3 -c 'import json,sys; print(json.dumps(json.load(open("handoff/run-matrix.json"))))' 2>/dev/null || echo "INVALID")" - [[ "$RUN_MATRIX" != "INVALID" ]] || fail "run-matrix.json not valid JSON" - else - RUN_MATRIX='{"include":[]}' - fi - - { - echo "found=true" - echo "pr_number=$PR_NUMBER" - echo "head_sha=$HEAD_SHA" - echo "base_sha=$BASE_SHA" - echo "should_run=$SHOULD_RUN" - echo "label=$LABEL" - echo "is_release_pr=$IS_RELEASE_PR" - echo "problems=$PROBLEMS" - echo "suites=$SUITES" - echo "solvers=$SOLVERS" - echo "solver_matrix=$SOLVER_MATRIX" - echo "run_matrix=$RUN_MATRIX" - } >> "$GITHUB_OUTPUT" - echo "Recovered handoff for PR #$PR_NUMBER (head $HEAD_SHA, should_run=$SHOULD_RUN)" - - # ── Build changed solver images → private GHCR : ───────────── - build: - name: Build solver images - needs: [recover-meta] - if: >- - needs.recover-meta.outputs.found == 'true' - && needs.recover-meta.outputs.should_run == 'true' - && needs.recover-meta.outputs.solver_matrix != '[]' - uses: ./.github/workflows/build-tesseracts.yml - with: - matrix: ${{ needs.recover-meta.outputs.solver_matrix }} - ref: ${{ needs.recover-meta.outputs.head_sha }} - image_sha: ${{ needs.recover-meta.outputs.head_sha }} - permissions: - contents: read - packages: write - # Deliberately NOT `secrets: inherit`. This job checks out and executes - # PR-authored code (the solver's build_base.sh and Dockerfile) to build the - # image, so it must see the smallest possible secret surface. build-tesseracts - # needs only the auto-provided GITHUB_TOKEN for GHCR; RTD_TOKEN / the bot PAT - # stay out of reach of fork build scripts. (The images are then run in the - # same sandboxed runners same-repo PRs already use.) - - # ── Run benchmarks (ics + solver suites) ─────────────────────────────── - benchmarks: - name: Run benchmarks - needs: [recover-meta, build] - if: >- - always() && !cancelled() - && needs.recover-meta.outputs.found == 'true' - && needs.recover-meta.outputs.should_run == 'true' - && (needs.build.result == 'success' || needs.build.result == 'skipped') - uses: ./.github/workflows/benchmark-run.yml - with: - matrix: ${{ needs.recover-meta.outputs.run_matrix }} - problems: ${{ needs.recover-meta.outputs.problems }} - suites: ${{ needs.recover-meta.outputs.suites }} - solvers: ${{ needs.recover-meta.outputs.solvers }} - # Primary tag: this PR's HEAD (built by the build job above). Fallback: - # the PR's base SHA — always built by the push-to-main run and immutable, - # so unchanged solvers pull a stable image rather than the mutable :latest. - image_tag: ${{ needs.recover-meta.outputs.head_sha }} - base_image_tag: ${{ needs.recover-meta.outputs.base_sha }} - timeout: 600 - permissions: - contents: read - packages: read - # No `secrets: inherit`: this job runs the fork-built solver container, so - # it sees the smallest secret surface. benchmark-run.yml needs only the - # auto-provided GITHUB_TOKEN (GHCR pull); RTD_TOKEN / the bot PAT stay out. - - # ── Merge results + render docs preview ───────────────────────────────── - # Runs for every recovered PR (even benchmark:none / docs-only), so the RTD - # preview always reflects this PR. Benchmark-result steps are guarded on - # should_run; the render always runs. Mirrors the old benchmark.yml `report` - # job, but here in the trusted context it also consumes the results produced - # by the `benchmarks` job in *this* run. - report: - name: Report & render - needs: [recover-meta, benchmarks] - if: >- - always() && !cancelled() - && needs.recover-meta.outputs.found == 'true' - && (needs.benchmarks.result == 'success' || needs.benchmarks.result == 'skipped') - runs-on: ubuntu-latest - outputs: - report_ok: ${{ steps.done.outputs.ok }} - steps: - - uses: actions/checkout@v6 - # Base-repo default branch: our own trusted scripts, never PR code. - - - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Install Mosaic - run: cp production.uv.lock uv.lock && uv sync --frozen - - - name: Download all results - # Only runs that ran benchmarks produce results-* artifacts. They live - # in *this* run (benchmarks job above), so a same-run download works. - if: needs.recover-meta.outputs.should_run == 'true' - uses: actions/download-artifact@v8 - with: - pattern: results-* - path: staging-results/ - - - name: Merge results from CPU/GPU artifacts - if: needs.recover-meta.outputs.should_run == 'true' - run: | - mkdir -p staging-results - uv run python .github/scripts/merge-results.py staging-results/ mosaic-results/ - - - name: Download baseline artifact - if: needs.recover-meta.outputs.is_release_pr != 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPO: ${{ github.repository }} - run: | - # Exit 2 = baseline not found (non-fatal: first run has no baseline). - bash .github/scripts/fetch-artifact.sh baseline baseline/mosaic-results/ \ - || [ "$?" -eq 2 ] - - - name: Overlay PR results on baseline - if: needs.recover-meta.outputs.is_release_pr != 'true' - run: | - if [ -d baseline/mosaic-results ] && [ "$(ls baseline/mosaic-results/ 2>/dev/null)" ]; then - if [ -d mosaic-results ] && [ "$(ls mosaic-results/ 2>/dev/null)" ]; then - MERGED=$(mktemp -d) - uv run python .github/scripts/overlay-results.py \ - baseline/mosaic-results mosaic-results "$MERGED" - rm -rf mosaic-results - mv "$MERGED" mosaic-results - else - rm -rf mosaic-results - cp -r baseline/mosaic-results mosaic-results - fi - fi - - - name: Extract baseline snapshot - id: baseline - if: needs.recover-meta.outputs.should_run == 'true' - run: | - BASELINE_PATH="" - if [ -f baseline/mosaic-results/snapshot.json ]; then - cp baseline/mosaic-results/snapshot.json baseline-snapshot.json - BASELINE_PATH="baseline-snapshot.json" - echo "Fetched baseline snapshot" - fi - echo "path=$BASELINE_PATH" >> "$GITHUB_OUTPUT" - - - name: Generate status snapshot (JSON) - if: needs.recover-meta.outputs.should_run == 'true' - run: uv run mosaic status --format json > snapshot.json - - - name: Generate status report (markdown) - if: needs.recover-meta.outputs.should_run == 'true' - run: | - DIFF_FLAG="" - if [[ -n "${{ steps.baseline.outputs.path }}" ]]; then - DIFF_FLAG="--diff-against ${{ steps.baseline.outputs.path }}" - fi - uv run mosaic status --format md $DIFF_FLAG > status-report.md - - - name: Set up Quarto - uses: quarto-dev/quarto-actions/setup@v2 - with: - version: "1.7.31" - - - name: Regenerate plots from merged results - run: uv run mosaic run --plots-only - - - name: Generate results pages - run: uv run python docs/generate_results.py - - - name: Render docs site - run: uv run quarto render --output-dir _site - - - name: Upload rendered docs site - uses: actions/upload-artifact@v7 - with: - name: docs-site-pr-${{ needs.recover-meta.outputs.pr_number }} - path: _site/ - overwrite: true - retention-days: 90 - - - name: Stash status report for finalize - if: needs.recover-meta.outputs.should_run == 'true' - uses: actions/upload-artifact@v7 - with: - name: status-report-${{ needs.recover-meta.outputs.pr_number }} - path: status-report.md - if-no-files-found: ignore - overwrite: true - retention-days: 7 - - - id: done - run: echo "ok=true" >> "$GITHUB_OUTPUT" - - # ── Post the bench-ok status + RTD preview + PR comment ──────────────── - # Runs for EVERY triggering PR run (not gated on found/should_run), because - # its first job is the terminal-state guarantee: whenever the planner seeded - # bench-ok=pending, something here must resolve it, or the PR is stuck - # unmergeable forever. So this job always posts a definitive bench-ok: - # - # • handoff missing/invalid (found=false) → error (couldn't verify) - # • benchmarks ran (should_run=true) → success | failure by results - # • nothing ran (should_run=false) → skip; plan-gate seeded success - # - # The head SHA comes from the validated handoff when present, else from - # github.event.workflow_run.head_sha (the PR head of the triggering run) — - # both identify the same commit; post-commit-status.sh re-validates it. - finalize: - name: Finalize (status, RTD, comment) - needs: [recover-meta, build, benchmarks, report] - if: always() - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - # ── bench-ok commit status: the merge gate + deadlock backstop ──── - # Only act when the PLANNER SUCCEEDED. If it concluded failure/cancelled - # (policy block, or a handoff-upload failure), plan-gate already owns the - # terminal bench-ok — either it posted `failure` for the policy violation, - # or nothing was seeded. Posting here in that case would clobber a correct - # status (e.g. flip a docs-only PR's green to error). So this step runs - # only for a successful planner run, which is exactly when a pending may - # have been seeded and we owe the terminal verdict. - - name: Post bench-ok status - if: github.event.workflow_run.conclusion == 'success' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FOUND: ${{ needs.recover-meta.outputs.found }} - SHOULD_RUN: ${{ needs.recover-meta.outputs.should_run }} - META_SHA: ${{ needs.recover-meta.outputs.head_sha }} - EVENT_SHA: ${{ github.event.workflow_run.head_sha }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - HEAD_SHA="${META_SHA:-$EVENT_SHA}" - - if [[ "$FOUND" != "true" ]]; then - # Planner succeeded (so a pending was seeded) but we couldn't recover - # the handoff — resolve the pending as error rather than orphan it. - if [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then - bash .github/scripts/post-commit-status.sh "$HEAD_SHA" error \ - "Could not recover benchmark handoff — re-run required" "$RUN_URL" - else - echo "No valid head SHA available — cannot post status." - fi - exit 0 - fi - - # benchmark:none / docs-only: no benchmarks ran, so bench-ok is - # success. plan-gate already seeded this, but re-assert it as the - # authoritative terminal post in case the fork seed couldn't write. - if [[ "$SHOULD_RUN" != "true" ]]; then - bash .github/scripts/post-commit-status.sh "$HEAD_SHA" success \ - "No benchmarks to run for this PR" "$RUN_URL" - exit 0 - fi - - BUILD="${{ needs.build.result }}" - BENCH="${{ needs.benchmarks.result }}" - REPORT="${{ needs.report.result }}" - echo "build=$BUILD benchmarks=$BENCH report=$REPORT" - - STATE="success"; DESC="Benchmarks passed" - for r in "$BUILD" "$BENCH" "$REPORT"; do - if [[ "$r" != "success" && "$r" != "skipped" ]]; then - STATE="failure"; DESC="A benchmark job failed" - fi - done - bash .github/scripts/post-commit-status.sh "$HEAD_SHA" "$STATE" "$DESC" "$RUN_URL" - - # The remaining steps (RTD, PR comment) only make sense with a recovered - # handoff; skip them cleanly when found=false. - - name: Download status report - if: needs.recover-meta.outputs.found == 'true' && needs.recover-meta.outputs.should_run == 'true' - continue-on-error: true - uses: actions/download-artifact@v8 - with: - name: status-report-${{ needs.recover-meta.outputs.pr_number }} - path: report/ - - # ── Trigger RTD preview rebuild + resolve URL (best-effort) ──────── - - name: Trigger RTD PR preview rebuild - if: needs.recover-meta.outputs.found == 'true' - continue-on-error: true - env: - RTD_TOKEN: ${{ secrets.RTD_TOKEN }} - RTD_PROJECT: ${{ vars.RTD_PROJECT }} - run: bash .github/scripts/rtd-trigger.sh "${{ needs.recover-meta.outputs.pr_number }}" - - - name: Resolve RTD preview URL - id: rtd_url - if: needs.recover-meta.outputs.found == 'true' - continue-on-error: true - env: - RTD_TOKEN: ${{ secrets.RTD_TOKEN }} - RTD_PROJECT: ${{ vars.RTD_PROJECT }} - run: bash .github/scripts/rtd-preview-url.sh "${{ needs.recover-meta.outputs.pr_number }}" - - # ── Post / update the PR status comment ─────────────────────────── - - name: Post status comment on PR - if: needs.recover-meta.outputs.found == 'true' - env: - GH_TOKEN: ${{ secrets.PL_PASTEURBOT_PAT_PUBLIC }} - GITHUB_REPOSITORY: ${{ github.repository }} - DOCS_PREVIEW_URL: ${{ steps.rtd_url.outputs.preview_url }} - run: | - REPORT="" - if [[ -f report/status-report.md ]]; then - REPORT="report/status-report.md" - fi - .github/scripts/post-status-comment.sh \ - "${{ needs.recover-meta.outputs.pr_number }}" \ - "$REPORT" diff --git a/.github/workflows/benchmark-run.yml b/.github/workflows/benchmark-run.yml index cc90e6cb..a92f3cc6 100644 --- a/.github/workflows/benchmark-run.yml +++ b/.github/workflows/benchmark-run.yml @@ -15,14 +15,11 @@ on: description: "Comma-separated suites (or 'all') — used for IC suite filter" type: string default: "all" - image_tag: - description: "GHCR image tag to pull (typically a commit SHA)" - type: string - required: true base_image_tag: description: >- - Fallback GHCR tag (the base/main commit SHA) for solvers not built at - image_tag. Always an immutable, already-built SHA — never :latest. + GHCR tag (the base/main commit SHA) to pull the unchanged solvers at. + Always an immutable, already-built SHA — never :latest. Changed + solvers are loaded from build artifacts, not pulled. type: string default: "" solvers: @@ -94,7 +91,42 @@ jobs: - name: Install Mosaic run: cp production.uv.lock uv.lock && uv sync --frozen - - name: Log in to GHCR + # ── Load PR-built solver images from artifacts ──────────────────── + # The build job exported each changed solver's image as a `solver-image-*` + # artifact (a fork PR can't push to GHCR). Load them back to + # `:latest` — the local tag `mosaic run --no-build` resolves — + # so they're interchangeable with a registry pull. Absent on runs that + # built nothing (e.g. benchmark:all pulling everything from base SHA). + - name: Download PR-built solver images + id: dl_images + continue-on-error: true # no artifacts on runs that rebuilt nothing + uses: actions/download-artifact@v8 + with: + pattern: solver-image-* + path: solver-images/ + merge-multiple: true + + - name: Load solver images and list them + id: loaded + run: | + LOADED="" + shopt -s nullglob + for tar in solver-images/*.tar; do + echo "Loading $tar" + docker load -i "$tar" + NAME="$(basename "$tar" .tar)" + LOADED="${LOADED:+$LOADED,}${NAME}" + done + echo "loaded_images=$LOADED" >> "$GITHUB_OUTPUT" + echo "Loaded images: ${LOADED:-}" + + # ── Pull the remaining (unchanged) solvers from GHCR ────────────── + # Unchanged solvers are pulled at the base/main SHA (immutable, always + # published by the push-to-main build). The GHCR packages are public, so + # no login is required — this works from fork PRs. Login is best-effort to + # raise rate limits when a token happens to be available. + - name: Log in to GHCR (best-effort) + continue-on-error: true uses: docker/login-action@v3 with: registry: ghcr.io @@ -105,13 +137,16 @@ jobs: run: | SOLVERS="${{ inputs.solvers }}" BASE_TAG="${{ inputs.base_image_tag }}" + LOADED="${{ steps.loaded.outputs.loaded_images }}" + # No --tag: the PR-head SHA is never published (images come from + # artifacts). Everything not loaded locally is pulled at the base SHA. uv run python .github/scripts/pull-solver-images.py \ --registry "ghcr.io/${{ github.repository_owner }}/mosaic" \ --problems "${{ matrix.problem }}" \ --hardware "${{ matrix.hardware }}" \ - --tag "${{ inputs.image_tag }}" \ ${BASE_TAG:+--fallback-tag "$BASE_TAG"} \ - ${SOLVERS:+--solvers "$SOLVERS"} + ${SOLVERS:+--solvers "$SOLVERS"} \ + ${LOADED:+--skip-images "$LOADED"} - name: Run ${{ matrix.suite }} | ${{ matrix.problem }} | ${{ matrix.hardware }} run: | diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b4bcdc61..5f04f67c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -12,19 +12,10 @@ on: description: "Comma-separated suites to run (or 'all')" default: "all" -# This workflow runs on `pull_request`, so on fork PRs it gets a read-only -# token and no repo secrets. It therefore does only the secret-free work: -# planning and a metadata handoff. The privileged half — building solver -# images, pushing to (private) GHCR, running benchmarks, rendering docs, and -# commenting — lives in benchmark-execute.yml, which fires on `workflow_run` -# in the trusted base-repo context. See that file's header for the trust model. -# -# `statuses: write` lets the plan-gate job post the `bench-ok` commit status -# (the name branch protection requires) to this PR's head SHA — pending when a -# trusted run will follow, or an immediate success/failure when it won't. permissions: contents: read - statuses: write + pull-requests: write + packages: read actions: read concurrency: @@ -185,150 +176,293 @@ jobs: echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "Matrix: $MATRIX" - # Guard: should_run=true but an empty include matrix means the plan - # decided to run yet generated no jobs (e.g. the should_run filter and - # the matrix generator disagreed). That would yield a green bench-ok - # with nothing benchmarked — fail loudly instead of passing silently. - N=$(echo "$MATRIX" | python3 -c "import json,sys; print(len(json.load(sys.stdin).get('include', [])))") - if [[ "$N" -eq 0 ]]; then - echo "::error::should_run=true but the benchmark matrix is empty — planning inconsistency." - exit 1 - fi + # ── Build changed solver images → image artifacts ───────────────────── + # On PRs the build exports each changed solver as a `solver-image-*` + # artifact rather than pushing to GHCR: a fork PR's token can neither push + # to nor (for a private package) pull from GHCR. The benchmarks job below + # loads those artifacts. No packages:write needed — the PR path never pushes. + build: + name: Build solver images + needs: plan + if: needs.plan.outputs.should_run == 'true' && needs.plan.outputs.solver_matrix != '[]' + uses: ./.github/workflows/build-tesseracts.yml + with: + matrix: ${{ needs.plan.outputs.solver_matrix }} + permissions: + contents: read + # Must satisfy build-tesseracts.yml's top-level `packages: write` (a + # reusable workflow can't be granted less than it declares, or the run + # fails at startup). The PR path never pushes — the push steps are gated + # to non-workflow_call — so this scope simply goes unused on PRs. + packages: write + secrets: inherit - # ── Plan-time merge gate + bench-ok status seeding ───────────────────── - # Renamed from `bench-ok` (that name is now the *commit status* the trusted - # benchmark-execute run posts — see below). This job enforces only what is - # decidable at plan time (the label policy) and seeds the `bench-ok` status: - # - # • policy violation (mosaic/ touched, no label, non-release) → failure - # • nothing will run (benchmark:none, no solver change, docs-only) → the - # trusted run still renders docs but posts no gate, so seed success here - # • a trusted run will run benchmarks → seed pending; benchmark-execute - # flips it to success/failure when it finishes - # - # Branch protection keeps requiring the `bench-ok` context, so the merge - # can't go green until either this job short-circuits it or the trusted run - # reports real results. `github.actor` from a fork can still write a status - # to its own head SHA with the (read-only-elsewhere) default token. - plan-gate: - name: Plan gate - if: always() - needs: [plan] + # ── Run benchmarks (ics + solver suites) ─────────────────────────────── + benchmarks: + name: Run benchmarks + needs: [plan, build] + if: always() && needs.plan.outputs.should_run == 'true' && !cancelled() && needs.plan.result == 'success' && (needs.build.result == 'success' || needs.build.result == 'skipped') + uses: ./.github/workflows/benchmark-run.yml + with: + matrix: ${{ needs.plan.outputs.matrix }} + problems: ${{ needs.plan.outputs.problems }} + suites: ${{ needs.plan.outputs.suites }} + solvers: ${{ needs.plan.outputs.solvers }} + # Changed solvers come from the build's image artifacts (loaded in the run + # job). Unchanged solvers are pulled from GHCR at the PR's base SHA — the + # main commit this branch sits on, always published by the push-to-main + # build and immutable, so no stale/racy :latest is ever served. GHCR is + # public-read, so the pull works from fork PRs without a push. + base_image_tag: ${{ github.event.pull_request.base.sha || github.sha }} + timeout: 600 + permissions: + contents: read + actions: read # download the solver-image-* artifacts + packages: read + secrets: inherit + + # ── Merge results, report, publish ───────────────────────────────────── + report: + name: Report & publish + needs: [plan, benchmarks] + # Runs on every PR (and every dispatch where benchmarks ran), even for + # benchmark:none PRs that ran no benchmarks. The benchmark-result steps + # below are guarded on should_run; the docs render + upload steps run + # regardless so doc-only PRs still get a Read the Docs preview. A + # benchmark:none PR still fetches the published baseline, so its preview + # shows the full site populated with the last released results. + # + # This job runs in the PR head's context and uses NO repo secrets, so it + # works identically for fork PRs. The secret-requiring follow-up (trigger + # the RTD rebuild, resolve the preview URL, post the PR comment) lives in + # docs-publish.yml, which fires on workflow_run in the base-repo context. + if: >- + always() && !cancelled() + && needs.plan.result == 'success' + && (github.event_name == 'pull_request' || needs.plan.outputs.should_run == 'true') runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - name: Evaluate policy and seed bench-ok status - if: github.event_name == 'pull_request' + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install Mosaic + run: cp production.uv.lock uv.lock && uv sync --frozen + + - name: Download all results + # Only PR runs that actually ran benchmarks produce results-* artifacts. + # benchmark:none PRs skip this and rely on the baseline below. + if: needs.plan.outputs.should_run == 'true' + uses: actions/download-artifact@v8 + with: + pattern: results-* + path: staging-results/ + + - name: Merge results from CPU/GPU artifacts + if: needs.plan.outputs.should_run == 'true' + run: | + # download-artifact doesn't create the path dir when nothing matches. + mkdir -p staging-results + uv run python .github/scripts/merge-results.py staging-results/ mosaic-results/ + + - name: Download baseline artifact + # The baseline artifact is produced by a *different* workflow run + # (publish-results.yml on main), so actions/download-artifact (which + # only sees the current run) can't fetch it. fetch-artifact.sh looks up + # the most recent baseline across the whole repo via the API. + if: >- + github.event_name == 'pull_request' + && needs.plan.outputs.is_release_pr != 'true' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPO: ${{ github.repository }} + run: | + # Exit 2 = baseline not found (non-fatal: first run has no baseline). + bash .github/scripts/fetch-artifact.sh baseline baseline/mosaic-results/ \ + || [ "$?" -eq 2 ] + + - name: Overlay PR results on baseline + if: >- + github.event_name == 'pull_request' + && needs.plan.outputs.is_release_pr != 'true' + run: | + if [ -d baseline/mosaic-results ] && [ "$(ls baseline/mosaic-results/ 2>/dev/null)" ]; then + if [ -d mosaic-results ] && [ "$(ls mosaic-results/ 2>/dev/null)" ]; then + # PR ran benchmarks: overlay its (possibly partial) results on the + # baseline, PR winning per solver. + MERGED=$(mktemp -d) + uv run python .github/scripts/overlay-results.py \ + baseline/mosaic-results mosaic-results "$MERGED" + rm -rf mosaic-results + mv "$MERGED" mosaic-results + else + # benchmark:none (or a PR that ran nothing): no PR results to + # overlay, so the preview just shows the published baseline. + rm -rf mosaic-results + cp -r baseline/mosaic-results mosaic-results + fi + fi + + # The status snapshot/report exist only to summarise *this run's* results + # against the baseline. When no benchmarks ran (e.g. benchmark:none or a + # docs-only PR), the result tree is just a copy of the baseline, so the + # report would be an all-zero diff — pure noise. Skip it; the PR comment + # then carries only the docs-preview link. + - name: Extract baseline snapshot + id: baseline + if: needs.plan.outputs.should_run == 'true' + run: | + BASELINE_PATH="" + if [ -f baseline/mosaic-results/snapshot.json ]; then + cp baseline/mosaic-results/snapshot.json baseline-snapshot.json + BASELINE_PATH="baseline-snapshot.json" + echo "Fetched baseline snapshot" + fi + echo "path=$BASELINE_PATH" >> "$GITHUB_OUTPUT" + + - name: Generate status snapshot (JSON) + if: needs.plan.outputs.should_run == 'true' + run: uv run mosaic status --format json > snapshot.json + + - name: Generate status report (markdown) + if: needs.plan.outputs.should_run == 'true' + run: | + DIFF_FLAG="" + if [[ -n "${{ steps.baseline.outputs.path }}" ]]; then + DIFF_FLAG="--diff-against ${{ steps.baseline.outputs.path }}" + fi + uv run mosaic status --format md $DIFF_FLAG > status-report.md + + # ── Render docs site on GHA ──────────────────────────────────────── + # The docs are rendered here (where the full mosaic/jax stack is already + # installed and memory is ample) rather than on Read the Docs, whose + # constrained builders OOM-kill the heavy `import mosaic`. RTD only + # downloads and serves the pre-rendered _site/ (see .readthedocs.yaml). + - name: Set up Quarto + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: "1.7.31" + + # Regenerate plots for ALL solvers from the merged result.json files. + # The baseline artifact stores data only (plots are stripped before + # upload), and only the changed solver re-ran here, so without this step + # generate_results.py would glob just the changed solver's PNGs and the + # docs would omit every baseline solver. --plots-only is no-Docker/no-GPU. + - name: Regenerate plots from merged results + run: uv run mosaic run --plots-only + + - name: Generate results pages + run: uv run python docs/generate_results.py + + - name: Render docs site + # Always render the full site, so doc-only (benchmark:none) PRs preview + # their prose changes, not just the results pages. Results pages are + # populated from the merged PR+baseline result tree above. + run: uv run quarto render --output-dir _site + + # ── Metadata for the downstream publish workflow ─────────────────── + # The RTD trigger, preview-URL lookup, and PR comment all need repo + # secrets, which GitHub withholds from pull_request runs on fork PRs. + # So this job (which runs in the *fork's* context) does the secret-free + # work — render + upload — and hands off to docs-publish.yml, which runs + # on workflow_run in the *base* repo's context (trusted main code, full + # secret access). The handoff carries everything that workflow needs but + # can't otherwise recover: the PR number and whether benchmarks ran. + - name: Write publish metadata + if: github.event_name == 'pull_request' + run: | + { + echo "pr_number=${{ github.event.pull_request.number }}" + echo "should_run=${{ needs.plan.outputs.should_run }}" + } > publish-meta.txt + # Ship the status report alongside the site so the downstream comment + # can include it. Absent (benchmark:none) → downstream posts only the + # preview banner. + if [[ "${{ needs.plan.outputs.should_run }}" == "true" ]]; then + cp status-report.md _site-status-report.md + fi + + - name: Upload rendered docs site + uses: actions/upload-artifact@v7 + with: + # One artifact per PR (RTD serves it keyed off the PR number). For + # non-PR runs (e.g. workflow_dispatch) fall back to the commit SHA. + name: docs-site-pr-${{ github.event.pull_request.number || github.sha }} + path: _site/ + overwrite: true + retention-days: 90 + + - name: Upload publish handoff + # Consumed by docs-publish.yml (workflow_run). Kept separate from the + # _site artifact so the downstream can grab just the small metadata + + # report without unpacking the whole site. + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: docs-publish-meta-${{ github.event.pull_request.number }} + path: | + publish-meta.txt + _site-status-report.md + if-no-files-found: ignore + overwrite: true + retention-days: 7 + + # ── Merge gate ───────────────────────────────────────────────────────── + bench-ok: + name: Benchmark checks passed + if: always() + needs: [plan, build, benchmarks, report] + runs-on: ubuntu-latest + steps: + - name: Decide status run: | echo "plan: ${{ needs.plan.result }}" + echo "build: ${{ needs.build.result }}" + echo "benchmarks: ${{ needs.benchmarks.result }}" + echo "report: ${{ needs.report.result }}" echo "label: ${{ needs.plan.outputs.label }}" echo "mosaic_touched: ${{ needs.plan.outputs.mosaic_touched }}" - echo "should_run: ${{ needs.plan.outputs.should_run }}" - - # The seed posts are BEST-EFFORT: a fork token that can't write a - # status must not fail this step (that would flip the run conclusion - # to failure and gate out the trusted executor, orphaning the gate). - # The executor's finalize job is the authoritative terminal post for - # every found=true PR — these seeds only surface state sooner. The - # policy `exit 1` paths below stay fatal (they intentionally block). - post() { bash .github/scripts/post-commit-status.sh "$HEAD_SHA" "$1" "$2" "$RUN_URL" || echo "::warning::bench-ok seed ($1) could not be posted; finalize will set it."; } # plan must always succeed if [[ "${{ needs.plan.result }}" != "success" ]]; then - post failure "Plan job failed" echo "::error::Plan failed." exit 1 fi # PR that touches mosaic/ without a benchmark label → block merge - # (release PRs are exempt — they always run full benchmarks). The - # planner run then concludes failure, so the executor never runs and - # this failure status is the final word until a label is applied. - if [[ "${{ needs.plan.outputs.mosaic_touched }}" == "true" \ + # (release PRs are exempt — they always run full benchmarks) + if [[ "${{ github.event_name }}" == "pull_request" \ + && "${{ needs.plan.outputs.mosaic_touched }}" == "true" \ && "${{ needs.plan.outputs.is_release_pr }}" != "true" \ && -z "${{ needs.plan.outputs.label }}" ]]; then - post failure "Add a benchmark:{none,solver,all} label" echo "::error::This PR modifies mosaic/ code but has no benchmark label." echo "::error::A maintainer must add exactly one of: benchmark:none, benchmark:solver, benchmark:all" exit 1 fi - # Nothing will run (benchmark:none, no solver change, docs-only): - # seed success now so the check shows green promptly. finalize - # re-asserts success for this case too, so a failed seed still - # resolves. - if [[ "${{ needs.plan.outputs.should_run }}" != "true" ]]; then - post success "No benchmarks to run for this PR" - echo "No benchmarks will run — bench-ok seeded success." + # benchmark:none → no benchmarks to check, but the report job still + # runs to render the docs preview. A failed render (e.g. a broken + # .qmd) should block merge. RTD publishing / PR-commenting happen in + # docs-publish.yml (workflow_run) and never gate the merge. + if [[ "${{ needs.plan.outputs.label }}" == "none" ]]; then + if [[ "${{ github.event_name }}" == "pull_request" \ + && "${{ needs.report.result }}" != "success" \ + && "${{ needs.report.result }}" != "skipped" ]]; then + echo "::error::Docs preview (report job) failed (${{ needs.report.result }})." + exit 1 + fi + echo "benchmark:none — skipping benchmark checks (docs preview OK)" exit 0 fi - # Benchmarks will run in the trusted benchmark-execute workflow. - # Seed pending; finalize flips bench-ok to success/failure. - post pending "Benchmarks running in benchmark-execute" - echo "Seeded bench-ok=pending; trusted run will report results." - - # ── Handoff to the trusted benchmark-execute workflow ────────────────── - # The privileged work (build + push to private GHCR, run benchmarks, render - # docs, comment) can't happen here on fork PRs. Ship everything the trusted - # workflow_run needs — but can't recover from its own context — as a small - # artifact. Every value is a plan output (numeric / enum / SHA / JSON) and is - # re-validated on the far side before use. - handoff: - name: Handoff to benchmark-execute - needs: [plan] - if: always() && needs.plan.result == 'success' && github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Write handoff metadata - # All values go through env rather than inline ${{ }} interpolation: - # PR-controlled strings (labels, and especially any free-form field) - # must never be spliced into a shell script directly. The executor - # re-validates each one before use. - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - SHOULD_RUN: ${{ needs.plan.outputs.should_run }} - LABEL: ${{ needs.plan.outputs.label }} - IS_RELEASE_PR: ${{ needs.plan.outputs.is_release_pr }} - PROBLEMS: ${{ needs.plan.outputs.problems }} - SUITES: ${{ needs.plan.outputs.suites }} - SOLVERS: ${{ needs.plan.outputs.solvers }} - SOLVER_MATRIX: ${{ needs.plan.outputs.solver_matrix }} - RUN_MATRIX: ${{ needs.plan.outputs.matrix }} - run: | - { - echo "pr_number=${PR_NUMBER}" - echo "head_sha=${HEAD_SHA}" - echo "base_sha=${BASE_SHA}" - echo "should_run=${SHOULD_RUN}" - echo "label=${LABEL}" - echo "is_release_pr=${IS_RELEASE_PR}" - echo "problems=${PROBLEMS}" - echo "suites=${SUITES}" - echo "solvers=${SOLVERS}" - } > handoff-meta.txt - # Matrices are JSON — keep them in their own files to avoid quoting - # hazards in the key=value metadata. - printf '%s\n' "$SOLVER_MATRIX" > solver-matrix.json - printf '%s\n' "$RUN_MATRIX" > run-matrix.json - echo "--- handoff-meta.txt ---" - cat handoff-meta.txt - - - name: Upload handoff - uses: actions/upload-artifact@v7 - with: - name: bench-handoff-${{ github.event.pull_request.number }} - path: | - handoff-meta.txt - solver-matrix.json - run-matrix.json - if-no-files-found: error - overwrite: true - retention-days: 7 + # When benchmarks ran, all downstream jobs must succeed (or be skipped) + if [[ "${{ needs.plan.outputs.should_run }}" == "true" ]]; then + for result in "${{ needs.build.result }}" "${{ needs.benchmarks.result }}" "${{ needs.report.result }}"; do + if [[ "$result" != "success" && "$result" != "skipped" ]]; then + echo "::error::Benchmark job failed ($result)." + exit 1 + fi + done + fi diff --git a/.github/workflows/build-tesseracts.yml b/.github/workflows/build-tesseracts.yml index a88256c2..e955536c 100644 --- a/.github/workflows/build-tesseracts.yml +++ b/.github/workflows/build-tesseracts.yml @@ -7,20 +7,6 @@ on: description: 'JSON array of {"domain", "solver"} objects to build' type: string required: true - ref: - description: >- - Commit SHA to check out and build. The trusted benchmark-execute - workflow passes a PR's validated HEAD so it builds the PR's solver - code from the base-repo context. Empty → the workflow's own ref. - type: string - default: "" - image_sha: - description: >- - SHA to tag the pushed image with. benchmark-execute passes the same - validated PR HEAD so the tag matches what the benchmark run pulls. - Empty → the event's PR-head/commit SHA. - type: string - default: "" push: branches: [main] workflow_dispatch: @@ -89,11 +75,6 @@ jobs: include: ${{ fromJSON(inputs.matrix || needs.discover.outputs.matrix) }} steps: - uses: actions/checkout@v6 - with: - # When invoked by benchmark-execute (workflow_run), check out the PR's - # HEAD commit explicitly rather than the default-branch ref this - # trusted run starts on, so we build the PR's solver code. - ref: ${{ inputs.ref || github.sha }} - name: Free disk space run: | @@ -110,10 +91,14 @@ jobs: run: pip install tesseract-core - name: Log in to GHCR - # Runs either on push-to-main or from the trusted benchmark-execute - # (workflow_run) context — both hold a token with package:write, so the - # push below always succeeds. (Fork `pull_request` runs no longer reach - # this workflow; they hand off to benchmark-execute instead.) + # Only the publish (push-to-main / dispatch) path pushes to GHCR, so + # only it needs auth. That path is distinguished by `inputs.matrix` + # being empty — it's set only when benchmark.yml calls this via + # workflow_call for a PR. (`github.event_name` can't tell them apart: + # in a called workflow it reflects the caller's event, not + # 'workflow_call'.) The PR path exports an artifact instead of pushing, + # so it needs no login and works from fork PRs. + if: inputs.matrix == '' uses: docker/login-action@v3 with: registry: ghcr.io @@ -140,6 +125,9 @@ jobs: echo "ghcr_base=${GHCR_BASE}" >> "$GITHUB_OUTPUT" - name: Build Tesseract image + # cache-from is best-effort: BuildKit warns and does a full build if the + # :latest cache ref is missing or unreadable, so this never hard-fails — + # including on fork PRs (which read the public GHCR cache without login). env: TESSERACT_DOCKER_BUILD_ARGS: >- --cache-from type=registry,ref=${{ steps.image.outputs.ghcr_base }}:latest @@ -148,33 +136,59 @@ jobs: SOLVER_DIR="mosaic/tesseracts/${{ matrix.domain }}/${{ matrix.solver }}" tesseract --loglevel debug build --tag latest "${SOLVER_DIR}" + # ── main-push path: publish to GHCR ────────────────────────────── + # Push-to-main (and manual dispatch) publish the baseline images that PR + # builds later read via `cache-from` and that unchanged solvers are pulled + # from at their base SHA. PR builds go through workflow_call and take the + # artifact path below instead — they never push. - name: Push to GHCR - # Always push : so the benchmark run can pull the exact image - # built for this commit. Only push :latest on push-to-main so PR - # builds don't bump main's latest pointer. No auth-failure tolerance: - # both callers (push-to-main, trusted benchmark-execute) hold - # package:write, so a push failure is a real error worth failing on. + # Publish path only (empty inputs.matrix ⇒ push-to-main / dispatch). + if: inputs.matrix == '' env: - # benchmark-execute passes the validated PR HEAD via image_sha so the - # tag matches what the benchmark run pulls; push-to-main falls back to - # the commit SHA. - IMAGE_SHA: ${{ inputs.image_sha || github.event.pull_request.head.sha || github.sha }} + IMAGE_SHA: ${{ github.sha }} run: | IMAGE_NAME="${{ steps.image.outputs.image_name }}" GHCR_BASE="${{ steps.image.outputs.ghcr_base }}" LOCAL_IMAGE="${IMAGE_NAME}:latest" - # Always tag and push : + # Always tag and push : (immutable, what PRs pull unchanged + # solvers by). docker tag "${LOCAL_IMAGE}" "${GHCR_BASE}:${IMAGE_SHA}" docker push "${GHCR_BASE}:${IMAGE_SHA}" echo "Pushed ${GHCR_BASE}:${IMAGE_SHA}" - # :latest tracks main only. - IS_MAIN_PUSH='${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}' - if [[ "${IS_MAIN_PUSH}" == "true" ]]; then + # :latest tracks main only (the cache-from ref). + if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then docker tag "${LOCAL_IMAGE}" "${GHCR_BASE}:latest" docker push "${GHCR_BASE}:latest" echo "Pushed ${GHCR_BASE}:latest" else echo "Skipping :latest tag — only main pushes update :latest" fi + + # ── PR path: export the built image as an artifact ─────────────── + # A fork PR's default token can neither push to nor (for a private + # package) pull from GHCR. So instead of publishing, save the built image + # to a tarball and hand it to the benchmark run via an artifact. The run + # `docker load`s it back to `:latest` — exactly the local tag + # `mosaic run --no-build` resolves — so it's interchangeable with a pull. + # No registry write, works identically for fork and same-repo PRs. + - name: Save built image as artifact + # PR path only (inputs.matrix set ⇒ called from benchmark.yml). + if: inputs.matrix != '' + run: | + IMAGE_NAME="${{ steps.image.outputs.image_name }}" + mkdir -p image-out + docker save "${IMAGE_NAME}:latest" -o "image-out/${IMAGE_NAME}.tar" + echo "Saved ${IMAGE_NAME}:latest ($(du -h image-out/${IMAGE_NAME}.tar | cut -f1))" + + - name: Upload solver image artifact + if: inputs.matrix != '' + uses: actions/upload-artifact@v7 + with: + name: solver-image-${{ steps.image.outputs.image_name }} + path: image-out/*.tar + if-no-files-found: error + # Only consumed by the benchmarks job in the same run; a few days of + # slack covers a manual re-run of just that job without re-building. + retention-days: 3 diff --git a/.github/workflows/docs-publish.yml b/.github/workflows/docs-publish.yml new file mode 100644 index 00000000..1e80dd04 --- /dev/null +++ b/.github/workflows/docs-publish.yml @@ -0,0 +1,116 @@ +name: Publish docs preview + +# Second half of the fork-safe docs-preview flow. The Benchmark workflow's +# `report` job runs in the PR head's context (fork PRs included) and does the +# secret-free work: render the site and upload it as an artifact. It cannot +# trigger the RTD rebuild, resolve the preview URL, or post the PR comment, +# because GitHub withholds repo secrets from pull_request runs on fork PRs. +# +# This workflow fills that gap. `workflow_run` always runs in the BASE repo's +# context using the workflow definition from the default branch — trusted code +# with full secret access — regardless of whether the triggering run came from +# a fork. It downloads the artifacts the report job produced and performs the +# privileged steps. Because it never checks out or executes PR-authored code, +# exposing secrets here is safe. + +on: + workflow_run: + workflows: ["Benchmark"] + types: [completed] + +permissions: + contents: read + actions: read # download artifacts from the triggering run + pull-requests: write # post the status comment + +jobs: + publish: + name: Publish RTD preview & comment + runs-on: ubuntu-latest + # Only for PR-triggered Benchmark runs that succeeded. A failed report job + # means there's nothing worth publishing (and the merge gate already + # surfaces that failure on the PR). + if: >- + github.event.workflow_run.event == 'pull_request' + && github.event.workflow_run.conclusion == 'success' + steps: + - uses: actions/checkout@v6 + + # ── Recover the handoff from the triggering run ──────────────────── + # download-artifact with an explicit run-id reaches into the upstream + # run (a different workflow), which the default same-run lookup can't do. + - name: Download publish handoff + id: dl_meta + uses: actions/download-artifact@v8 + continue-on-error: true + with: + pattern: docs-publish-meta-* + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: handoff/ + merge-multiple: true + + # No handoff artifact → the report job didn't reach the upload (e.g. a + # non-PR dispatch, or a render failure). Nothing to publish; exit clean. + - name: Parse handoff + id: meta + run: | + META="handoff/publish-meta.txt" + if [[ ! -f "$META" ]]; then + echo "No handoff metadata — nothing to publish." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + PR_NUMBER=$(grep -oP '^pr_number=\K.*' "$META" || true) + SHOULD_RUN=$(grep -oP '^should_run=\K.*' "$META" || true) + # Guard: the PR number originates from fork-controlled workflow + # output, so validate it's purely numeric before using it in API + # calls / URLs. Anything else → bail rather than act on it. + if [[ ! "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + echo "::warning::Handoff PR number '${PR_NUMBER}' is not numeric — skipping." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "found=true" >> "$GITHUB_OUTPUT" + echo "pr_number=${PR_NUMBER}" >> "$GITHUB_OUTPUT" + echo "should_run=${SHOULD_RUN}" >> "$GITHUB_OUTPUT" + echo "Publishing for PR #${PR_NUMBER} (should_run=${SHOULD_RUN})" + + # ── Trigger RTD PR preview rebuild ───────────────────────────────── + # RTD's webhook already built a preview on PR push, but that build ran + # before this run rendered the results pages. Trigger a fresh build so + # the preview reflects the merged PR+baseline result tree. + - name: Trigger RTD PR preview rebuild + if: steps.meta.outputs.found == 'true' + continue-on-error: true + env: + RTD_TOKEN: ${{ secrets.RTD_TOKEN }} + RTD_PROJECT: ${{ vars.RTD_PROJECT }} + run: bash .github/scripts/rtd-trigger.sh "${{ steps.meta.outputs.pr_number }}" + + # ── Resolve RTD preview URL ──────────────────────────────────────── + - name: Resolve RTD preview URL + id: rtd_url + if: steps.meta.outputs.found == 'true' + continue-on-error: true + env: + RTD_TOKEN: ${{ secrets.RTD_TOKEN }} + RTD_PROJECT: ${{ vars.RTD_PROJECT }} + run: bash .github/scripts/rtd-preview-url.sh "${{ steps.meta.outputs.pr_number }}" + + # ── Post PR comment ──────────────────────────────────────────────── + # Include the status report if the report job shipped one (benchmarks + # ran); otherwise the comment is just the preview banner. + - name: Post status comment on PR + if: steps.meta.outputs.found == 'true' + env: + GH_TOKEN: ${{ secrets.PL_PASTEURBOT_PAT_PUBLIC }} + DOCS_PREVIEW_URL: ${{ steps.rtd_url.outputs.preview_url }} + run: | + REPORT="" + if [[ -f handoff/_site-status-report.md ]]; then + REPORT="handoff/_site-status-report.md" + fi + .github/scripts/post-status-comment.sh \ + "${{ steps.meta.outputs.pr_number }}" \ + "$REPORT"