From 47db5d02cedaddd0ca1e4503cfe603a5c1cbfe98 Mon Sep 17 00:00:00 2001
From: Andrey Esipov <61357035+andrey-esipov@users.noreply.github.com>
Date: Sat, 25 Jul 2026 07:26:45 -0500
Subject: [PATCH 1/3] ci: skip eval jobs deterministically on fork PRs

Fork PRs cannot receive repository secrets or a writable GITHUB_TOKEN, so the
eval shards, the report comment, and the ghcr.io image push can never succeed
from a fork. Today the outcome depends on Docker layer cache state rather than
on the code: a warm cache lets build-image pass, the eval shards then run and
fail at SDK auth; a cold cache makes build-image fail its push and the evals
skip entirely. Identical code produces opposite CI verdicts.

Gate build-image, evals, and report on the PR head being in the base
repository. Same-repo PRs, pushes, and workflow_dispatch are unchanged and
keep full coverage.
---
 .github/workflows/evals.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index f5a0d9e40a..33049a8b7a 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -14,8 +14,16 @@ env:
 
 jobs:
   # Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change)
+  # Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al) and get a
+  # read-only GITHUB_TOKEN, so nothing below can succeed from a fork: the eval
+  # shards fail at SDK auth, `report` cannot post its comment, and this job cannot
+  # push to ghcr.io. Skip deterministically instead of leaving the outcome to
+  # Docker-cache luck (a warm cache made these run and fail; a cold one made them
+  # fail to push and skip). Same-repo PRs, pushes, and workflow_dispatch are
+  # unaffected. Maintainers get real coverage via a trusted base-repo branch.
   build-image:
     runs-on: ubicloud-standard-8
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     permissions:
       contents: read
       packages: write
@@ -58,6 +66,7 @@ jobs:
   evals:
     runs-on: ${{ matrix.suite.runner || 'ubicloud-standard-8' }}
     needs: build-image
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     container:
       image: ${{ needs.build-image.outputs.image-tag }}
       credentials:
@@ -263,7 +272,7 @@ jobs:
   report:
     runs-on: ubicloud-standard-8
     needs: evals
-    if: always() && github.event_name == 'pull_request'
+    if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
     timeout-minutes: 5
     permissions:
       contents: read

From 0f9904c2f62c349eded5b711223a6ff1a1ef1a21 Mon Sep 17 00:00:00 2001
From: Andrey Esipov <61357035+andrey-esipov@users.noreply.github.com>
Date: Sat, 25 Jul 2026 07:34:39 -0500
Subject: [PATCH 2/3] ci: keep build-image running on forks to avoid startup
 failure

The first cut gated build-image, evals, and report on the PR head being in the
base repo. That skipped every job in the workflow, and a run in which no job
starts is recorded as a startup failure (conclusion: failure, zero jobs) rather
than a skip -- trading one bogus red for another.

Let build-image always run instead, and make only its push conditional. A fork
still builds the CI image, so Dockerfile.ci changes stay validated, it just does
not publish with a packages:read token. That guarantees at least one job starts
while evals and report skip deterministically.
---
 .github/workflows/evals.yml | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index 33049a8b7a..3f38a72280 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -14,16 +14,8 @@ env:
 
 jobs:
   # Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change)
-  # Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al) and get a
-  # read-only GITHUB_TOKEN, so nothing below can succeed from a fork: the eval
-  # shards fail at SDK auth, `report` cannot post its comment, and this job cannot
-  # push to ghcr.io. Skip deterministically instead of leaving the outcome to
-  # Docker-cache luck (a warm cache made these run and fail; a cold one made them
-  # fail to push and skip). Same-repo PRs, pushes, and workflow_dispatch are
-  # unaffected. Maintainers get real coverage via a trusted base-repo branch.
   build-image:
     runs-on: ubicloud-standard-8
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     permissions:
       contents: read
       packages: write
@@ -53,16 +45,26 @@ jobs:
       - if: steps.check.outputs.exists == 'false'
         run: cp package.json bun.lock .github/docker/
 
+      # A fork PR's GITHUB_TOKEN only has `packages: read`, so pushing fails.
+      # Still BUILD (validates Dockerfile.ci changes), just don't publish. This
+      # job intentionally has no `if:` — it must always start, because a run in
+      # which every job is skipped registers as a startup failure.
       - if: steps.check.outputs.exists == 'false'
         uses: docker/build-push-action@v6
         with:
           context: .github/docker
           file: .github/docker/Dockerfile.ci
-          push: true
+          push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
           tags: |
             ${{ steps.meta.outputs.tag }}
             ${{ env.IMAGE }}:latest
 
+  # Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so every
+  # API-calling eval fails at SDK auth before a model runs. Skip deterministically
+  # rather than leaving the outcome to Docker-cache luck: a warm cache let these
+  # run and fail, a cold one made build-image fail its push and the shards skip.
+  # Same-repo PRs, pushes, and workflow_dispatch keep full coverage. Fork work
+  # gets real coverage via a trusted base-repo branch.
   evals:
     runs-on: ${{ matrix.suite.runner || 'ubicloud-standard-8' }}
     needs: build-image

From a60522f649f4452a5cb594b5531807e0a6aee01d Mon Sep 17 00:00:00 2001
From: Andrey Esipov <61357035+andrey-esipov@users.noreply.github.com>
Date: Sat, 25 Jul 2026 07:42:00 -0500
Subject: [PATCH 3/3] ci: correct rationale comment on build-image fork
 behavior

The previous comment claimed an all-skipped run registers as a startup
failure. That is not the mechanism -- an all-skipped run is simply grey.
The real reason build-image stays ungated is that fork PRs should still
validate Dockerfile.ci and get one honest green check.
---
 .github/workflows/evals.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index 3f38a72280..c72db3d850 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -47,8 +47,8 @@ jobs:
 
       # A fork PR's GITHUB_TOKEN only has `packages: read`, so pushing fails.
       # Still BUILD (validates Dockerfile.ci changes), just don't publish. This
-      # job intentionally has no `if:` — it must always start, because a run in
-      # which every job is skipped registers as a startup failure.
+      # job intentionally keeps no `if:` so fork PRs still get one real, honest
+      # green check here instead of a run where every job is grey.
       - if: steps.check.outputs.exists == 'false'
         uses: docker/build-push-action@v6
         with: