feat(dogfood): Athens in-cluster Go module cache (Q244)

deploy/athens/deployment.yaml

@@ -0,0 +1,89 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: athens
+  namespace: gag-dogfood
+  labels:
+    app: athens
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: athens
+  template:
+    metadata:
+      labels:
+        app: athens
+    spec:
+      # No toleration for dedicated=workers, so this lands on the system node pool
+      # and stays up when spot workers are scaled to zero.
+      containers:
+        - name: athens
+          # v0.18.0 bundles Go 1.26.2; older tags shipped EOL Go (v0.15.1 had
+          # 1.20.14) that Athens uses to rebuild module zips on direct fetches.
+          image: gomods/athens:v0.18.0
+          ports:
+            - containerPort: 3000
+          env:
+            - name: ATHENS_STORAGE_TYPE
+              value: disk
+            - name: ATHENS_DISK_STORAGE_ROOT
+              value: /var/lib/athens
+            - name: ATHENS_DOWNLOAD_MODE
+              value: sync
+            - name: ATHENS_LOG_LEVEL
+              value: info
+            # Fetch upstream modules through proxy.golang.org, not direct-from-VCS.
+            # This overrides Athens' built-in default (GoBinaryEnvVars =
+            # ["GOPROXY=direct"]), which forces its internal `go` subprocess to clone
+            # each module from its origin and rebuild the zip locally. Athens' bundled
+            # (older) Go computes a different module hash than the notarized one for
+            # modules declaring a newer go directive (e.g. ginkgo/v2@v2.32.0, go 1.25),
+            # so its own checksum check rejects the rebuild as a mismatch. The public
+            # proxy instead serves the immutable, notarized zip whose hash matches
+            # sum.golang.org and our committed go.sum — the check passes. The proxy is
+            # also faster (pre-built zips over a CDN) and more available (one upstream
+            # vs. every module's origin host). A plain container-level GOPROXY env var
+            # does NOT work — Athens overrides it for the subprocess via this config.
+            # The ",direct" fallback still resolves any module proxy.golang.org does
+            # not mirror; Athens splits this var on ";", so the comma stays inside the
+            # GOPROXY value (matching Go's own default GOPROXY).
+            - name: ATHENS_GO_BINARY_ENV_VARS
+              value: GOPROXY=https://proxy.golang.org,direct
+            # Athens verifies fetched modules against sum.golang.org — it has free
+            # egress (no workload NetworkPolicy on this pod). Empty GONOSUMCHECK means
+            # "verify every module"; integrity is enforced end-to-end. Workers, which
+            # have no egress to sum.golang.org, instead set GONOSUMDB=* and rely on the
+            # committed go.sum.
+            - name: ATHENS_GONOSUMCHECK
+              value: ""
+          volumeMounts:
+            - name: storage
+              mountPath: /var/lib/athens
+          resources:
+            requests:
+              # Kept small so Athens reliably co-schedules with the AGC on the single
+              # system node, whose CPU-request budget is tight during CI worker bursts.
+              # Athens is I/O-bound (serving cached zips from the PVC); the generous
+              # limits still allow bursting for cold-cache `go mod download` fetches.
+              cpu: "100m"
+              memory: "192Mi"
+            limits:
+              cpu: "1"
+              memory: "1Gi"
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 3000
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 3000
+            initialDelaySeconds: 15
+            periodSeconds: 20
+      volumes:
+        - name: storage
+          persistentVolumeClaim:
+            claimName: athens-storage

deploy/athens/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml
+  - networkpolicy.yaml

deploy/athens/networkpolicy.yaml

@@ -0,0 +1,37 @@
+# Allow worker pods to reach Athens on port 3000.
+#
+# The GMC-managed workload NetworkPolicy restricts egress for pods labelled
+# actions-gateway/component=workload to DNS + the egress proxy (or GitHub CIDRs
+# in direct mode). That policy does not allow port 3000. This additive policy
+# opens the one extra rule needed: workload pods → Athens pods on 3000.
+#
+# Athens pods (app=athens) are not labelled actions-gateway/component=workload,
+# so they are not covered by the workload NetworkPolicy and retain free egress
+# to fetch modules from proxy.golang.org and validate against sum.golang.org.
+#
+# Plain HTTP (no TLS) is intentional here. Athens serves public Go module zips;
+# confidentiality is not a concern. Integrity is upheld by the Go toolchain's
+# go.sum verification — every downloaded module is checked against the committed
+# go.sum regardless of GONOSUMDB — so a tampered response would be caught before
+# it ever reaches the build. Adding TLS would require cert management (cert-manager
+# or a self-signed CA wired into every worker) for no meaningful security gain in
+# this single-tenant dogfood cluster. Revisit if Athens is extended to a shared
+# multi-tenant cluster or used to serve private modules.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: athens-worker-access
+  namespace: gag-dogfood
+spec:
+  podSelector:
+    matchLabels:
+      actions-gateway/component: workload
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app: athens
+      ports:
+        - protocol: TCP
+          port: 3000

deploy/athens/pvc.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: athens-storage
+  namespace: gag-dogfood
+spec:
+  accessModes: [ReadWriteOnce]
+  resources:
+    requests:
+      storage: 20Gi
+  # GKE default: standard-rwo (Balanced PD, ReadWriteOnce, faster than standard).
+  storageClassName: standard-rwo

deploy/athens/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  # Named go-module-proxy, not athens: a service named "athens" causes Kubernetes
+  # to inject ATHENS_PORT=tcp://clusterip:3000 into pods in the namespace, which
+  # Athens reads as its own listen address and fails with "too many colons".
+  name: go-module-proxy
+  namespace: gag-dogfood
+spec:
+  selector:
+    app: athens
+  ports:
+    - name: http
+      port: 3000
+      targetPort: 3000
+  type: ClusterIP

docs/STATUS.md

@@ -52,10 +52,9 @@ Specific actionable items in priority order. Pick from the top; skip 🚫 items
 
 | ID | Item | Labels | St | Sz | Notes |
 |---|---|---|---|---|---|
-| <a id="Q224"></a>Q224 | [GKE dogfood: route production CI (green CI blocked)](plan/gke-dogfood.md) | `milestone` `infra` | 🚫 | M | rc.4 turn-on validated; toolchain unblocked (Q239). Egress half blocked: [Q242](#Q242) shipped but CiliumFQDN unusable on GKE DPv2 (no CiliumNetworkPolicy CRD); unblock via [Q244](#Q244) cache or [Q245](#Q245) GKEFQDN. On-demand. |
-| <a id="Q242"></a>Q242 | [Implement G.1 proxy destination allowlist](plan/q242-g1-proxy-destination-allowlist.md) | `security` `infra` | ▶ | L | Impl merged #460–#464. Remaining: dogfood ([Q224](#Q224)) blocked — GKE DPv2 managed Cilium lacks CiliumNetworkPolicy CRD (CiliumFQDN→Degraded); unblock via [Q244](#Q244) cache or [Q245](#Q245) GKEFQDN. v2beta1 blocker. |
+| <a id="Q224"></a>Q224 | [GKE dogfood: route production CI (green CI blocked)](plan/gke-dogfood.md) | `milestone` `infra` | 🔲 | M | rc.4 turn-on validated; toolchain unblocked (Q239). Egress blocker for vendor-check/tidy-check resolved: Athens in-cluster cache deployed (Q244). Run `scripts/dogfood-start.sh` + validate CI green end-to-end. On-demand. |
+| <a id="Q242"></a>Q242 | [Implement G.1 proxy destination allowlist](plan/q242-g1-proxy-destination-allowlist.md) | `security` `infra` | ▶ | L | Impl merged #460–#464. Dogfood vendor-check/tidy-check unblocked via Athens (Q244). Remaining: flip `GAG_RUNNER`, validate green CI ([Q224](#Q224)); FQDN intent/backend split ([Q245](#Q245)). v2beta1 blocker. |
 | <a id="Q243"></a>Q243 | [Per-tenant egress-IP reference architecture (cloud)](plan/gke-dogfood.md) | `security` `infra` `docs` | 🔲 | L | Substantiate the per-tenant egress-IP isolation claim: spike + validate Cilium Egress Gateway vs per-tenant NAT on a cloud; doc single-tenant-direct vs production topology + cost. Dogfood stays direct (single-tenant). v2beta1 blocker. |
-| <a id="Q244"></a>Q244 | [In-cluster Go module cache (Athens) for the dogfood](plan/q242-g1-proxy-destination-allowlist.md) | `infra` | 🔲 | M | Stand up an in-cluster Athens Go-module cache so dogfood vendor-check/tidy-check fetch modules without per-worker external egress — closes [Q224](#Q224) on GKE without an FQDN backend (mirror path; workers reach it via destinationCIDRs). |
 | <a id="Q245"></a>Q245 | [FQDN egress: split intent from CNI backend + GKE backend](plan/q242-g1-proxy-destination-allowlist.md#provider-fqdn-egress-fragmentation-post-implementation-finding) | `security` `infra` | 🔲 | L | egressPolicyMode FQDN variants encode a per-CNI kind, fragmented across GKE/AKS/EKS/OVN. Decouple tenant intent (CIDR\|FQDN) from a platform --fqdn-policy-backend; add gke backend (networking.gke.io FQDNNetworkPolicy). Fold into v2beta1 (Q74). |
 | <a id="Q225"></a>Q225 | [Operator docs: Kata Containers for DinD workloads](operations/in-runner-image-builds.md) | `docs` | 🔲 | S | Document runtimeClassName: kata-qemu on podTemplate, /dev/kvm machine-type requirement (N2+ on GCP), and Kata DaemonSet setup. Extend in-runner-image-builds.md or new page depending on density. |
 | <a id="Q226"></a>Q226 | [Kata Containers on GKE — secure CI reference architecture](plan/kata-on-gke.md) | `security` `infra` | 🔲 | M | OSS untrusted-PR threat + GAG dogfood requirement rule out privileged DinD. Spike: GKE nested-virt node pool + Kata RuntimeClass: kind in micro-VM, no privileged pod. Reference arch. [plan](plan/kata-on-gke.md) |

docs/plan/gke-dogfood.md

@@ -23,7 +23,7 @@ profile or paste them at the start of each terminal session.
 
 ```bash
 CLUSTER=gag-dogfood
-ZONE=us-central1-a
+ZONE=us-central1-b
 PROJECT=actions-gateway-dogfood   # must be globally unique; append 4 digits if needed
 REPO=actions-gateway/github-actions-gateway
 APP_ID=3752347
@@ -271,6 +271,46 @@ spec:
 EOF
 ```
 
+### B6b. Deploy Athens in-cluster Go module cache
+
+`vendor-check` and `tidy-check` re-fetch Go modules from `proxy.golang.org` on
+a cold cache. GKE Dataplane V2's managed Cilium lacks the `CiliumNetworkPolicy`
+CRD, so the `CiliumFQDN` egress mode is unusable here; a CIDR allowlist for
+Google-fronted hosts like `proxy.golang.org` would be a footgun (it opens all
+of Google's frontend). Athens sidesteps both constraints: it runs in-cluster
+with free egress and serves cached modules to workers over a plain HTTP port
+that does not need a CNI FQDN backend.
+
+Athens is not covered by the workload `NetworkPolicy` (`actions-gateway/component: workload`
+label). Workers reach it via an additive `NetworkPolicy` that opens port 3000
+from workload pods to Athens pods. The Service is named `go-module-proxy`
+(not `athens`) to avoid Kubernetes injecting `ATHENS_PORT=tcp://...` into
+pods in the namespace — Athens misreads that as its listen address.
+
+```bash
+kubectl apply -k deploy/athens
+kubectl rollout status deployment/athens -n gag-dogfood --timeout=120s
+```
+
+Verify Athens is healthy:
+
+```bash
+kubectl get pods -n gag-dogfood -l app=athens
+kubectl logs -n gag-dogfood -l app=athens --tail=20
+```
+
+Athens pre-warms lazily — the first `vendor-check`/`tidy-check` run is slower
+while modules download; subsequent runs are cache hits from the PVC.
+
+> **Why plain HTTP (no TLS)?** Athens serves public Go module zips; there is
+> nothing confidential in transit. Integrity is upheld by the Go toolchain's
+> `go.sum` verification — every module downloaded from Athens is checked against
+> the committed `go.sum` regardless of `GONOSUMDB`, so a tampered response is
+> caught before it reaches the build. Adding TLS would require cert management
+> (cert-manager or a self-signed CA wired into every worker image) for no
+> meaningful security gain in this single-tenant cluster. Revisit if Athens is
+> extended to a shared multi-tenant cluster or used to serve private modules.
+
 ### B7. Create the v2 tenant objects
 
 The v2 API decomposes the v1 monolithic `ActionsGateway` into `ActionsGateway`
@@ -318,6 +358,15 @@ spec:
           # repo's make-based CI fails `make: command not found` on it (see the
           # Known gap below). For green CI, set a build-capable workerImage here;
           # injection still applies on top of any base.
+          env:
+            # Athens in-cluster Go module proxy (Q244). Workers cannot reach
+            # proxy.golang.org directly (egress NetworkPolicy, GKE DPv2 no FQDN NP).
+            # GONOSUMDB=* prevents direct sum.golang.org queries; Athens validates
+            # checksums when it fetches from proxy.golang.org upstream.
+            - name: GOPROXY
+              value: "http://go-module-proxy.gag-dogfood.svc.cluster.local:3000,off"
+            - name: GONOSUMDB
+              value: "*"
           resources:
             requests:
               cpu: "2"
@@ -390,25 +439,22 @@ gh api /repos/"$REPO"/actions/runners \
 > which failed `make: command not found` on the bare image, ran green on
 > `dogfood-runner:2.335.1` with the wrapper injected (`make` 4.3, `gcc` 13.3.0).
 >
-> **Residual blocker for `vendor-check` / `tidy-check`.** Those two jobs re-fetch Go
-> modules from `proxy.golang.org` on a cold cache, which the GitHub-only worker
-> egress allowlist blocks — independent of the toolchain. The offline-capable jobs
-> (`lint`, `shellcheck`, `unit-test`, `coverage`) build from `vendor/` and pull
-> Go/shellcheck from GitHub releases, so the image unblocks those.
+> **`vendor-check` / `tidy-check` unblocked by Athens (Q244, implemented).** An
+> Athens in-cluster Go module proxy (`deploy/athens/`, applied by `dogfood-setup.sh`)
+> caches Go modules so workers never need to reach `proxy.golang.org` directly.
+> Athens pods (app=athens) are not covered by the workload NetworkPolicy and have
+> free egress; workers reach Athens via an additive NetworkPolicy (port 3000) and
+> are wired with `GOPROXY=http://go-module-proxy.gag-dogfood.svc.cluster.local:3000,off`
+> plus `GONOSUMDB=*` in the RunnerTemplate.
 >
-> **The proxy destination allowlist (Q242 G.1) shipped, but its `CiliumFQDN` mode
-> does NOT work on this cluster.** GKE Dataplane V2's *managed* Cilium does not
+> **Background (for reference):** GKE Dataplane V2's *managed* Cilium does not
 > expose the `cilium.io/v2 CiliumNetworkPolicy` CRD (dropped since GKE
 > 1.21.5-gke.1300), so an `EgressProxy` with `egressPolicyMode: CiliumFQDN` goes
-> `Degraded` (`no matches for kind "CiliumNetworkPolicy"`, verified 2026-06-29 — the
-> fail-closed posture worked, nothing opened). `destinationCIDRs` is no substitute
-> for `proxy.golang.org`/`sum.golang.org` (Google-fronted ⇒ a CIDR allowlist opens
-> all of Google's frontend). Two ways to close this, both on the Queue: (a) an
-> **in-cluster Go module cache** (Athens — the design-recommended path, works on GKE
-> today); (b) a **GKEFQDN backend** emitting `networking.gke.io FQDNNetworkPolicy`
-> (`--enable-fqdn-network-policy`). Detail + the provider matrix:
+> `Degraded` (`no matches for kind "CiliumNetworkPolicy"`, verified 2026-06-29).
+> `destinationCIDRs` is no substitute for `proxy.golang.org`/`sum.golang.org`
+> (Google-fronted ⇒ a CIDR allowlist opens all of Google's frontend). The FQDN
+> intent/mechanism split (Q245) remains open. Detail + provider matrix:
 > [Q242 plan § Provider FQDN-egress fragmentation](q242-g1-proxy-destination-allowlist.md#provider-fqdn-egress-fragmentation-post-implementation-finding).
-> Until one lands, keep `vendor-check`/`tidy-check` on `ubuntu-latest`.
 
 ---
 

scripts/dogfood-e2e-setup.sh

@@ -10,7 +10,7 @@
 # Required env vars (export before running):
 #   PROJECT          GCP project ID (e.g. actions-gateway-dogfood)
 #   CLUSTER          GKE cluster name (e.g. gag-dogfood)
-#   ZONE             GCP zone (e.g. us-central1-a)
+#   ZONE             GCP zone (e.g. us-central1-b)
 #   REPO             GitHub repo slug (e.g. actions-gateway/github-actions-gateway)
 #   APP_ID           GitHub App numeric ID (3752347)
 #   INSTALLATION_ID  GitHub App installation ID for this repo

scripts/dogfood-setup.sh

@@ -19,7 +19,7 @@
 # Required env vars (export before running):
 #   PROJECT          GCP project ID (e.g. actions-gateway-dogfood)
 #   CLUSTER          GKE cluster name (e.g. gag-dogfood)
-#   ZONE             GCP zone (e.g. us-central1-a)
+#   ZONE             GCP zone (e.g. us-central1-b)
 #   REPO             GitHub repo slug (e.g. actions-gateway/github-actions-gateway)
 #   APP_ID           GitHub App numeric ID (3752347)
 #   INSTALLATION_ID  GitHub App installation ID for this repo/org
@@ -337,6 +337,23 @@ EOF
 }
 
 # ---------------------------------------------------------------------------
+# Part B6b — Athens in-cluster Go module proxy (Q244). Athens caches Go
+# modules so vendor-check/tidy-check can run on GAG runners without external
+# egress to proxy.golang.org. The Athens pod (app=athens) is not labelled
+# actions-gateway/component=workload, so it is not covered by the workload
+# NetworkPolicy and retains free egress to fetch modules. Worker pods reach
+# Athens via an additive NetworkPolicy in deploy/athens/networkpolicy.yaml
+# that opens port 3000 from workload pods to Athens pods. Workers are wired
+# via GOPROXY/GONOSUMDB env vars in the RunnerTemplate (Part B7 below).
+# ---------------------------------------------------------------------------
+
+apply_athens() {
+	echo "Applying Athens in-cluster Go module cache..."
+	kubectl apply -k "${REPO_ROOT}/deploy/athens"
+	echo "  Waiting for Athens to be ready..."
+	kubectl rollout status deployment/athens -n gag-dogfood --timeout=120s
+}
+
 # Part B7 — the v2 tenant objects. The v2 API decomposes the v1 monolithic
 # ActionsGateway into ActionsGateway (gateway + credentials) + RunnerTemplate
 # (worker pod shape) + RunnerSet (runner group). Minimal direct-egress form:
@@ -392,6 +409,16 @@ ${runner_image_field}
           # own make-based CI fails make-command-not-found on it; export
           # DOGFOOD_RUNNER_IMAGE (built by scripts/dogfood-runner-build.sh) to pin
           # a build-capable image above instead (Q239). Injection still applies.
+          env:
+            # Route Go module fetches through Athens (Q244). Workers cannot reach
+            # proxy.golang.org directly (egress NetworkPolicy, GKE DPv2 no FQDN NP).
+            # Athens fetches from upstream on first request and caches to PVC.
+            # GONOSUMDB=* prevents direct sum.golang.org queries from workers;
+            # Athens validates checksums when it fetches from proxy.golang.org.
+            - name: GOPROXY
+              value: "http://go-module-proxy.gag-dogfood.svc.cluster.local:3000,off"
+            - name: GONOSUMDB
+              value: "*"
           resources:
             requests:
               cpu: "2"
@@ -457,6 +484,7 @@ main() {
 	create_namespace
 	create_secret
 	apply_quota
+	apply_athens
 	apply_cr
 
 	echo ""
@@ -473,6 +501,9 @@ main() {
 	echo "  2. Route CI to GAG:   scripts/dogfood-start.sh"
 	echo "  3. Take it offline:   scripts/dogfood-stop.sh"
 	echo "  4. One-time e2e pool: scripts/dogfood-e2e-setup.sh"
+	echo ""
+	echo "vendor-check and tidy-check are now routed to GAG runners. Athens"
+	echo "pre-warms on first request — expect a slower first run per module."
 }
 
 main "$@"

scripts/dogfood-start.sh

@@ -5,7 +5,7 @@
 # Required env vars (export before running):
 #   PROJECT   GCP project ID (e.g. actions-gateway-dogfood)
 #   CLUSTER   GKE cluster name (e.g. gag-dogfood)
-#   ZONE      GCP zone (e.g. us-central1-a)
+#   ZONE      GCP zone (e.g. us-central1-b)
 #   REPO      GitHub repo slug (e.g. actions-gateway/github-actions-gateway)
 set -euo pipefail