diff --git a/deploy/athens/deployment.yaml b/deploy/athens/deployment.yaml new file mode 100644 index 000000000..717b5d617 --- /dev/null +++ b/deploy/athens/deployment.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: athens + namespace: gag-dogfood + labels: + app: athens +spec: + replicas: 1 + selector: + matchLabels: + app: athens + template: + metadata: + labels: + app: athens + spec: + # No toleration for dedicated=workers, so this lands on the system node pool + # and stays up when spot workers are scaled to zero. + containers: + - name: athens + # v0.18.0 bundles Go 1.26.2; older tags shipped EOL Go (v0.15.1 had + # 1.20.14) that Athens uses to rebuild module zips on direct fetches. + image: gomods/athens:v0.18.0 + ports: + - containerPort: 3000 + env: + - name: ATHENS_STORAGE_TYPE + value: disk + - name: ATHENS_DISK_STORAGE_ROOT + value: /var/lib/athens + - name: ATHENS_DOWNLOAD_MODE + value: sync + - name: ATHENS_LOG_LEVEL + value: info + # Fetch upstream modules through proxy.golang.org, not direct-from-VCS. + # This overrides Athens' built-in default (GoBinaryEnvVars = + # ["GOPROXY=direct"]), which forces its internal `go` subprocess to clone + # each module from its origin and rebuild the zip locally. Athens' bundled + # (older) Go computes a different module hash than the notarized one for + # modules declaring a newer go directive (e.g. ginkgo/v2@v2.32.0, go 1.25), + # so its own checksum check rejects the rebuild as a mismatch. The public + # proxy instead serves the immutable, notarized zip whose hash matches + # sum.golang.org and our committed go.sum — the check passes. The proxy is + # also faster (pre-built zips over a CDN) and more available (one upstream + # vs. every module's origin host). A plain container-level GOPROXY env var + # does NOT work — Athens overrides it for the subprocess via this config. + # The ",direct" fallback still resolves any module proxy.golang.org does + # not mirror; Athens splits this var on ";", so the comma stays inside the + # GOPROXY value (matching Go's own default GOPROXY). + - name: ATHENS_GO_BINARY_ENV_VARS + value: GOPROXY=https://proxy.golang.org,direct + # Athens verifies fetched modules against sum.golang.org — it has free + # egress (no workload NetworkPolicy on this pod). Empty GONOSUMCHECK means + # "verify every module"; integrity is enforced end-to-end. Workers, which + # have no egress to sum.golang.org, instead set GONOSUMDB=* and rely on the + # committed go.sum. + - name: ATHENS_GONOSUMCHECK + value: "" + volumeMounts: + - name: storage + mountPath: /var/lib/athens + resources: + requests: + # Kept small so Athens reliably co-schedules with the AGC on the single + # system node, whose CPU-request budget is tight during CI worker bursts. + # Athens is I/O-bound (serving cached zips from the PVC); the generous + # limits still allow bursting for cold-cache `go mod download` fetches. + cpu: "100m" + memory: "192Mi" + limits: + cpu: "1" + memory: "1Gi" + readinessProbe: + httpGet: + path: /healthz + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: 3000 + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - name: storage + persistentVolumeClaim: + claimName: athens-storage diff --git a/deploy/athens/kustomization.yaml b/deploy/athens/kustomization.yaml new file mode 100644 index 000000000..ef959f64d --- /dev/null +++ b/deploy/athens/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - pvc.yaml + - deployment.yaml + - service.yaml + - networkpolicy.yaml diff --git a/deploy/athens/networkpolicy.yaml b/deploy/athens/networkpolicy.yaml new file mode 100644 index 000000000..486d312ee --- /dev/null +++ b/deploy/athens/networkpolicy.yaml @@ -0,0 +1,37 @@ +# Allow worker pods to reach Athens on port 3000. +# +# The GMC-managed workload NetworkPolicy restricts egress for pods labelled +# actions-gateway/component=workload to DNS + the egress proxy (or GitHub CIDRs +# in direct mode). That policy does not allow port 3000. This additive policy +# opens the one extra rule needed: workload pods → Athens pods on 3000. +# +# Athens pods (app=athens) are not labelled actions-gateway/component=workload, +# so they are not covered by the workload NetworkPolicy and retain free egress +# to fetch modules from proxy.golang.org and validate against sum.golang.org. +# +# Plain HTTP (no TLS) is intentional here. Athens serves public Go module zips; +# confidentiality is not a concern. Integrity is upheld by the Go toolchain's +# go.sum verification — every downloaded module is checked against the committed +# go.sum regardless of GONOSUMDB — so a tampered response would be caught before +# it ever reaches the build. Adding TLS would require cert management (cert-manager +# or a self-signed CA wired into every worker) for no meaningful security gain in +# this single-tenant dogfood cluster. Revisit if Athens is extended to a shared +# multi-tenant cluster or used to serve private modules. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: athens-worker-access + namespace: gag-dogfood +spec: + podSelector: + matchLabels: + actions-gateway/component: workload + policyTypes: [Egress] + egress: + - to: + - podSelector: + matchLabels: + app: athens + ports: + - protocol: TCP + port: 3000 diff --git a/deploy/athens/pvc.yaml b/deploy/athens/pvc.yaml new file mode 100644 index 000000000..5b106e82a --- /dev/null +++ b/deploy/athens/pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: athens-storage + namespace: gag-dogfood +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 20Gi + # GKE default: standard-rwo (Balanced PD, ReadWriteOnce, faster than standard). + storageClassName: standard-rwo diff --git a/deploy/athens/service.yaml b/deploy/athens/service.yaml new file mode 100644 index 000000000..618d96521 --- /dev/null +++ b/deploy/athens/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + # Named go-module-proxy, not athens: a service named "athens" causes Kubernetes + # to inject ATHENS_PORT=tcp://clusterip:3000 into pods in the namespace, which + # Athens reads as its own listen address and fails with "too many colons". + name: go-module-proxy + namespace: gag-dogfood +spec: + selector: + app: athens + ports: + - name: http + port: 3000 + targetPort: 3000 + type: ClusterIP diff --git a/docs/STATUS.md b/docs/STATUS.md index 0d3c1896b..f92a9ac72 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -52,10 +52,9 @@ Specific actionable items in priority order. Pick from the top; skip 🚫 items | ID | Item | Labels | St | Sz | Notes | |---|---|---|---|---|---| -| Q224 | [GKE dogfood: route production CI (green CI blocked)](plan/gke-dogfood.md) | `milestone` `infra` | 🚫 | M | rc.4 turn-on validated; toolchain unblocked (Q239). Egress half blocked: [Q242](#Q242) shipped but CiliumFQDN unusable on GKE DPv2 (no CiliumNetworkPolicy CRD); unblock via [Q244](#Q244) cache or [Q245](#Q245) GKEFQDN. On-demand. | -| Q242 | [Implement G.1 proxy destination allowlist](plan/q242-g1-proxy-destination-allowlist.md) | `security` `infra` | ▶ | L | Impl merged #460–#464. Remaining: dogfood ([Q224](#Q224)) blocked — GKE DPv2 managed Cilium lacks CiliumNetworkPolicy CRD (CiliumFQDN→Degraded); unblock via [Q244](#Q244) cache or [Q245](#Q245) GKEFQDN. v2beta1 blocker. | +| Q224 | [GKE dogfood: route production CI (green CI blocked)](plan/gke-dogfood.md) | `milestone` `infra` | 🔲 | M | rc.4 turn-on validated; toolchain unblocked (Q239). Egress blocker for vendor-check/tidy-check resolved: Athens in-cluster cache deployed (Q244). Run `scripts/dogfood-start.sh` + validate CI green end-to-end. On-demand. | +| Q242 | [Implement G.1 proxy destination allowlist](plan/q242-g1-proxy-destination-allowlist.md) | `security` `infra` | ▶ | L | Impl merged #460–#464. Dogfood vendor-check/tidy-check unblocked via Athens (Q244). Remaining: flip `GAG_RUNNER`, validate green CI ([Q224](#Q224)); FQDN intent/backend split ([Q245](#Q245)). v2beta1 blocker. | | Q243 | [Per-tenant egress-IP reference architecture (cloud)](plan/gke-dogfood.md) | `security` `infra` `docs` | 🔲 | L | Substantiate the per-tenant egress-IP isolation claim: spike + validate Cilium Egress Gateway vs per-tenant NAT on a cloud; doc single-tenant-direct vs production topology + cost. Dogfood stays direct (single-tenant). v2beta1 blocker. | -| Q244 | [In-cluster Go module cache (Athens) for the dogfood](plan/q242-g1-proxy-destination-allowlist.md) | `infra` | 🔲 | M | Stand up an in-cluster Athens Go-module cache so dogfood vendor-check/tidy-check fetch modules without per-worker external egress — closes [Q224](#Q224) on GKE without an FQDN backend (mirror path; workers reach it via destinationCIDRs). | | Q245 | [FQDN egress: split intent from CNI backend + GKE backend](plan/q242-g1-proxy-destination-allowlist.md#provider-fqdn-egress-fragmentation-post-implementation-finding) | `security` `infra` | 🔲 | L | egressPolicyMode FQDN variants encode a per-CNI kind, fragmented across GKE/AKS/EKS/OVN. Decouple tenant intent (CIDR\|FQDN) from a platform --fqdn-policy-backend; add gke backend (networking.gke.io FQDNNetworkPolicy). Fold into v2beta1 (Q74). | | Q225 | [Operator docs: Kata Containers for DinD workloads](operations/in-runner-image-builds.md) | `docs` | 🔲 | S | Document runtimeClassName: kata-qemu on podTemplate, /dev/kvm machine-type requirement (N2+ on GCP), and Kata DaemonSet setup. Extend in-runner-image-builds.md or new page depending on density. | | Q226 | [Kata Containers on GKE — secure CI reference architecture](plan/kata-on-gke.md) | `security` `infra` | 🔲 | M | OSS untrusted-PR threat + GAG dogfood requirement rule out privileged DinD. Spike: GKE nested-virt node pool + Kata RuntimeClass: kind in micro-VM, no privileged pod. Reference arch. [plan](plan/kata-on-gke.md) | diff --git a/docs/plan/gke-dogfood.md b/docs/plan/gke-dogfood.md index 7cdb1e9af..1cdbe7cbe 100644 --- a/docs/plan/gke-dogfood.md +++ b/docs/plan/gke-dogfood.md @@ -23,7 +23,7 @@ profile or paste them at the start of each terminal session. ```bash CLUSTER=gag-dogfood -ZONE=us-central1-a +ZONE=us-central1-b PROJECT=actions-gateway-dogfood # must be globally unique; append 4 digits if needed REPO=actions-gateway/github-actions-gateway APP_ID=3752347 @@ -271,6 +271,46 @@ spec: EOF ``` +### B6b. Deploy Athens in-cluster Go module cache + +`vendor-check` and `tidy-check` re-fetch Go modules from `proxy.golang.org` on +a cold cache. GKE Dataplane V2's managed Cilium lacks the `CiliumNetworkPolicy` +CRD, so the `CiliumFQDN` egress mode is unusable here; a CIDR allowlist for +Google-fronted hosts like `proxy.golang.org` would be a footgun (it opens all +of Google's frontend). Athens sidesteps both constraints: it runs in-cluster +with free egress and serves cached modules to workers over a plain HTTP port +that does not need a CNI FQDN backend. + +Athens is not covered by the workload `NetworkPolicy` (`actions-gateway/component: workload` +label). Workers reach it via an additive `NetworkPolicy` that opens port 3000 +from workload pods to Athens pods. The Service is named `go-module-proxy` +(not `athens`) to avoid Kubernetes injecting `ATHENS_PORT=tcp://...` into +pods in the namespace — Athens misreads that as its listen address. + +```bash +kubectl apply -k deploy/athens +kubectl rollout status deployment/athens -n gag-dogfood --timeout=120s +``` + +Verify Athens is healthy: + +```bash +kubectl get pods -n gag-dogfood -l app=athens +kubectl logs -n gag-dogfood -l app=athens --tail=20 +``` + +Athens pre-warms lazily — the first `vendor-check`/`tidy-check` run is slower +while modules download; subsequent runs are cache hits from the PVC. + +> **Why plain HTTP (no TLS)?** Athens serves public Go module zips; there is +> nothing confidential in transit. Integrity is upheld by the Go toolchain's +> `go.sum` verification — every module downloaded from Athens is checked against +> the committed `go.sum` regardless of `GONOSUMDB`, so a tampered response is +> caught before it reaches the build. Adding TLS would require cert management +> (cert-manager or a self-signed CA wired into every worker image) for no +> meaningful security gain in this single-tenant cluster. Revisit if Athens is +> extended to a shared multi-tenant cluster or used to serve private modules. + ### B7. Create the v2 tenant objects The v2 API decomposes the v1 monolithic `ActionsGateway` into `ActionsGateway` @@ -318,6 +358,15 @@ spec: # repo's make-based CI fails `make: command not found` on it (see the # Known gap below). For green CI, set a build-capable workerImage here; # injection still applies on top of any base. + env: + # Athens in-cluster Go module proxy (Q244). Workers cannot reach + # proxy.golang.org directly (egress NetworkPolicy, GKE DPv2 no FQDN NP). + # GONOSUMDB=* prevents direct sum.golang.org queries; Athens validates + # checksums when it fetches from proxy.golang.org upstream. + - name: GOPROXY + value: "http://go-module-proxy.gag-dogfood.svc.cluster.local:3000,off" + - name: GONOSUMDB + value: "*" resources: requests: cpu: "2" @@ -390,25 +439,22 @@ gh api /repos/"$REPO"/actions/runners \ > which failed `make: command not found` on the bare image, ran green on > `dogfood-runner:2.335.1` with the wrapper injected (`make` 4.3, `gcc` 13.3.0). > -> **Residual blocker for `vendor-check` / `tidy-check`.** Those two jobs re-fetch Go -> modules from `proxy.golang.org` on a cold cache, which the GitHub-only worker -> egress allowlist blocks — independent of the toolchain. The offline-capable jobs -> (`lint`, `shellcheck`, `unit-test`, `coverage`) build from `vendor/` and pull -> Go/shellcheck from GitHub releases, so the image unblocks those. +> **`vendor-check` / `tidy-check` unblocked by Athens (Q244, implemented).** An +> Athens in-cluster Go module proxy (`deploy/athens/`, applied by `dogfood-setup.sh`) +> caches Go modules so workers never need to reach `proxy.golang.org` directly. +> Athens pods (app=athens) are not covered by the workload NetworkPolicy and have +> free egress; workers reach Athens via an additive NetworkPolicy (port 3000) and +> are wired with `GOPROXY=http://go-module-proxy.gag-dogfood.svc.cluster.local:3000,off` +> plus `GONOSUMDB=*` in the RunnerTemplate. > -> **The proxy destination allowlist (Q242 G.1) shipped, but its `CiliumFQDN` mode -> does NOT work on this cluster.** GKE Dataplane V2's *managed* Cilium does not +> **Background (for reference):** GKE Dataplane V2's *managed* Cilium does not > expose the `cilium.io/v2 CiliumNetworkPolicy` CRD (dropped since GKE > 1.21.5-gke.1300), so an `EgressProxy` with `egressPolicyMode: CiliumFQDN` goes -> `Degraded` (`no matches for kind "CiliumNetworkPolicy"`, verified 2026-06-29 — the -> fail-closed posture worked, nothing opened). `destinationCIDRs` is no substitute -> for `proxy.golang.org`/`sum.golang.org` (Google-fronted ⇒ a CIDR allowlist opens -> all of Google's frontend). Two ways to close this, both on the Queue: (a) an -> **in-cluster Go module cache** (Athens — the design-recommended path, works on GKE -> today); (b) a **GKEFQDN backend** emitting `networking.gke.io FQDNNetworkPolicy` -> (`--enable-fqdn-network-policy`). Detail + the provider matrix: +> `Degraded` (`no matches for kind "CiliumNetworkPolicy"`, verified 2026-06-29). +> `destinationCIDRs` is no substitute for `proxy.golang.org`/`sum.golang.org` +> (Google-fronted ⇒ a CIDR allowlist opens all of Google's frontend). The FQDN +> intent/mechanism split (Q245) remains open. Detail + provider matrix: > [Q242 plan § Provider FQDN-egress fragmentation](q242-g1-proxy-destination-allowlist.md#provider-fqdn-egress-fragmentation-post-implementation-finding). -> Until one lands, keep `vendor-check`/`tidy-check` on `ubuntu-latest`. --- diff --git a/scripts/dogfood-e2e-setup.sh b/scripts/dogfood-e2e-setup.sh index 612107520..b9b546439 100755 --- a/scripts/dogfood-e2e-setup.sh +++ b/scripts/dogfood-e2e-setup.sh @@ -10,7 +10,7 @@ # Required env vars (export before running): # PROJECT GCP project ID (e.g. actions-gateway-dogfood) # CLUSTER GKE cluster name (e.g. gag-dogfood) -# ZONE GCP zone (e.g. us-central1-a) +# ZONE GCP zone (e.g. us-central1-b) # REPO GitHub repo slug (e.g. actions-gateway/github-actions-gateway) # APP_ID GitHub App numeric ID (3752347) # INSTALLATION_ID GitHub App installation ID for this repo diff --git a/scripts/dogfood-setup.sh b/scripts/dogfood-setup.sh index 40205b57b..8d23237ce 100755 --- a/scripts/dogfood-setup.sh +++ b/scripts/dogfood-setup.sh @@ -19,7 +19,7 @@ # Required env vars (export before running): # PROJECT GCP project ID (e.g. actions-gateway-dogfood) # CLUSTER GKE cluster name (e.g. gag-dogfood) -# ZONE GCP zone (e.g. us-central1-a) +# ZONE GCP zone (e.g. us-central1-b) # REPO GitHub repo slug (e.g. actions-gateway/github-actions-gateway) # APP_ID GitHub App numeric ID (3752347) # INSTALLATION_ID GitHub App installation ID for this repo/org @@ -337,6 +337,23 @@ EOF } # --------------------------------------------------------------------------- +# Part B6b — Athens in-cluster Go module proxy (Q244). Athens caches Go +# modules so vendor-check/tidy-check can run on GAG runners without external +# egress to proxy.golang.org. The Athens pod (app=athens) is not labelled +# actions-gateway/component=workload, so it is not covered by the workload +# NetworkPolicy and retains free egress to fetch modules. Worker pods reach +# Athens via an additive NetworkPolicy in deploy/athens/networkpolicy.yaml +# that opens port 3000 from workload pods to Athens pods. Workers are wired +# via GOPROXY/GONOSUMDB env vars in the RunnerTemplate (Part B7 below). +# --------------------------------------------------------------------------- + +apply_athens() { + echo "Applying Athens in-cluster Go module cache..." + kubectl apply -k "${REPO_ROOT}/deploy/athens" + echo " Waiting for Athens to be ready..." + kubectl rollout status deployment/athens -n gag-dogfood --timeout=120s +} + # Part B7 — the v2 tenant objects. The v2 API decomposes the v1 monolithic # ActionsGateway into ActionsGateway (gateway + credentials) + RunnerTemplate # (worker pod shape) + RunnerSet (runner group). Minimal direct-egress form: @@ -392,6 +409,16 @@ ${runner_image_field} # own make-based CI fails make-command-not-found on it; export # DOGFOOD_RUNNER_IMAGE (built by scripts/dogfood-runner-build.sh) to pin # a build-capable image above instead (Q239). Injection still applies. + env: + # Route Go module fetches through Athens (Q244). Workers cannot reach + # proxy.golang.org directly (egress NetworkPolicy, GKE DPv2 no FQDN NP). + # Athens fetches from upstream on first request and caches to PVC. + # GONOSUMDB=* prevents direct sum.golang.org queries from workers; + # Athens validates checksums when it fetches from proxy.golang.org. + - name: GOPROXY + value: "http://go-module-proxy.gag-dogfood.svc.cluster.local:3000,off" + - name: GONOSUMDB + value: "*" resources: requests: cpu: "2" @@ -457,6 +484,7 @@ main() { create_namespace create_secret apply_quota + apply_athens apply_cr echo "" @@ -473,6 +501,9 @@ main() { echo " 2. Route CI to GAG: scripts/dogfood-start.sh" echo " 3. Take it offline: scripts/dogfood-stop.sh" echo " 4. One-time e2e pool: scripts/dogfood-e2e-setup.sh" + echo "" + echo "vendor-check and tidy-check are now routed to GAG runners. Athens" + echo "pre-warms on first request — expect a slower first run per module." } main "$@" diff --git a/scripts/dogfood-start.sh b/scripts/dogfood-start.sh index dff54dd31..ae71d3bcf 100755 --- a/scripts/dogfood-start.sh +++ b/scripts/dogfood-start.sh @@ -5,7 +5,7 @@ # Required env vars (export before running): # PROJECT GCP project ID (e.g. actions-gateway-dogfood) # CLUSTER GKE cluster name (e.g. gag-dogfood) -# ZONE GCP zone (e.g. us-central1-a) +# ZONE GCP zone (e.g. us-central1-b) # REPO GitHub repo slug (e.g. actions-gateway/github-actions-gateway) set -euo pipefail diff --git a/scripts/dogfood-stop.sh b/scripts/dogfood-stop.sh index 9838f1ccb..5a2c437a0 100755 --- a/scripts/dogfood-stop.sh +++ b/scripts/dogfood-stop.sh @@ -5,7 +5,7 @@ # Required env vars (export before running): # PROJECT GCP project ID (e.g. actions-gateway-dogfood) # CLUSTER GKE cluster name (e.g. gag-dogfood) -# ZONE GCP zone (e.g. us-central1-a) +# ZONE GCP zone (e.g. us-central1-b) # REPO GitHub repo slug (e.g. actions-gateway/github-actions-gateway) set -euo pipefail