From fd5e8a26cc064c601cf37694a62107dc2d68d93b Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 24 Jul 2026 11:40:28 -0700 Subject: [PATCH 1/5] ci(bazel): shallow checkout for non-root matrix rows Only the root row (path '.') diffs a PR against its base and needs full history; every other container row and both docker-host rows build their whole scope regardless. Use fetch-depth 1 for those to skip a full-history clone on ~19 jobs per full matrix. detect keeps fetch-depth 0. Co-authored-by: Balaji Ganesan --- .github/workflows/bazel.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index b6c4b3d45..fa29ff244 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -331,9 +331,12 @@ jobs: steps: - uses: actions/checkout@v4 with: - # Full history so the root row can diff a PR against its base to - # compute affected targets (see "Determine build targets"). - fetch-depth: 0 + # Only the root row (path ".") needs full history: it diffs a PR + # against its base to compute affected targets (see "Determine build + # targets"). Every other row builds its whole scope regardless, so a + # shallow checkout there skips a full-history clone. (GHA treats 0 as + # falsy, so the ternary is written non-root-first to return 0 for root.) + fetch-depth: ${{ matrix.subtree.path != '.' && 1 || 0 }} # A subtree can be selected but not yet have its Bazel scaffold on # the mirror (its upstream OSS-flip MR is still open, or the @@ -717,7 +720,9 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 + # docker-host rows never build the root scope, so no git history is + # needed here; a shallow checkout is enough. + fetch-depth: 1 # The Java tree builds with local_jdk (root .bazelrc). Unlike the fast # matrix, this lane runs on a bare runner rather than the bazel-ci image, # so provide JDK 25 explicitly for the Java requires-docker tests. From 881a51e6086bf5ef43953b5fb5a8926af294cd9d Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 24 Jul 2026 13:41:15 -0700 Subject: [PATCH 2/5] ci(bazel): source the bazel-ci job container from a repo variable The bazel-ci image was hardcoded in four places and had already drifted to three different versions: bazel.yml on 0.13.0, image-push-manual.yml and chart-push-manual.yml on 0.8.0, and BAZEL.md documenting 0.12.0. The release path building shipping images on a different toolchain than CI validates with is a correctness problem, not just untidiness. Source the container from the BAZEL_CI_IMAGE repository variable so the image is bumped in one place. Each workflow keeps the current pin as a || fallback so CI still runs when the variable is unavailable, for example on a fork. It must be vars, not env: GitHub Actions evaluates job-level container.image before the workflow-level env: context is reliably available, which is why the previous note told maintainers to keep the tag literal. The vars context does not have that ordering problem, so the value can now be centralized. Co-authored-by: Balaji Ganesan --- .github/workflows/bazel.yml | 22 ++++++++++++++-------- .github/workflows/chart-push-manual.yml | 2 +- .github/workflows/image-push-manual.yml | 2 +- BAZEL.md | 13 ++++++++++--- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index fa29ff244..111f4ee44 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -52,13 +52,19 @@ concurrency: group: bazel-${{ github.ref }} cancel-in-progress: true -# NOTE: keep the bazel-ci image tag as a string literal in -# container.image and any tag-bump bookkeeping. GitHub Actions -# evaluates job-level `container.image` before the workflow-level -# `env:` context is reliably available, so referencing -# `${{ env.BAZEL_CI_IMAGE }}` here causes the matrix to expand to -# zero jobs on push events (validated empirically; PR-event runs -# happen to work). The duplication is intentional. +# The bazel-ci job container comes from the repository variable +# BAZEL_CI_IMAGE, so the image is bumped in one place (repo settings) instead +# of being edited in every workflow that runs in it. +# +# It must be `vars`, NOT `env`. GitHub Actions evaluates job-level +# `container.image` before the workflow-level `env:` context is reliably +# available, so `${{ env.BAZEL_CI_IMAGE }}` made the matrix expand to zero jobs +# on push events (validated empirically; PR-event runs happened to work). The +# `vars` context does not have that ordering problem. +# +# The `||` fallback keeps CI working if the variable is unset or unavailable +# (for example on a fork). Keep the fallback in step with the variable; it is a +# safety net, not the source of truth. jobs: detect: @@ -314,7 +320,7 @@ jobs: # Bazelisk then selects the root .bazelversion release. Update this tag # only after the corresponding internal image has been published and # mirrored. - image: ghcr.io/nvidia/nvcf/bazel-ci:0.13.0 + image: ${{ vars.BAZEL_CI_IMAGE || 'ghcr.io/nvidia/nvcf/bazel-ci:0.13.0' }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/chart-push-manual.yml b/.github/workflows/chart-push-manual.yml index 6053c7273..f1b7f01cd 100644 --- a/.github/workflows/chart-push-manual.yml +++ b/.github/workflows/chart-push-manual.yml @@ -34,7 +34,7 @@ jobs: name: package + push chart to ncp-dev runs-on: ubuntu-latest container: - image: ghcr.io/nvidia/nvcf/bazel-ci:0.8.0 + image: ${{ vars.BAZEL_CI_IMAGE || 'ghcr.io/nvidia/nvcf/bazel-ci:0.13.0' }} defaults: run: # In container jobs Actions falls back to plain `sh` (dash), which diff --git a/.github/workflows/image-push-manual.yml b/.github/workflows/image-push-manual.yml index fcf6adf6e..f2949f5ee 100644 --- a/.github/workflows/image-push-manual.yml +++ b/.github/workflows/image-push-manual.yml @@ -48,7 +48,7 @@ jobs: name: push to ncp-dev runs-on: ubuntu-latest container: - image: ghcr.io/nvidia/nvcf/bazel-ci:0.8.0 + image: ${{ vars.BAZEL_CI_IMAGE || 'ghcr.io/nvidia/nvcf/bazel-ci:0.13.0' }} defaults: run: # In container jobs Actions falls back to plain `sh` (dash), which diff --git a/BAZEL.md b/BAZEL.md index ccd108a58..39ff304f7 100644 --- a/BAZEL.md +++ b/BAZEL.md @@ -429,13 +429,20 @@ build). ## CI -The public GitHub Bazel matrix in `.github/workflows/bazel.yml` consumes -`ghcr.io/nvidia/nvcf/bazel-ci:0.12.0`. That image is built in the internal +Every workflow that needs the Bazel toolchain runs in the `bazel-ci` job +container, sourced from the repository variable `BAZEL_CI_IMAGE` (currently +`ghcr.io/nvidia/nvcf/bazel-ci`). Each workflow carries the current pin as a +`||` fallback so CI still runs when the variable is unavailable, for example on +a fork. + +That image is built in the internal [`nvcf/bazel-ci-templates`](https://gitlab-master.nvidia.com/nvcf/bazel-ci-templates) project, stamped with a version, and mirrored to GHCR. The mirror is currently manual; automation is planned. To change the image's Bazel, Java, or operating system tooling, update the internal template first, publish and mirror a new -tag, and only then update the pinned `container.image` in `.github/workflows/bazel.yml`. +tag, then update the `BAZEL_CI_IMAGE` repository variable. Do not hardcode the +tag per workflow: it drifted to three different versions across workflows and +this document before the variable was introduced. The root `ci/Dockerfile.bazel` and `.github/workflows/bazel-ci-image.yml` were a stale, divergent copy (no Java, older Bazel) and have been removed. The image is From 7f6df99a87f038bb7d8af9789a150dabb4a96858 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 24 Jul 2026 19:58:01 -0700 Subject: [PATCH 3/5] ci(image-push): discover services and module layout instead of hardcoding The manual image-push workflow carried a hardcoded choice list of 11 service paths, so every new service needed a workflow edit before its image could be pushed for pre-merge testing. cloud-tasks, for example, was not selectable. GitHub Actions cannot populate a choice input dynamically, so drop the list: service_path is a free-form path, and the workflow discovers what to build. It also assumed every subtree is its own Bazel module: it ran bazel from inside the subtree and queried //... . That is false for services in the repo-root module (the Java services), where the query finds nothing. The workflow now derives the layout from whether the subtree owns a MODULE.bazel, querying from the subtree with //... or from the root with a path-scoped pattern. Both were verified against a standalone module (grpc-proxy resolves //:image_index) and a root-module service (cloud-tasks resolves //src/control-plane-services/cloud-tasks/nvct-service:nvct-service-oss-image_index). An unknown path now fails with the list of subtrees owning a Bazel module, which replaces the discoverability the dropdown provided. Co-authored-by: Balaji Ganesan --- .github/workflows/image-push-manual.yml | 52 +++++++++++++++++-------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/.github/workflows/image-push-manual.yml b/.github/workflows/image-push-manual.yml index f2949f5ee..73808330c 100644 --- a/.github/workflows/image-push-manual.yml +++ b/.github/workflows/image-push-manual.yml @@ -18,21 +18,12 @@ on: workflow_dispatch: inputs: service_path: - description: Service subtree to build and push + description: >- + Service subtree to build and push, e.g. + src/invocation-plane-services/grpc-proxy. Any subtree with an + oci_image_index target works; the list is not maintained here. required: true - type: choice - options: - - src/invocation-plane-services/grpc-proxy - - src/invocation-plane-services/http-invocation - - src/invocation-plane-services/llm-api-gateway - - src/invocation-plane-services/ratelimiter - - src/invocation-plane-services/vanity-gateway - - src/control-plane-services/nats-auth-callout - - src/control-plane-services/function-autoscaler - - src/control-plane-services/helm-reval - - src/compute-plane-services/nvca - - src/compute-plane-services/ess-agent - - src/compute-plane-services/image-credential-helper + type: string permissions: contents: read @@ -116,10 +107,38 @@ jobs: printf '{"auths":{"%s":{"auth":"%s"}}}\n' "${REGISTRY}" "${auth}" > "$HOME/.docker/config.json" chmod 600 "$HOME/.docker/config.json" + # Resolve where Bazel must run and how to scope the query. A subtree with + # its own MODULE.bazel is queried from inside itself; a service that + # lives in the repo-root module (the Java services) is queried from the + # root with a path-scoped pattern. Deriving this rather than hardcoding + # it means a new service needs no change to this workflow. + - name: Resolve Bazel module layout + id: layout + env: + SVC_PATH: ${{ github.event.inputs.service_path }} + run: | + set -euo pipefail + if [ ! -d "$SVC_PATH" ]; then + echo "ERROR: no such subtree: $SVC_PATH" >&2 + echo "Subtrees owning a Bazel module:" >&2 + find src -maxdepth 4 -name MODULE.bazel -printf ' %h\n' 2>/dev/null | sort >&2 + exit 1 + fi + if [ -f "$SVC_PATH/MODULE.bazel" ]; then + echo "workdir=$SVC_PATH" >> "$GITHUB_OUTPUT" + echo "scope=//..." >> "$GITHUB_OUTPUT" + echo "layout: standalone module rooted at $SVC_PATH" + else + echo "workdir=." >> "$GITHUB_OUTPUT" + echo "scope=//$SVC_PATH/..." >> "$GITHUB_OUTPUT" + echo "layout: root module, scoped to //$SVC_PATH/..." + fi + - name: Build and push multi-arch image(s) - working-directory: ${{ github.event.inputs.service_path }} + working-directory: ${{ steps.layout.outputs.workdir }} env: SVC_PATH: ${{ github.event.inputs.service_path }} + SCOPE: ${{ steps.layout.outputs.scope }} TAG: ${{ steps.meta.outputs.tag }} REGISTRY: ${{ secrets.NCP_DEV_REGISTRY }} run: | @@ -127,9 +146,10 @@ jobs: REGISTRY="${REGISTRY%/}" svc="$(basename "$SVC_PATH")" export BAZEL_DISK_CACHE="${HOME}/.bazel-disk-cache" - mapfile -t indexes < <(bazel query --remote_cache= 'kind("oci_image_index", //...)') + mapfile -t indexes < <(bazel query --remote_cache= "kind(\"oci_image_index\", ${SCOPE})") if [ "${#indexes[@]}" -eq 0 ]; then echo "ERROR: no oci_image_index targets under ${SVC_PATH}" >&2 + echo "The subtree must declare an image target (go_oci_image, java_oci_image, ...)." >&2 exit 1 fi echo "discovered: ${indexes[*]}" From 7d4fcc4985596d7bf1270d4aa9a203ef0e11a91c Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 24 Jul 2026 20:05:23 -0700 Subject: [PATCH 4/5] fix(image-push): derive the push repo name for hyphenated image targets The repo-name rule only stripped an _image suffix, so a target using hyphens fell through to the default branch and got both the service prefix and the suffix left on. byoo-otel-collector-image became byoo-otel-collector-byoo-otel-collector-image, and cloud-tasks would have pushed nvct-service-oss as cloud-tasks-nvct-service-oss-image. The tree uses the separator to mean two different things: image the service's sole image; repo is the service _image a sub-component; repo is - -image the target already carries the full image name; use it as-is without the service prefix Add the hyphenated case. Verified against every image target in the tree: the 11 services currently selectable are unchanged, and byoo-otel-collector (2 targets) and cloud-tasks are corrected. byoo was latent because it is not in the old hardcoded list; making the input free-form would have exposed it. Co-authored-by: Balaji Ganesan --- .github/workflows/image-push-manual.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/image-push-manual.yml b/.github/workflows/image-push-manual.yml index 73808330c..a7fbb340a 100644 --- a/.github/workflows/image-push-manual.yml +++ b/.github/workflows/image-push-manual.yml @@ -155,9 +155,21 @@ jobs: echo "discovered: ${indexes[*]}" for tgt in "${indexes[@]}"; do name="${tgt##*:}"; name="${name%_index}" + # Two naming conventions exist in the tree and they mean different + # things, distinguished by the separator: + # image -> the service's sole image; repo is the service + # _image -> a sub-component; repo is - + # (nvcf-unbound webhook, llm-api-gateway + # rate_limit_sync_worker, nvsnap agent/server) + # -image -> the target already carries the full image + # name; use it as-is, do NOT prefix the + # service (byoo-otel-collector, cloud-tasks) + # Previously the hyphenated form fell through to the default and + # produced names like byoo-otel-collector-byoo-otel-collector-image. case "$name" in image) repo="${svc}" ;; *_image) sub="$(printf '%s' "${name%_image}" | tr '_' '-')"; repo="${svc}-${sub}" ;; + *-image) repo="${name%-image}" ;; *) sub="$(printf '%s' "$name" | tr '_' '-')"; repo="${svc}-${sub}" ;; esac dest="${REGISTRY}/${repo}" From 054f2c1cc7626d358237fcf884944214afff891e Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 24 Jul 2026 20:56:14 -0700 Subject: [PATCH 5/5] fix(image-push): surface bazel query failures instead of masking them The target discovery used mapfile with process substitution, so only stdout reached mapfile and the query's exit status was discarded. A failing query (a BUILD error, an unloadable package, a bad scope) produced an empty array and was then misreported as "no oci_image_index targets under ", pointing the operator at the wrong problem. Run the query directly, check its status, and build the array explicitly. The array is built in a loop rather than from a here-string because a here-string of empty output yields one empty element instead of an empty array, which would defeat the existing no-targets check. Verified all four cases: failing query surfaces the error, empty result gives 0 targets, single and multiple results parse correctly. Co-authored-by: Balaji Ganesan --- .github/workflows/image-push-manual.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/image-push-manual.yml b/.github/workflows/image-push-manual.yml index a7fbb340a..6555c5d09 100644 --- a/.github/workflows/image-push-manual.yml +++ b/.github/workflows/image-push-manual.yml @@ -146,7 +146,21 @@ jobs: REGISTRY="${REGISTRY%/}" svc="$(basename "$SVC_PATH")" export BAZEL_DISK_CACHE="${HOME}/.bazel-disk-cache" - mapfile -t indexes < <(bazel query --remote_cache= "kind(\"oci_image_index\", ${SCOPE})") + # Run the query directly rather than inside process substitution. + # With `mapfile < <(bazel query ...)` only stdout reaches mapfile, so a + # failing query (BUILD error, unloadable package) yields an empty array + # and would be misreported below as "no image targets" instead of + # surfacing the real error. + if ! query_out="$(bazel query --remote_cache= "kind(\"oci_image_index\", ${SCOPE})")"; then + echo "ERROR: bazel query failed for scope ${SCOPE}" >&2 + exit 1 + fi + # Build the array by hand: a here-string of empty output would produce + # a single empty element rather than an empty array. + indexes=() + while IFS= read -r line; do + [ -n "$line" ] && indexes+=("$line") + done <<< "$query_out" if [ "${#indexes[@]}" -eq 0 ]; then echo "ERROR: no oci_image_index targets under ${SVC_PATH}" >&2 echo "The subtree must declare an image target (go_oci_image, java_oci_image, ...)." >&2