From 5ce357c68ddd147df56c191cee3c72085e7cb4f3 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 24 Jul 2026 17:07:06 -0700 Subject: [PATCH 1/4] chore(bazel): remove references to the decommissioned remote cache The internal Buildbarn host was retired, but it was still referenced in nine files. The per-service .bazel-remote-probe scripts defaulted NVCF_BAZEL_REMOTE_HOST to it, so an unset variable pointed the probe at a dead host rather than cleanly disabling the cache. - Probe scripts (6): no default host. An unset NVCF_BAZEL_REMOTE_HOST now disables the cache and builds local-only, which is the safe fallback the scripts already document. No new endpoint is hardcoded in its place; the caller supplies it, matching the root .bazelrc. - nvsnap/.bazelrc: drop the dead endpoint from the :remote profile; the profile keeps only policy and tuning flags. - byoo README, grpc-proxy geo BUILD.bazel: comment references. While in the geo BUILD.bazel comment, corrected a factual error: it claimed does NOT bypass result caching. It does. is the union of no-sandbox, no-remote-exec and no-cache. The same misconception is what kept byoo-otel-collector recompiling on every CI run. Co-authored-by: Balaji Ganesan --- .../byoo-otel-collector/README.md | 2 +- .../ess-agent/scripts/.bazel-remote-probe | 13 +++++++------ .../nvca/scripts/.bazel-remote-probe | 13 +++++++------ src/compute-plane-services/nvsnap/.bazelrc | 17 ++++++++--------- .../scripts/.bazel-remote-probe | 13 +++++++------ .../helm-reval/scripts/.bazel-remote-probe | 13 +++++++------ .../grpc-proxy/proxy/geo/BUILD.bazel | 9 +++++---- .../http-invocation/scripts/.bazel-remote-probe | 13 +++++++------ .../llm-api-gateway/scripts/.bazel-remote-probe | 13 +++++++------ 9 files changed, 56 insertions(+), 50 deletions(-) diff --git a/src/compute-plane-services/byoo-otel-collector/README.md b/src/compute-plane-services/byoo-otel-collector/README.md index 248fb6cfa..8b244368f 100644 --- a/src/compute-plane-services/byoo-otel-collector/README.md +++ b/src/compute-plane-services/byoo-otel-collector/README.md @@ -76,7 +76,7 @@ Cache contract: Bazel rebuilds the genrule when any input `local = True` + `tags = ["no-sandbox"]` so it can resolve `go` from `$PATH` and write to the standard Go module cache. The wrapper binary, in contrast, is a regular `go_binary` and benefits from full -Bazel hermeticity + nvcfbarn remote-cache reuse. +Bazel hermeticity + remote-cache reuse. A containerized Go application that provides a complete observability solution by orchestrating three functional components: it generates OpenTelemetry Collector configurations, extracts and manages secrets from ESS (Encrypted Secret Store), and runs a custom-built OpenTelemetry Collector binary. diff --git a/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe b/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe index 5812cc017..fe51d99c3 100755 --- a/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe +++ b/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -10,24 +10,25 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI diff --git a/src/compute-plane-services/nvca/scripts/.bazel-remote-probe b/src/compute-plane-services/nvca/scripts/.bazel-remote-probe index 0f7d82a3c..f5ad1640b 100755 --- a/src/compute-plane-services/nvca/scripts/.bazel-remote-probe +++ b/src/compute-plane-services/nvca/scripts/.bazel-remote-probe @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -8,24 +8,25 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI diff --git a/src/compute-plane-services/nvsnap/.bazelrc b/src/compute-plane-services/nvsnap/.bazelrc index 4c4f819b7..328970108 100644 --- a/src/compute-plane-services/nvsnap/.bazelrc +++ b/src/compute-plane-services/nvsnap/.bazelrc @@ -42,21 +42,20 @@ build:release --stamp # a second pipeline on the same runner skips work the first one # already did. Devs benefit too; the path is gitignored. # -# 2. Remote cache (opt-in via --config=remote). Points at nvcfbarn, -# the team Buildbarn at NVIDIA. Default is OFF in this repo -# because the nvsnap project has no probe wired yet -- enabling -# --config=remote on a runner that cannot reach nvcfbarn fails -# the build hard (Bazel's initial GetCapabilities RPC is not -# covered by --remote_local_fallback). CI sets +# 2. Remote cache (opt-in via --config=remote). The caller supplies the +# endpoint with --remote_cache; this file carries only the policy and +# tuning flags. Default is OFF in this repo because the nvsnap project +# has no probe wired yet -- enabling --config=remote on a runner that +# cannot reach the cache fails the build hard (Bazel's initial +# GetCapabilities RPC is not covered by --remote_local_fallback). CI sets # --config=remote-write explicitly in .gitlab-ci.yml; devs and # ad-hoc builds stay on disk-only. build --disk_cache=.bazel-cache/disk build --repository_cache=.bazel-cache/repo -# Read-only remote cache profile. Pulls hits from nvcfbarn but does -# not upload. Layer on top of this with --config=remote-write to +# Read-only remote cache profile. Pulls hits but does not upload. Supply +# the endpoint with --remote_cache; layer --config=remote-write on top to # also write back results. -build:remote --remote_cache=grpc://nvcfbarn.nvidia.com:8980 build:remote --remote_upload_local_results=false build:remote --remote_cache_compression build:remote --remote_local_fallback diff --git a/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe b/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe index 5812cc017..fe51d99c3 100755 --- a/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe +++ b/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -10,24 +10,25 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI diff --git a/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe b/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe index 0f7d82a3c..f5ad1640b 100755 --- a/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe +++ b/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -8,24 +8,25 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI diff --git a/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel b/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel index 7bf9b333f..3c0056085 100644 --- a/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel +++ b/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel @@ -74,10 +74,11 @@ go_test( local = True, # external: testcontainers spins up the localstack Docker container, # so the test's effective inputs (Docker daemon, localstack image - # tag) live outside Bazel's hermetic action hash. `local = True` - # forces local-only execution but does NOT bypass result caching; - # the `external` tag is what stops nvcfbarn from pinning a green - # forever. requires-docker is kept so callers can filter out + # tag) live outside Bazel's hermetic action hash. Note `local = True` + # is the union of no-sandbox, no-remote-exec AND no-cache, so it does + # bypass result caching; the `external` tag additionally stops a cached + # pass from pinning a green forever. requires-docker is kept so callers + # can filter out # docker-needing tests with --test_tag_filters=-requires-docker. tags = [ "external", diff --git a/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe b/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe index 5812cc017..fe51d99c3 100755 --- a/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe +++ b/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -10,24 +10,25 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI diff --git a/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe b/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe index 5812cc017..fe51d99c3 100755 --- a/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe +++ b/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -10,24 +10,25 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI From 8f8951448a9c217e879b6fef231275c0aa095541 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 24 Jul 2026 17:55:53 -0700 Subject: [PATCH 2/4] chore(bazel): drop byoo README from the cache-reference cleanup The byoo-otel-collector subtree has a version gate that requires bumping its VERSION file whenever ANY file under the subtree changes, with no exemption for documentation. Bumping the collector's version to fix a one-word comment would cut a spurious release, so leave that reference for the next byoo change that bumps VERSION for a real reason. All functional references are still removed; the remaining one is a single prose mention in a README. Co-authored-by: Balaji Ganesan --- src/compute-plane-services/byoo-otel-collector/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compute-plane-services/byoo-otel-collector/README.md b/src/compute-plane-services/byoo-otel-collector/README.md index 8b244368f..248fb6cfa 100644 --- a/src/compute-plane-services/byoo-otel-collector/README.md +++ b/src/compute-plane-services/byoo-otel-collector/README.md @@ -76,7 +76,7 @@ Cache contract: Bazel rebuilds the genrule when any input `local = True` + `tags = ["no-sandbox"]` so it can resolve `go` from `$PATH` and write to the standard Go module cache. The wrapper binary, in contrast, is a regular `go_binary` and benefits from full -Bazel hermeticity + remote-cache reuse. +Bazel hermeticity + nvcfbarn remote-cache reuse. A containerized Go application that provides a complete observability solution by orchestrating three functional components: it generates OpenTelemetry Collector configurations, extracts and manages secrets from ESS (Encrypted Secret Store), and runs a custom-built OpenTelemetry Collector binary. From d9f3b96a3aa94cf2242f50be6135a84792ec583e Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 25 Jul 2026 09:02:53 -0700 Subject: [PATCH 3/4] test(ci): add behavioral tests for the remote-cache probes Two review findings. Behavioral tests. bash -n only checks syntax, and this PR changed what the probes decide, not just how they parse. Add tools/ci/test-bazel-remote-probe: it sources every subtree probe with a stubbed grpcurl (no network) and asserts the enable/disable decision across four cases: host unset, remote explicitly off, host configured, and TLS requested without a CA pem. Six probes, four cases each. The host-unset case deliberately sets NVCF_BAZEL_REMOTE_TLS=0. Without that it was vacuous: with TLS on, an unset CA pem also disables the cache, so the case passed even with the old default host restored. Verified by reintroducing the old default on one probe, confirming the test then fails, and restoring it. Comment accuracy. The geo BUILD.bazel comment equated with disabling all caching. Measured on Bazel 9.1.1 with a scratch workspace: a genrule re-executes after a clean against a warm disk cache while an otherwise identical no-sandbox + no-remote-exec genrule reports a disk cache hit. So does stop disk/remote reuse, but it is not a blanket no-cache: the tag is what prevents reusing a cached test result. Comment now says exactly that. Co-authored-by: Balaji Ganesan --- .../grpc-proxy/proxy/geo/BUILD.bazel | 9 +- tools/ci/test-bazel-remote-probe | 84 +++++++++++++++++++ 2 files changed, 89 insertions(+), 4 deletions(-) create mode 100755 tools/ci/test-bazel-remote-probe diff --git a/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel b/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel index 3c0056085..de529a5f5 100644 --- a/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel +++ b/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel @@ -74,10 +74,11 @@ go_test( local = True, # external: testcontainers spins up the localstack Docker container, # so the test's effective inputs (Docker daemon, localstack image - # tag) live outside Bazel's hermetic action hash. Note `local = True` - # is the union of no-sandbox, no-remote-exec AND no-cache, so it does - # bypass result caching; the `external` tag additionally stops a cached - # pass from pinning a green forever. requires-docker is kept so callers + # tag) live outside Bazel's hermetic action hash. `local = True` forces + # local, unsandboxed execution and, measured on Bazel 9.1.1, also stops + # the result being reused from the disk/remote cache. It does not disable + # every cache: the `external` tag is what prevents Bazel reusing a + # previously cached test result. requires-docker is kept so callers # can filter out # docker-needing tests with --test_tag_filters=-requires-docker. tags = [ diff --git a/tools/ci/test-bazel-remote-probe b/tools/ci/test-bazel-remote-probe new file mode 100755 index 000000000..fc5f53cb6 --- /dev/null +++ b/tools/ci/test-bazel-remote-probe @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Behavioral test for every subtree .bazel-remote-probe. +# +# The probes are sourced, not executed, and they decide whether a build gets +# --remote_cache flags. The decision is the whole contract, so assert it +# directly rather than relying on `bash -n`, which only checks syntax. +# +# grpcurl is stubbed so the test never touches a network. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +stub_dir="$(mktemp -d)" +trap 'rm -rf "${stub_dir}"' EXIT + +# Stubbed grpcurl: prints a capabilities response advertising ZSTD, which is +# what the probes grep for when deciding the cache is healthy. +cat > "${stub_dir}/grpcurl" <<'STUB' +#!/usr/bin/env bash +echo '{"cacheCapabilities":{"supportedCompressors":["IDENTITY","ZSTD"]}}' +STUB +chmod +x "${stub_dir}/grpcurl" + +fail=0 + +check() { # name, expectation (enabled|disabled), probe, env assignments... + local name="$1" want="$2" probe="$3"; shift 3 + local out + out="$( + env -i PATH="${stub_dir}:/usr/bin:/bin" HOME="${stub_dir}" \ + CI_PROJECT_DIR="${stub_dir}" "$@" \ + bash -c 'BAZEL_REMOTE_FLAGS=""; . "$0"; printf "%s" "${BAZEL_REMOTE_FLAGS}"' "${probe}" 2>/dev/null + )" || true + local got="disabled" + [[ "${out}" == *--remote_cache=* ]] && got="enabled" + if [[ "${got}" != "${want}" ]]; then + printf ' FAIL %-34s %-9s want=%s got=%s\n' "$(basename "$(dirname "$(dirname "${probe}")")")" "${name}" "${want}" "${got}" >&2 + fail=1 + fi +} + +probes=() +while IFS= read -r p; do probes+=("$p"); done < <( + find "${repo_root}/src" -name .bazel-remote-probe -type f | sort +) +if [[ "${#probes[@]}" -eq 0 ]]; then + echo "no .bazel-remote-probe scripts found" >&2 + exit 1 +fi + +for probe in "${probes[@]}"; do + # Host unset: the cache must be disabled, not probed at a default host. + # This is the regression this test exists for; the previous default pointed + # at a decommissioned host. + # + # TLS is turned off deliberately. With TLS on, an unset BAZEL_REMOTE_CA_PEM + # also disables the cache, so the case would pass even with a default host + # restored, and the test would be vacuous for exactly the regression it + # guards. With TLS off, a reintroduced default host would reach the stubbed + # grpcurl and report enabled, which is what makes this assertion meaningful. + check "host-unset" disabled "${probe}" NVCF_BAZEL_REMOTE_TLS=0 + + # Explicitly disabled, even with a host configured. + check "remote-off" disabled "${probe}" \ + NVCF_BAZEL_REMOTE=0 NVCF_BAZEL_REMOTE_HOST=cache.example.com + + # Configured host, TLS off so no CA pem is required: cache enabled. + check "host-set" enabled "${probe}" \ + NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \ + NVCF_BAZEL_REMOTE_PORT=8980 NVCF_BAZEL_REMOTE_TLS=0 + + # TLS requested but no CA pem: must decline rather than emit an unusable URL. + check "tls-no-ca" disabled "${probe}" \ + NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \ + NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1 +done + +if [[ "${fail}" -ne 0 ]]; then + echo "bazel-remote-probe behavioral tests failed" >&2 + exit 1 +fi +echo "bazel-remote-probe: ${#probes[@]} probes x 4 cases OK" From 4cef9a07ca6703e8a24f656bc58b1af72256b87e Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 25 Jul 2026 09:47:58 -0700 Subject: [PATCH 4/4] fix(ci): decline unreadable CA pems; stop the probe test masking crashes Two review findings, and the first exposed a real defect in the probes. The test discarded stderr and forced success, so a probe that CRASHED was indistinguishable from one that deliberately disabled the cache, and every disabled-case assertion would pass on a broken script. It now captures the exit status separately and fails on any non-zero exit, printing the probe's stderr. That immediately surfaced a bug: the probes guarded the TLS path with -z BAZEL_REMOTE_CA_PEM, which only catches an unset variable. A variable set to a path that does not exist fell through to the below and failed the job instead of cleanly declining the cache. The guard is now -r, covering unset and unreadable alike, and the message reports the offending path. Added the two missing cases: TLS with a readable CA pem (the success path that emits --tls_certificate, previously uncovered, asserted to produce both grpcs:// and the certificate flag so a silent downgrade to plaintext cannot pass), and TLS with an unreadable CA path. Six probes, six cases. Every assertion is backed by a negative control, each verified to fail when the defect is reintroduced: dead default host restored, CA guard reverted to -z, TLS downgraded to plaintext, and a probe that exits non-zero. Co-authored-by: Balaji Ganesan --- .../ess-agent/scripts/.bazel-remote-probe | 8 ++- .../nvca/scripts/.bazel-remote-probe | 8 ++- .../scripts/.bazel-remote-probe | 8 ++- .../helm-reval/scripts/.bazel-remote-probe | 8 ++- .../scripts/.bazel-remote-probe | 8 ++- .../scripts/.bazel-remote-probe | 8 ++- tools/ci/test-bazel-remote-probe | 61 +++++++++++++++++-- 7 files changed, 86 insertions(+), 23 deletions(-) diff --git a/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe b/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe index fe51d99c3..1a91f4a15 100755 --- a/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe +++ b/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe @@ -24,7 +24,7 @@ else host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does @@ -32,8 +32,10 @@ else # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then # Env var is set but the file is missing or unreadable. Treat it # the same as unset and degrade to local-only rather than failing diff --git a/src/compute-plane-services/nvca/scripts/.bazel-remote-probe b/src/compute-plane-services/nvca/scripts/.bazel-remote-probe index f5ad1640b..b36e06639 100755 --- a/src/compute-plane-services/nvca/scripts/.bazel-remote-probe +++ b/src/compute-plane-services/nvca/scripts/.bazel-remote-probe @@ -22,7 +22,7 @@ else host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does @@ -30,8 +30,10 @@ else # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." else probe_args=(-d '{"instance_name":""}') cache_url="grpc://${host}:${port}" diff --git a/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe b/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe index fe51d99c3..1a91f4a15 100755 --- a/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe +++ b/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe @@ -24,7 +24,7 @@ else host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does @@ -32,8 +32,10 @@ else # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then # Env var is set but the file is missing or unreadable. Treat it # the same as unset and degrade to local-only rather than failing diff --git a/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe b/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe index f5ad1640b..b36e06639 100755 --- a/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe +++ b/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe @@ -22,7 +22,7 @@ else host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does @@ -30,8 +30,10 @@ else # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." else probe_args=(-d '{"instance_name":""}') cache_url="grpc://${host}:${port}" diff --git a/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe b/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe index fe51d99c3..1a91f4a15 100755 --- a/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe +++ b/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe @@ -24,7 +24,7 @@ else host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does @@ -32,8 +32,10 @@ else # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then # Env var is set but the file is missing or unreadable. Treat it # the same as unset and degrade to local-only rather than failing diff --git a/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe b/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe index fe51d99c3..1a91f4a15 100755 --- a/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe +++ b/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe @@ -24,7 +24,7 @@ else host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does @@ -32,8 +32,10 @@ else # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then # Env var is set but the file is missing or unreadable. Treat it # the same as unset and degrade to local-only rather than failing diff --git a/tools/ci/test-bazel-remote-probe b/tools/ci/test-bazel-remote-probe index fc5f53cb6..a14eb4ceb 100755 --- a/tools/ci/test-bazel-remote-probe +++ b/tools/ci/test-bazel-remote-probe @@ -23,21 +23,56 @@ echo '{"cacheCapabilities":{"supportedCompressors":["IDENTITY","ZSTD"]}}' STUB chmod +x "${stub_dir}/grpcurl" +# A readable (contents irrelevant) CA pem for the TLS success path. +ca_pem="${stub_dir}/ca.pem" +printf -- '-----BEGIN CERTIFICATE-----\nstub\n-----END CERTIFICATE-----\n' > "${ca_pem}" + fail=0 check() { # name, expectation (enabled|disabled), probe, env assignments... local name="$1" want="$2" probe="$3"; shift 3 - local out + local out status svc + svc="$(basename "$(dirname "$(dirname "${probe}")")")" + + # Capture stdout and the exit status separately. Discarding stderr and + # forcing success would let a probe that CRASHES look identical to one that + # deliberately disables the cache, so every disabled-case assertion would + # pass on a broken script. + set +e out="$( env -i PATH="${stub_dir}:/usr/bin:/bin" HOME="${stub_dir}" \ CI_PROJECT_DIR="${stub_dir}" "$@" \ - bash -c 'BAZEL_REMOTE_FLAGS=""; . "$0"; printf "%s" "${BAZEL_REMOTE_FLAGS}"' "${probe}" 2>/dev/null - )" || true + bash -c 'set -e; BAZEL_REMOTE_FLAGS=""; . "$0"; printf "%s" "${BAZEL_REMOTE_FLAGS}"' \ + "${probe}" 2>"${stub_dir}/stderr.txt" + )" + status=$? + set -e + + if [[ "${status}" -ne 0 ]]; then + printf ' FAIL %-26s %-11s probe exited %s (expected a clean %s decision)\n' \ + "${svc}" "${name}" "${status}" "${want}" >&2 + sed 's/^/ /' "${stub_dir}/stderr.txt" >&2 || true + fail=1 + return + fi + local got="disabled" [[ "${out}" == *--remote_cache=* ]] && got="enabled" if [[ "${got}" != "${want}" ]]; then - printf ' FAIL %-34s %-9s want=%s got=%s\n' "$(basename "$(dirname "$(dirname "${probe}")")")" "${name}" "${want}" "${got}" >&2 + printf ' FAIL %-26s %-11s want=%s got=%s\n' "${svc}" "${name}" "${want}" "${got}" >&2 fail=1 + return + fi + + # For the TLS-enabled case, the cache URL must be grpcs:// and the probe must + # emit --tls_certificate. Without this the case would pass on a probe that + # silently downgraded to plaintext. + if [[ "${name}" == "tls-with-ca" ]]; then + if [[ "${out}" != *--tls_certificate=* || "${out}" != *grpcs://* ]]; then + printf ' FAIL %-26s %-11s expected grpcs:// and --tls_certificate, got: %s\n' \ + "${svc}" "${name}" "${out}" >&2 + fail=1 + fi fi } @@ -72,13 +107,29 @@ for probe in "${probes[@]}"; do NVCF_BAZEL_REMOTE_PORT=8980 NVCF_BAZEL_REMOTE_TLS=0 # TLS requested but no CA pem: must decline rather than emit an unusable URL. + # Bazel's gRPC client cannot verify a private cache cert from the system + # trust store, so a grpcs:// URL without --tls_certificate fails mid-build. check "tls-no-ca" disabled "${probe}" \ NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \ NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1 + + # TLS requested WITH a readable CA pem: the success path that actually emits + # the certificate flag. Without this case nothing covers the grpcs:// branch. + check "tls-with-ca" enabled "${probe}" \ + NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \ + NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1 \ + BAZEL_REMOTE_CA_PEM="${ca_pem}" + + # TLS requested with an UNREADABLE CA path: must decline, not emit a flag + # pointing at a file that does not exist. + check "tls-bad-ca" disabled "${probe}" \ + NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \ + NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1 \ + BAZEL_REMOTE_CA_PEM="${stub_dir}/definitely-missing.pem" done if [[ "${fail}" -ne 0 ]]; then echo "bazel-remote-probe behavioral tests failed" >&2 exit 1 fi -echo "bazel-remote-probe: ${#probes[@]} probes x 4 cases OK" +echo "bazel-remote-probe: ${#probes[@]} probes x 6 cases OK"