diff --git a/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe b/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe index 5812cc017..1a91f4a15 100755 --- a/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe +++ b/src/compute-plane-services/ess-agent/scripts/.bazel-remote-probe @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -10,29 +10,32 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then # Env var is set but the file is missing or unreadable. Treat it # the same as unset and degrade to local-only rather than failing diff --git a/src/compute-plane-services/nvca/scripts/.bazel-remote-probe b/src/compute-plane-services/nvca/scripts/.bazel-remote-probe index 0f7d82a3c..b36e06639 100755 --- a/src/compute-plane-services/nvca/scripts/.bazel-remote-probe +++ b/src/compute-plane-services/nvca/scripts/.bazel-remote-probe @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -8,29 +8,32 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." else probe_args=(-d '{"instance_name":""}') cache_url="grpc://${host}:${port}" diff --git a/src/compute-plane-services/nvsnap/.bazelrc b/src/compute-plane-services/nvsnap/.bazelrc index 4c4f819b7..328970108 100644 --- a/src/compute-plane-services/nvsnap/.bazelrc +++ b/src/compute-plane-services/nvsnap/.bazelrc @@ -42,21 +42,20 @@ build:release --stamp # a second pipeline on the same runner skips work the first one # already did. Devs benefit too; the path is gitignored. # -# 2. Remote cache (opt-in via --config=remote). Points at nvcfbarn, -# the team Buildbarn at NVIDIA. Default is OFF in this repo -# because the nvsnap project has no probe wired yet -- enabling -# --config=remote on a runner that cannot reach nvcfbarn fails -# the build hard (Bazel's initial GetCapabilities RPC is not -# covered by --remote_local_fallback). CI sets +# 2. Remote cache (opt-in via --config=remote). The caller supplies the +# endpoint with --remote_cache; this file carries only the policy and +# tuning flags. Default is OFF in this repo because the nvsnap project +# has no probe wired yet -- enabling --config=remote on a runner that +# cannot reach the cache fails the build hard (Bazel's initial +# GetCapabilities RPC is not covered by --remote_local_fallback). CI sets # --config=remote-write explicitly in .gitlab-ci.yml; devs and # ad-hoc builds stay on disk-only. build --disk_cache=.bazel-cache/disk build --repository_cache=.bazel-cache/repo -# Read-only remote cache profile. Pulls hits from nvcfbarn but does -# not upload. Layer on top of this with --config=remote-write to +# Read-only remote cache profile. Pulls hits but does not upload. Supply +# the endpoint with --remote_cache; layer --config=remote-write on top to # also write back results. -build:remote --remote_cache=grpc://nvcfbarn.nvidia.com:8980 build:remote --remote_upload_local_results=false build:remote --remote_cache_compression build:remote --remote_local_fallback diff --git a/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe b/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe index 5812cc017..1a91f4a15 100755 --- a/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe +++ b/src/control-plane-services/function-autoscaler/scripts/.bazel-remote-probe @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -10,29 +10,32 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then # Env var is set but the file is missing or unreadable. Treat it # the same as unset and degrade to local-only rather than failing diff --git a/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe b/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe index 0f7d82a3c..b36e06639 100755 --- a/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe +++ b/src/control-plane-services/helm-reval/scripts/.bazel-remote-probe @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -8,29 +8,32 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." else probe_args=(-d '{"instance_name":""}') cache_url="grpc://${host}:${port}" diff --git a/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel b/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel index 7bf9b333f..de529a5f5 100644 --- a/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel +++ b/src/invocation-plane-services/grpc-proxy/proxy/geo/BUILD.bazel @@ -74,10 +74,12 @@ go_test( local = True, # external: testcontainers spins up the localstack Docker container, # so the test's effective inputs (Docker daemon, localstack image - # tag) live outside Bazel's hermetic action hash. `local = True` - # forces local-only execution but does NOT bypass result caching; - # the `external` tag is what stops nvcfbarn from pinning a green - # forever. requires-docker is kept so callers can filter out + # tag) live outside Bazel's hermetic action hash. `local = True` forces + # local, unsandboxed execution and, measured on Bazel 9.1.1, also stops + # the result being reused from the disk/remote cache. It does not disable + # every cache: the `external` tag is what prevents Bazel reusing a + # previously cached test result. requires-docker is kept so callers + # can filter out # docker-needing tests with --test_tag_filters=-requires-docker. tags = [ "external", diff --git a/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe b/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe index 5812cc017..1a91f4a15 100755 --- a/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe +++ b/src/invocation-plane-services/http-invocation/scripts/.bazel-remote-probe @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -10,29 +10,32 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then # Env var is set but the file is missing or unreadable. Treat it # the same as unset and degrade to local-only rather than failing diff --git a/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe b/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe index 5812cc017..1a91f4a15 100755 --- a/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe +++ b/src/invocation-plane-services/llm-api-gateway/scripts/.bazel-remote-probe @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Probe the nvcfbarn remote cache and export BAZEL_REMOTE_FLAGS with the +# Probe the configured remote cache and export BAZEL_REMOTE_FLAGS with the # right --remote_cache + --tls_certificate flags, or empty if the cache is # unreachable / disabled. Source this file (do NOT execute it) so the # exported var is visible in the parent shell. @@ -10,29 +10,32 @@ # Driven by four CI/CD variables; all default to safe values that fall back # to local-only cleanly: # NVCF_BAZEL_REMOTE "1" to enable, "0" to disable (default 1). -# NVCF_BAZEL_REMOTE_HOST cache host (default nvcfbarn.nvidia.com). +# NVCF_BAZEL_REMOTE_HOST cache host. Unset disables the cache (no default: +# the previous default host was decommissioned). # NVCF_BAZEL_REMOTE_PORT cache port (443 for the TLS sidecar). # NVCF_BAZEL_REMOTE_TLS "1" to use grpcs:// + --tls_certificate (default 1). # BAZEL_REMOTE_CA_PEM File-type CI variable holding the cache's PEM cert. # Required when TLS=1. BAZEL_REMOTE_FLAGS="" -if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ]; then - echo "[bazel-remote] NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}; remote cache disabled" +if [ "${NVCF_BAZEL_REMOTE:-1}" != "1" ] || [ -z "${NVCF_BAZEL_REMOTE_HOST:-}" ]; then + echo "[bazel-remote] remote cache disabled (NVCF_BAZEL_REMOTE=${NVCF_BAZEL_REMOTE:-unset}, NVCF_BAZEL_REMOTE_HOST=${NVCF_BAZEL_REMOTE_HOST:-unset}); building local-only" else - host="${NVCF_BAZEL_REMOTE_HOST:-nvcfbarn.nvidia.com}" + host="${NVCF_BAZEL_REMOTE_HOST}" port="${NVCF_BAZEL_REMOTE_PORT:-443}" tls="${NVCF_BAZEL_REMOTE_TLS:-1}" - if [ "${tls}" = "1" ] && [ -z "${BAZEL_REMOTE_CA_PEM:-}" ]; then + if [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM:-}" ]; then # TLS requires a CA pem. grpcurl can probe with `-insecure`, but # Bazel's own gRPC client cannot: with `grpcs://` and no # `--tls_certificate` it tries the system trust store, which does - # not contain the nvcfbarn self-signed cert, and fails mid-build + # not contain the cache's self-signed cert, and fails mid-build # with "General OpenSslEngine problem". Skip the cache cleanly # instead of greenlighting an unusable URL. Hit on # dns-cache/nvcf-unbound!41; set BAZEL_REMOTE_CA_PEM (file-type CI - # variable) to enable. - echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset; remote cache disabled. Set the file-type CI variable to enable." + # variable) to enable. The test is -r, not -z: a variable set to a + # path that does not exist would otherwise reach the `cp` below and + # fail the job instead of cleanly declining the cache. + echo "[bazel-remote] grpcs://${host}:${port} requested but BAZEL_REMOTE_CA_PEM is unset or unreadable (${BAZEL_REMOTE_CA_PEM:-unset}); remote cache disabled. Set the file-type CI variable to enable." elif [ "${tls}" = "1" ] && [ ! -r "${BAZEL_REMOTE_CA_PEM}" ]; then # Env var is set but the file is missing or unreadable. Treat it # the same as unset and degrade to local-only rather than failing diff --git a/tools/ci/test-bazel-remote-probe b/tools/ci/test-bazel-remote-probe new file mode 100755 index 000000000..a14eb4ceb --- /dev/null +++ b/tools/ci/test-bazel-remote-probe @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Behavioral test for every subtree .bazel-remote-probe. +# +# The probes are sourced, not executed, and they decide whether a build gets +# --remote_cache flags. The decision is the whole contract, so assert it +# directly rather than relying on `bash -n`, which only checks syntax. +# +# grpcurl is stubbed so the test never touches a network. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +stub_dir="$(mktemp -d)" +trap 'rm -rf "${stub_dir}"' EXIT + +# Stubbed grpcurl: prints a capabilities response advertising ZSTD, which is +# what the probes grep for when deciding the cache is healthy. +cat > "${stub_dir}/grpcurl" <<'STUB' +#!/usr/bin/env bash +echo '{"cacheCapabilities":{"supportedCompressors":["IDENTITY","ZSTD"]}}' +STUB +chmod +x "${stub_dir}/grpcurl" + +# A readable (contents irrelevant) CA pem for the TLS success path. +ca_pem="${stub_dir}/ca.pem" +printf -- '-----BEGIN CERTIFICATE-----\nstub\n-----END CERTIFICATE-----\n' > "${ca_pem}" + +fail=0 + +check() { # name, expectation (enabled|disabled), probe, env assignments... + local name="$1" want="$2" probe="$3"; shift 3 + local out status svc + svc="$(basename "$(dirname "$(dirname "${probe}")")")" + + # Capture stdout and the exit status separately. Discarding stderr and + # forcing success would let a probe that CRASHES look identical to one that + # deliberately disables the cache, so every disabled-case assertion would + # pass on a broken script. + set +e + out="$( + env -i PATH="${stub_dir}:/usr/bin:/bin" HOME="${stub_dir}" \ + CI_PROJECT_DIR="${stub_dir}" "$@" \ + bash -c 'set -e; BAZEL_REMOTE_FLAGS=""; . "$0"; printf "%s" "${BAZEL_REMOTE_FLAGS}"' \ + "${probe}" 2>"${stub_dir}/stderr.txt" + )" + status=$? + set -e + + if [[ "${status}" -ne 0 ]]; then + printf ' FAIL %-26s %-11s probe exited %s (expected a clean %s decision)\n' \ + "${svc}" "${name}" "${status}" "${want}" >&2 + sed 's/^/ /' "${stub_dir}/stderr.txt" >&2 || true + fail=1 + return + fi + + local got="disabled" + [[ "${out}" == *--remote_cache=* ]] && got="enabled" + if [[ "${got}" != "${want}" ]]; then + printf ' FAIL %-26s %-11s want=%s got=%s\n' "${svc}" "${name}" "${want}" "${got}" >&2 + fail=1 + return + fi + + # For the TLS-enabled case, the cache URL must be grpcs:// and the probe must + # emit --tls_certificate. Without this the case would pass on a probe that + # silently downgraded to plaintext. + if [[ "${name}" == "tls-with-ca" ]]; then + if [[ "${out}" != *--tls_certificate=* || "${out}" != *grpcs://* ]]; then + printf ' FAIL %-26s %-11s expected grpcs:// and --tls_certificate, got: %s\n' \ + "${svc}" "${name}" "${out}" >&2 + fail=1 + fi + fi +} + +probes=() +while IFS= read -r p; do probes+=("$p"); done < <( + find "${repo_root}/src" -name .bazel-remote-probe -type f | sort +) +if [[ "${#probes[@]}" -eq 0 ]]; then + echo "no .bazel-remote-probe scripts found" >&2 + exit 1 +fi + +for probe in "${probes[@]}"; do + # Host unset: the cache must be disabled, not probed at a default host. + # This is the regression this test exists for; the previous default pointed + # at a decommissioned host. + # + # TLS is turned off deliberately. With TLS on, an unset BAZEL_REMOTE_CA_PEM + # also disables the cache, so the case would pass even with a default host + # restored, and the test would be vacuous for exactly the regression it + # guards. With TLS off, a reintroduced default host would reach the stubbed + # grpcurl and report enabled, which is what makes this assertion meaningful. + check "host-unset" disabled "${probe}" NVCF_BAZEL_REMOTE_TLS=0 + + # Explicitly disabled, even with a host configured. + check "remote-off" disabled "${probe}" \ + NVCF_BAZEL_REMOTE=0 NVCF_BAZEL_REMOTE_HOST=cache.example.com + + # Configured host, TLS off so no CA pem is required: cache enabled. + check "host-set" enabled "${probe}" \ + NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \ + NVCF_BAZEL_REMOTE_PORT=8980 NVCF_BAZEL_REMOTE_TLS=0 + + # TLS requested but no CA pem: must decline rather than emit an unusable URL. + # Bazel's gRPC client cannot verify a private cache cert from the system + # trust store, so a grpcs:// URL without --tls_certificate fails mid-build. + check "tls-no-ca" disabled "${probe}" \ + NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \ + NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1 + + # TLS requested WITH a readable CA pem: the success path that actually emits + # the certificate flag. Without this case nothing covers the grpcs:// branch. + check "tls-with-ca" enabled "${probe}" \ + NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \ + NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1 \ + BAZEL_REMOTE_CA_PEM="${ca_pem}" + + # TLS requested with an UNREADABLE CA path: must decline, not emit a flag + # pointing at a file that does not exist. + check "tls-bad-ca" disabled "${probe}" \ + NVCF_BAZEL_REMOTE=1 NVCF_BAZEL_REMOTE_HOST=cache.example.com \ + NVCF_BAZEL_REMOTE_PORT=443 NVCF_BAZEL_REMOTE_TLS=1 \ + BAZEL_REMOTE_CA_PEM="${stub_dir}/definitely-missing.pem" +done + +if [[ "${fail}" -ne 0 ]]; then + echo "bazel-remote-probe behavioral tests failed" >&2 + exit 1 +fi +echo "bazel-remote-probe: ${#probes[@]} probes x 6 cases OK"